@kirrosh/zond 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +758 -3
- package/README.md +78 -15
- package/package.json +17 -10
- package/src/cli/argv.ts +122 -0
- package/src/cli/commands/add-api.ts +134 -0
- package/src/cli/commands/api/annotate/idempotency.ts +59 -0
- package/src/cli/commands/api/annotate/index.ts +525 -0
- package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
- package/src/cli/commands/api/annotate/overlay.ts +206 -0
- package/src/cli/commands/api/annotate/pagination.ts +60 -0
- package/src/cli/commands/api/annotate/prompts.ts +183 -0
- package/src/cli/commands/api/annotate/readback.ts +58 -0
- package/src/cli/commands/api/annotate/resources.ts +91 -0
- package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
- package/src/cli/commands/audit.ts +480 -0
- package/src/cli/commands/bootstrap.ts +710 -0
- package/src/cli/commands/catalog.ts +35 -0
- package/src/cli/commands/check.ts +348 -0
- package/src/cli/commands/checks.ts +756 -0
- package/src/cli/commands/ci-init.ts +55 -6
- package/src/cli/commands/clean.ts +212 -0
- package/src/cli/commands/cleanup.ts +262 -0
- package/src/cli/commands/completions.ts +192 -0
- package/src/cli/commands/coverage.ts +605 -132
- package/src/cli/commands/db.ts +180 -8
- package/src/cli/commands/describe.ts +37 -2
- package/src/cli/commands/discover.ts +1236 -0
- package/src/cli/commands/doctor.ts +607 -0
- package/src/cli/commands/fixtures.ts +402 -0
- package/src/cli/commands/generate.ts +420 -47
- package/src/cli/commands/init/agents-md.ts +61 -0
- package/src/cli/commands/init/bootstrap.ts +108 -0
- package/src/cli/commands/init/index.ts +244 -0
- package/src/cli/commands/init/skills.ts +98 -0
- package/src/cli/commands/init/templates/agents.md +77 -0
- package/src/cli/commands/init/templates/markdown.d.ts +4 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +397 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +210 -0
- package/src/cli/commands/init/templates/skills/zond.md +651 -0
- package/src/cli/commands/init/templates/zond-config.yml +14 -0
- package/src/cli/commands/prepare-fixtures.ts +135 -0
- package/src/cli/commands/probe/mass-assignment.ts +503 -0
- package/src/cli/commands/probe/security.ts +454 -0
- package/src/cli/commands/probe/static.ts +255 -0
- package/src/cli/commands/probe/webhooks.ts +161 -0
- package/src/cli/commands/probe.ts +459 -0
- package/src/cli/commands/reference.ts +87 -0
- package/src/cli/commands/refresh-api.ts +169 -0
- package/src/cli/commands/remove-api.ts +150 -0
- package/src/cli/commands/report-bundle.ts +318 -0
- package/src/cli/commands/report.ts +241 -0
- package/src/cli/commands/request.ts +379 -4
- package/src/cli/commands/run.ts +911 -33
- package/src/cli/commands/session.ts +244 -0
- package/src/cli/commands/use.ts +74 -0
- package/src/cli/index.ts +36 -607
- package/src/cli/json-envelope.ts +112 -3
- package/src/cli/json-schemas.ts +263 -0
- package/src/cli/program.ts +218 -0
- package/src/cli/resolve.ts +105 -0
- package/src/cli/status-filter.ts +124 -0
- package/src/cli/util/api-context.ts +85 -0
- package/src/cli/version.ts +8 -0
- package/src/core/anti-fp/bootstrap.ts +34 -0
- package/src/core/anti-fp/index.ts +33 -0
- package/src/core/anti-fp/registry.ts +44 -0
- package/src/core/anti-fp/rules/baseline-echo.ts +74 -0
- package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +52 -0
- package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +38 -0
- package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +35 -0
- package/src/core/anti-fp/rules/schemathesis/index.ts +24 -0
- package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +53 -0
- package/src/core/anti-fp/rules/subscription-gated/index.ts +11 -0
- package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +75 -0
- package/src/core/anti-fp/types.ts +68 -0
- package/src/core/checks/checks/_crud-helpers.ts +133 -0
- package/src/core/checks/checks/_negative_mutator.ts +133 -0
- package/src/core/checks/checks/_readback-helpers.ts +133 -0
- package/src/core/checks/checks/content_type_conformance.ts +39 -0
- package/src/core/checks/checks/cross_call_references.ts +134 -0
- package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
- package/src/core/checks/checks/idempotency_replay.ts +246 -0
- package/src/core/checks/checks/ignored_auth.ts +211 -0
- package/src/core/checks/checks/index.ts +65 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +273 -0
- package/src/core/checks/checks/missing_required_header.ts +40 -0
- package/src/core/checks/checks/negative_data_rejection.ts +45 -0
- package/src/core/checks/checks/not_a_server_error.ts +27 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +131 -0
- package/src/core/checks/checks/pagination_invariants.ts +238 -0
- package/src/core/checks/checks/positive_data_acceptance.ts +36 -0
- package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
- package/src/core/checks/checks/response_headers_conformance.ts +74 -0
- package/src/core/checks/checks/response_schema_conformance.ts +30 -0
- package/src/core/checks/checks/status_code_conformance.ts +61 -0
- package/src/core/checks/checks/unsupported_method.ts +63 -0
- package/src/core/checks/checks/use_after_free.ts +78 -0
- package/src/core/checks/index.ts +30 -0
- package/src/core/checks/mode.ts +79 -0
- package/src/core/checks/recommended-action.ts +64 -0
- package/src/core/checks/registry.ts +78 -0
- package/src/core/checks/runner.ts +874 -0
- package/src/core/checks/sarif.ts +230 -0
- package/src/core/checks/stateful.ts +121 -0
- package/src/core/checks/types.ts +189 -0
- package/src/core/classifier/recommended-action.ts +222 -0
- package/src/core/context/current.ts +51 -0
- package/src/core/context/session.ts +78 -0
- package/src/core/coverage/loader.ts +185 -0
- package/src/core/coverage/reasons.ts +300 -0
- package/src/core/diagnostics/db-analysis.ts +161 -12
- package/src/core/diagnostics/failure-class.ts +120 -0
- package/src/core/diagnostics/failure-hints.ts +212 -9
- package/src/core/diagnostics/spec-pointer.ts +99 -0
- package/src/core/diagnostics/suggested-fixes.ts +156 -0
- package/src/core/exporter/case-study/index.ts +270 -0
- package/src/core/exporter/curl.ts +40 -0
- package/src/core/exporter/exporter.ts +48 -0
- package/src/core/exporter/html-report/escape.ts +24 -0
- package/src/core/exporter/html-report/index.ts +479 -0
- package/src/core/exporter/html-report/script.ts +100 -0
- package/src/core/exporter/html-report/styles.ts +408 -0
- package/src/core/generator/chunker.ts +53 -15
- package/src/core/generator/coverage-phase.ts +0 -0
- package/src/core/generator/create-body.ts +89 -0
- package/src/core/generator/data-factory.ts +490 -33
- package/src/core/generator/describe.ts +1 -1
- package/src/core/generator/fixtures-builder.ts +325 -0
- package/src/core/generator/index.ts +7 -5
- package/src/core/generator/openapi-reader.ts +55 -3
- package/src/core/generator/path-param-disambig.ts +114 -0
- package/src/core/generator/resources-builder.ts +648 -0
- package/src/core/generator/schema-utils.ts +11 -3
- package/src/core/generator/serializer.ts +114 -15
- package/src/core/generator/suite-generator.ts +484 -77
- package/src/core/generator/types.ts +8 -0
- package/src/core/identity/identity-file.ts +129 -0
- package/src/core/lint/affects.ts +28 -0
- package/src/core/lint/config.ts +96 -0
- package/src/core/lint/format.ts +42 -0
- package/src/core/lint/index.ts +94 -0
- package/src/core/lint/reporter.ts +128 -0
- package/src/core/lint/rules/consistency.ts +158 -0
- package/src/core/lint/rules/heuristics.ts +97 -0
- package/src/core/lint/rules/strictness.ts +109 -0
- package/src/core/lint/types.ts +96 -0
- package/src/core/lint/walker.ts +248 -0
- package/src/core/meta/meta-store.ts +6 -73
- package/src/core/output/README.md +91 -0
- package/src/core/output/index.ts +13 -0
- package/src/core/output/run.ts +126 -0
- package/src/core/output/types.ts +129 -0
- package/src/core/parser/env-interpolation.ts +104 -0
- package/src/core/parser/filter.ts +57 -0
- package/src/core/parser/schema.ts +132 -5
- package/src/core/parser/types.ts +29 -2
- package/src/core/parser/variables.ts +0 -0
- package/src/core/parser/yaml-parser.ts +108 -13
- package/src/core/probe/bootstrap.ts +34 -0
- package/src/core/probe/dry-run-envelope.ts +57 -0
- package/src/core/probe/mass-assignment-probe-class.ts +198 -0
- package/src/core/probe/mass-assignment-probe.ts +1122 -0
- package/src/core/probe/mass-assignment-template.ts +212 -0
- package/src/core/probe/method-probe.ts +164 -0
- package/src/core/probe/method-shared.ts +69 -0
- package/src/core/probe/negative-probe.ts +691 -0
- package/src/core/probe/orphan-tracker.ts +188 -0
- package/src/core/probe/path-discovery.ts +440 -0
- package/src/core/probe/probe-harness.ts +120 -0
- package/src/core/probe/registry.ts +89 -0
- package/src/core/probe/runner.ts +136 -0
- package/src/core/probe/security-probe-class.ts +201 -0
- package/src/core/probe/security-probe.ts +1453 -0
- package/src/core/probe/shared.ts +505 -0
- package/src/core/probe/static-probe-class.ts +125 -0
- package/src/core/probe/types.ts +165 -0
- package/src/core/probe/verdict-aggregator.ts +33 -0
- package/src/core/probe/webhooks-probe.ts +284 -0
- package/src/core/reporter/console.ts +69 -4
- package/src/core/reporter/index.ts +2 -3
- package/src/core/reporter/json.ts +15 -2
- package/src/core/reporter/junit.ts +27 -12
- package/src/core/reporter/ndjson.ts +37 -0
- package/src/core/reporter/types.ts +3 -0
- package/src/core/runner/assertions.ts +62 -2
- package/src/core/runner/async-pool.ts +108 -0
- package/src/core/runner/auth-path.ts +8 -0
- package/src/core/runner/ci-context.ts +72 -0
- package/src/core/runner/executor.ts +391 -52
- package/src/core/runner/form-encode.ts +51 -0
- package/src/core/runner/http-client.ts +115 -7
- package/src/core/runner/learn-drift.ts +293 -0
- package/src/core/runner/preflight-vars.ts +149 -0
- package/src/core/runner/progress-tracker.ts +73 -0
- package/src/core/runner/rate-limiter.ts +203 -0
- package/src/core/runner/run-kind.ts +39 -0
- package/src/core/runner/schema-validator.ts +312 -0
- package/src/core/runner/send-request.ts +153 -20
- package/src/core/runner/types.ts +38 -0
- package/src/core/secrets/registry.ts +164 -0
- package/src/core/secrets/secrets-file.ts +115 -0
- package/src/core/selectors/operation-filter.ts +144 -0
- package/src/core/setup-api.ts +419 -17
- package/src/core/severity/category.ts +94 -0
- package/src/core/severity/index.ts +121 -0
- package/src/core/spec/layers.ts +154 -0
- package/src/core/util/format-eta.ts +21 -0
- package/src/core/utils.ts +5 -1
- package/src/core/workspace/config.ts +129 -0
- package/src/core/workspace/manifest.ts +283 -0
- package/src/core/workspace/output-rotation.ts +62 -0
- package/src/core/workspace/root.ts +94 -0
- package/src/core/workspace/triage-path.ts +87 -0
- package/src/db/lint-runs.ts +47 -0
- package/src/db/migrate.ts +126 -0
- package/src/db/migrations/0001_run_kind.sql +25 -0
- package/src/db/migrations/sql.d.ts +4 -0
- package/src/db/queries/collections.ts +133 -0
- package/src/db/queries/coverage.ts +9 -0
- package/src/db/queries/dashboard.ts +59 -0
- package/src/db/queries/results.ts +128 -0
- package/src/db/queries/runs.ts +235 -0
- package/src/db/queries/sessions.ts +42 -0
- package/src/db/queries/settings.ts +28 -0
- package/src/db/queries/types.ts +172 -0
- package/src/db/queries.ts +72 -802
- package/src/db/schema.ts +179 -48
- package/src/cli/commands/export.ts +0 -144
- package/src/cli/commands/guide.ts +0 -127
- package/src/cli/commands/init.ts +0 -57
- package/src/cli/commands/serve.ts +0 -81
- package/src/cli/commands/sync.ts +0 -269
- package/src/cli/commands/update.ts +0 -189
- package/src/cli/commands/validate.ts +0 -34
- package/src/core/exporter/postman.ts +0 -963
- package/src/core/generator/guide-builder.ts +0 -253
- package/src/core/meta/types.ts +0 -21
- package/src/core/parser/index.ts +0 -21
- package/src/core/runner/execute-run.ts +0 -132
- package/src/core/runner/index.ts +0 -12
- package/src/core/sync/spec-differ.ts +0 -38
- package/src/web/data/collection-state.ts +0 -362
- package/src/web/routes/api.ts +0 -314
- package/src/web/routes/dashboard.ts +0 -350
- package/src/web/routes/runs.ts +0 -64
- package/src/web/schemas.ts +0 -121
- package/src/web/server.ts +0 -134
- package/src/web/static/htmx.min.cjs +0 -1
- package/src/web/static/style.css +0 -1148
- package/src/web/views/endpoints-tab.ts +0 -174
- package/src/web/views/explorer-tab.ts +0 -402
- package/src/web/views/health-strip.ts +0 -92
- package/src/web/views/layout.ts +0 -48
- package/src/web/views/results.ts +0 -210
- package/src/web/views/runs-tab.ts +0 -126
- package/src/web/views/suites-tab.ts +0 -181
|
@@ -0,0 +1,651 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: zond
|
|
3
|
+
description: |
|
|
4
|
+
Primary skill for any zond work. Loads on workspace touch
|
|
5
|
+
(`apis/<name>/`, `.env.yaml`, `.api-fixtures.yaml`) AND on intent —
|
|
6
|
+
full API audit, "find bugs", "raise coverage", "test the whole API",
|
|
7
|
+
contract-drift check, schema-drift detection, post-deploy regression,
|
|
8
|
+
case study, single-flow scenario authoring, fixture pack. Hand off to
|
|
9
|
+
`zond-checks` for depth-check reference / SARIF / stateful invariants,
|
|
10
|
+
to `zond-triage` for "what broke in the last run".
|
|
11
|
+
allowed-tools: [Read, Write, Edit, Bash(zond *), Bash(bunx zond *), Bash(sqlite3 *)]
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# zond — Primary skill
|
|
15
|
+
|
|
16
|
+
CLI-only. Run `zond --version` first; if missing:
|
|
17
|
+
`curl -fsSL https://raw.githubusercontent.com/kirrosh/zond/master/install.sh | sh`.
|
|
18
|
+
|
|
19
|
+
For depth-check reference (the catalog of conformance / security /
|
|
20
|
+
stateful checks, SARIF output, m-20 invariants) — see `zond-checks`.
|
|
21
|
+
For triage of a failing run — see `zond-triage`.
|
|
22
|
+
|
|
23
|
+
## Iron rules (read once, apply always)
|
|
24
|
+
|
|
25
|
+
- **NEVER read raw OpenAPI** with Read/cat/grep. The workspace has
|
|
26
|
+
pre-built artifacts (catalog/resources/fixtures) — use those. Drop
|
|
27
|
+
into `apis/<name>/spec.json` only when probe-class authoring needs
|
|
28
|
+
full schemas.
|
|
29
|
+
- **NEVER `curl` or `wget`** — use `zond request <method> <url> --api <name>`
|
|
30
|
+
for ad-hoc HTTP so it lands in the run DB and respects auth. Never
|
|
31
|
+
shell-substitute the token by hand (`$(yq …)` is blocked by the sandbox).
|
|
32
|
+
- **NEVER hardcode tokens** — put them in `apis/<name>/.secrets.yaml`
|
|
33
|
+
(auto-gitignored), reference from `.env.yaml` as `@secret:auth_token`.
|
|
34
|
+
Plain `${SHELL_VAR}` references also work. Tests read the resolved
|
|
35
|
+
value as `{{auth_token}}`.
|
|
36
|
+
- **NEVER read `.secrets.yaml` directly.** Use `zond doctor --api <name> --json`
|
|
37
|
+
— reports `set | unset` and value length only.
|
|
38
|
+
- **NEVER edit `.api-*.yaml`** (catalog / resources / fixtures) by hand —
|
|
39
|
+
regenerated by `zond refresh-api`. `.api-resources.local.yaml` IS the
|
|
40
|
+
hand-editable / annotate-target overlay; safe to edit. `.env.yaml`
|
|
41
|
+
values are editable (see ARV-114).
|
|
42
|
+
- **NEVER invent endpoints.** Only use entries from `.api-catalog.yaml`.
|
|
43
|
+
- **NEVER run destructive ops on a shared / production org without
|
|
44
|
+
`--dry-run` first.** Probes, `prepare-fixtures --apply`, `cleanup` hit
|
|
45
|
+
live APIs and can delete user data. `--dry-run` once → inspect → drop
|
|
46
|
+
the flag.
|
|
47
|
+
- **NEVER share triage artefacts** outbound without `--redact-identity`.
|
|
48
|
+
Secret-redaction is on by default; identity values (org/member slugs,
|
|
49
|
+
real ids) need the extra flag.
|
|
50
|
+
- **NEVER report cleanup failure as an API bug.** POST 200 → DELETE 5xx
|
|
51
|
+
is *probably* a fixture-isolation issue (orphan accumulation, race).
|
|
52
|
+
Re-run with `--no-cleanup` or isolated namespace before filing.
|
|
53
|
+
- **`recommended_action: report_backend_bug` / 5xx → STOP** in interactive
|
|
54
|
+
mode (surface the request/response, get a decision; never `expect:`-mask).
|
|
55
|
+
In autonomous / loop / audit-sweep mode (no user-in-the-loop), log to
|
|
56
|
+
`api-bugs-<NN>.md`, continue the sweep — bailing on bug #1 forfeits #2..N.
|
|
57
|
+
- **CRUD-run ≥80% 401/403 / `permission_denied` → `env_issue`, not bug.**
|
|
58
|
+
Token scope issue. Confirm via `zond db diagnose <run-id> --env-only`;
|
|
59
|
+
do not generate case-studies, do not `expect:`-mask.
|
|
60
|
+
- **MUST run `zond doctor --api <name> --missing-only` before generating
|
|
61
|
+
fixtures or touching `.env.yaml`** — identifies unfilled keys early.
|
|
62
|
+
- **Cascade cap is 8 passes** (`zond prepare-fixtures --cascade [--seed]`)
|
|
63
|
+
— never override without a written reason.
|
|
64
|
+
|
|
65
|
+
## Workspace artifact model
|
|
66
|
+
|
|
67
|
+
After `zond add api <name> --spec <path|url>`, `apis/<name>/` contains:
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
spec.json ← dereferenced OpenAPI (machine source)
|
|
71
|
+
.api-catalog.yaml ← endpoint index (method, path, params)
|
|
72
|
+
.api-resources.yaml ← CRUD chains + FK deps (auto)
|
|
73
|
+
.api-resources.local.yaml ← overlay: extensions + annotate output (hand-edit OK; appears after first `zond api annotate apply` — or create by hand to use as overlay)
|
|
74
|
+
.api-fixtures.yaml ← MANIFEST: required vars + descriptions (read-only)
|
|
75
|
+
.env.yaml ← VALUES: variable values (user / prepare-fixtures writes)
|
|
76
|
+
.secrets.yaml ← raw secret values (auto-gitignored)
|
|
77
|
+
.identity.yaml ← non-secret identifying values (gitignored)
|
|
78
|
+
tests/ ← generated/handwritten suites
|
|
79
|
+
scenarios/ ← handwritten flow suites
|
|
80
|
+
probes/ ← probe-emitted suites
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Manifest vs values (the #1 source of confusion)
|
|
84
|
+
|
|
85
|
+
**`.api-fixtures.yaml` = the list of vars this API needs (manifest).
|
|
86
|
+
`.env.yaml` = their values.**
|
|
87
|
+
|
|
88
|
+
| Op | Touches manifest? | Touches env? |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| `zond add api` / `refresh-api` | rebuilds | seeds skeleton |
|
|
91
|
+
| `zond generate` | extends (adds new `{{vars}}` discovered) | does **not** modify |
|
|
92
|
+
| `zond prepare-fixtures` | reads | writes values |
|
|
93
|
+
| `zond fixtures add` / `import` | reads | writes values (manual) |
|
|
94
|
+
| user editing | never | always |
|
|
95
|
+
|
|
96
|
+
- `{{var}}` in a generated test but NOT in `.api-fixtures.yaml` is a
|
|
97
|
+
bug in the manifest builder / generator. Don't "fix" it by adding to
|
|
98
|
+
`.env.yaml`.
|
|
99
|
+
- A key in `.env.yaml` but NOT in `.api-fixtures.yaml` is shadow —
|
|
100
|
+
`prepare-fixtures` warns `not in manifest, ignored`.
|
|
101
|
+
- "Generate should sync `.env.yaml`" is a rejected design (decision-7,
|
|
102
|
+
m-17).
|
|
103
|
+
|
|
104
|
+
### Manual fixture-bootstrap (when `prepare-fixtures --seed` can't help)
|
|
105
|
+
|
|
106
|
+
`prepare-fixtures --seed` requires a discoverable list endpoint or a
|
|
107
|
+
postable create. Path-id ids that live only in a vendor dashboard
|
|
108
|
+
(Stripe `cus_*`, GitHub PR numbers, Sentry issue ids) need a manual
|
|
109
|
+
hand-off:
|
|
110
|
+
|
|
111
|
+
| Input you have | Command |
|
|
112
|
+
|---|---|
|
|
113
|
+
| Concrete id values | `zond fixtures add <var>=<id> [--validate] --apply` |
|
|
114
|
+
| Curl from devtools / dashboard | `pbpaste \| zond fixtures import --from-curl --apply` (URL is matched against spec paths to extract `{var}` bindings) |
|
|
115
|
+
|
|
116
|
+
`--validate` GETs the spec's read-by-id endpoint and classifies the
|
|
117
|
+
fixture as `live` / `stale` / `unknown` so dead ids don't silently
|
|
118
|
+
break later runs (ARV-32). Both commands write to `apis/<name>/.env.yaml`
|
|
119
|
+
with a `.env.yaml.bak` backup, mirroring `prepare-fixtures --apply`.
|
|
120
|
+
|
|
121
|
+
### When to read which file
|
|
122
|
+
|
|
123
|
+
| To know… | Read |
|
|
124
|
+
|---|---|
|
|
125
|
+
| What endpoints exist | `.api-catalog.yaml` |
|
|
126
|
+
| How resources chain (CRUD, FK, ETag) | `.api-resources.yaml` |
|
|
127
|
+
| What vars are needed and why | `.api-fixtures.yaml` |
|
|
128
|
+
| What var values are set | `.env.yaml` |
|
|
129
|
+
| Annotation overlay (seed_body, lifecycle, pagination, readback_diff) | `.api-resources.local.yaml` |
|
|
130
|
+
| Full request/response schema (only when needed) | `spec.json` |
|
|
131
|
+
|
|
132
|
+
## --json envelope (uniform across commands)
|
|
133
|
+
|
|
134
|
+
```jsonc
|
|
135
|
+
{
|
|
136
|
+
"ok": true|false,
|
|
137
|
+
"command": "<name>",
|
|
138
|
+
"data": { /* command-specific */ },
|
|
139
|
+
"warnings": ["..."],
|
|
140
|
+
"errors": [{ "code": "ZondErrorCode", "message": "...", "details": {} }],
|
|
141
|
+
"exit_code": 0|1|2
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Route on `errors[].code`, not message. Stdout discipline: `--json`
|
|
146
|
+
emits only the envelope on stdout; progress/hints to stderr.
|
|
147
|
+
|
|
148
|
+
`--json` ≠ `--report json`. `--json` wraps the **command result**;
|
|
149
|
+
`--report json` wraps the **test-run artefact**. `zond run` doesn't
|
|
150
|
+
accept `--json` — use `--report json`.
|
|
151
|
+
|
|
152
|
+
## Entry points — pick by intent
|
|
153
|
+
|
|
154
|
+
| User asked… | Start at |
|
|
155
|
+
|---|---|
|
|
156
|
+
| "audit this API", "test the whole API", "raise coverage" | Phase 0 → `zond audit --api <name>` (one-shot) OR walk Phase 0–8 |
|
|
157
|
+
| "find bugs", "test for 5xx", "probe sweep" | Phase 0 → Phase 7 (Probes) |
|
|
158
|
+
| "security only / SSRF / CRLF" | Phase 7.2 directly with `--dry-run` first |
|
|
159
|
+
| "deep depth-checks", "SARIF", "stateful invariants" | Hand off → `zond-checks` |
|
|
160
|
+
| "what broke in last run", "diagnose run X" | Hand off → `zond-triage` |
|
|
161
|
+
| "write a test for X flow", "smoke this checkout" | Phase S (Scenarios) |
|
|
162
|
+
| "what vars does this API need" | `zond doctor --api <name> --json` |
|
|
163
|
+
| "share results", "case study" | Phase 9 (Share) |
|
|
164
|
+
| "workspace looks messy" | `zond clean --api <name>` (dry-run → `--force`) |
|
|
165
|
+
|
|
166
|
+
## Phase 0 — Orient
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
zond doctor --api <name> --json # gaps + freshness
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Then read the three artifacts (NOT the raw spec):
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
cat apis/<name>/.api-catalog.yaml | head -80
|
|
176
|
+
cat apis/<name>/.api-resources.yaml
|
|
177
|
+
cat apis/<name>/.api-fixtures.yaml
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
If doctor reports stale → `zond refresh-api <name>`.
|
|
181
|
+
|
|
182
|
+
## Phase 1 — Fixture pack
|
|
183
|
+
|
|
184
|
+
`zond prepare-fixtures` walks `.api-resources.yaml`, hits each owner
|
|
185
|
+
list endpoint, fills `.env.yaml`. Always `--apply`; add `--cascade --seed`
|
|
186
|
+
for empty workspaces.
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
zond prepare-fixtures --api <name> # dry-run preview
|
|
190
|
+
zond prepare-fixtures --api <name> --apply # single-pass
|
|
191
|
+
zond prepare-fixtures --api <name> --apply --cascade --seed # cascade + POST-create when list is empty
|
|
192
|
+
zond prepare-fixtures --api <name> --refresh # = --verify --apply: drop stale ids, re-resolve
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
`--seed` POSTs to create endpoints when a list returns `[]`, using a
|
|
196
|
+
schema-derived body. Bracket-notation nested params (Stripe's
|
|
197
|
+
`card[number]`, `items[0][price]`) are a known weak spot (ARV-196) —
|
|
198
|
+
some APIs will 400 here; **don't ask the user to fill missing ids
|
|
199
|
+
manually first**, jump to Phase 2 (annotate seed_body) instead.
|
|
200
|
+
|
|
201
|
+
If `--seed` converges but vars stay UNSET, the reason is outside the
|
|
202
|
+
API path (email verify / paid plan / SCIM / TOS limits). Document the
|
|
203
|
+
gap; don't loop further.
|
|
204
|
+
|
|
205
|
+
Editing `.env.yaml` by hand is the sanctioned fallback when the CLI
|
|
206
|
+
cannot harvest a value (write-only ingest endpoints, SDK-only ids,
|
|
207
|
+
slugs that live in user's project config). Touch values only — never
|
|
208
|
+
add keys not in `.api-fixtures.yaml`.
|
|
209
|
+
|
|
210
|
+
## Phase 2 — Annotate (NEW, m-20 / ARV-187)
|
|
211
|
+
|
|
212
|
+
**Critical**: zond itself does NOT call any LLM. The flow is
|
|
213
|
+
**dump → agent fills YAML → apply**. The agent (you) generates the
|
|
214
|
+
overlay; zond validates and writes it into `.api-resources.local.yaml`.
|
|
215
|
+
|
|
216
|
+
`zond api annotate` has 6 dump-subcommands (one per aspect). Pick what
|
|
217
|
+
your downstream check needs:
|
|
218
|
+
|
|
219
|
+
| Aspect | Dump produces | Feeds into |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| `--seed-bodies` | request-body schema per resource | `prepare-fixtures --seed`, all stateful checks |
|
|
222
|
+
| `--readback` | create+read pair (write-shape vs read-shape) | `cross_call_references` ignore_fields, write_to_read_map |
|
|
223
|
+
| `--idempotency` | header candidates | `idempotency_replay` (header, scope, body_hash_fields) |
|
|
224
|
+
| `--pagination` | list-endpoint cursor/page detection | `pagination_invariants` (type, cursor_param) |
|
|
225
|
+
| `--lifecycle` | action-endpoint candidates per resource | `lifecycle_transitions` (states, transitions) |
|
|
226
|
+
| `--resources` | orphan endpoints not in resource graph | `.api-resources.local.yaml` extensions |
|
|
227
|
+
|
|
228
|
+
**Workflow** (for ANY aspect):
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
# 1. Dump — emits JSON slices on stdout. Restrict scope with --only.
|
|
232
|
+
zond api annotate dump --api <name> --seed-bodies --only r1,r2 > /tmp/dump.json
|
|
233
|
+
|
|
234
|
+
# 2. Agent reads the dump and writes a YAML response file (see format below).
|
|
235
|
+
|
|
236
|
+
# 3. Apply — dry-run first (diff), then --yes to write.
|
|
237
|
+
zond api annotate apply --api <name> --seed-bodies --input /tmp/out.yaml # dry-run + diff
|
|
238
|
+
zond api annotate apply --api <name> --seed-bodies --input /tmp/out.yaml --yes # write
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
YAML response format — a top-level list of entries, each one resource:
|
|
242
|
+
|
|
243
|
+
```yaml
|
|
244
|
+
- resource: <name>
|
|
245
|
+
seed_body: # (--seed-bodies)
|
|
246
|
+
content_type: application/x-www-form-urlencoded
|
|
247
|
+
body:
|
|
248
|
+
amount: 1000
|
|
249
|
+
currency: usd
|
|
250
|
+
rationale: 'why this shape' # optional, helps future review
|
|
251
|
+
confidence: high # high|medium|low — optional
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
Same shape for `readback_diff`, `idempotency`, `pagination`,
|
|
255
|
+
`lifecycle` blocks (see `zond-checks` for the per-aspect schemas).
|
|
256
|
+
|
|
257
|
+
**When to annotate**: run dump+apply BEFORE Phase 6 (stateful checks)
|
|
258
|
+
for any production API. Without annotation, stateful checks fall back
|
|
259
|
+
to defaults and miss API-specific quirks (custom pagination params,
|
|
260
|
+
non-standard lifecycle field names, write-only fields in create body).
|
|
261
|
+
|
|
262
|
+
**For Stripe-style form-encoded APIs**: `--seed-bodies` is the single
|
|
263
|
+
biggest win — it fixes `--seed` 400s for the bulk of resources.
|
|
264
|
+
|
|
265
|
+
## Phase 3 — Generate tests
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
zond generate apis/<name>/spec.json --output apis/<name>/tests [--tag <spec-tag>] [--uncovered-only]
|
|
269
|
+
zond check tests apis/<name>/tests
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
`generate` fills bodies with `{{$randomString}}`. Format-strict APIs
|
|
273
|
+
reject many — that's a **test-fix**, not a backend bug. Pair with
|
|
274
|
+
Phase 1 fixtures + Phase 2 annotation.
|
|
275
|
+
|
|
276
|
+
If a CRUD chain you expected is missing, run
|
|
277
|
+
`zond generate <spec> --explain` — diagnostic table shows every POST
|
|
278
|
+
endpoint with verdict (no GET-by-id, non-`{id}` item param,
|
|
279
|
+
trailing-slash mismatch, …).
|
|
280
|
+
|
|
281
|
+
## Phase 4 — Spec-lint (hygiene, separate workflow)
|
|
282
|
+
|
|
283
|
+
```bash
|
|
284
|
+
zond lint --api <name> # 0 network requests
|
|
285
|
+
zond lint --api <name> --json | jq '.data.stats'
|
|
286
|
+
# Equivalent: `zond check spec --api <name>`
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
ARV-255 (m-21 pivot): **spec-lint is hygiene, not security or
|
|
290
|
+
contract.** It runs separately from `zond audit` / `zond probe` /
|
|
291
|
+
`zond checks run` — those produce the security/reliability/contract
|
|
292
|
+
report, lint stays out of that pile.
|
|
293
|
+
|
|
294
|
+
Severity capped at **LOW / INFO** by design:
|
|
295
|
+
- LOW: real spec violations (format mismatch in example, missing
|
|
296
|
+
path-param format, response without schema).
|
|
297
|
+
- INFO: style and documentation gaps (additionalProperties missing,
|
|
298
|
+
naming, missing examples).
|
|
299
|
+
|
|
300
|
+
No HIGH/MEDIUM ever — static analysis has no runtime evidence, so the
|
|
301
|
+
severity matrix forbids escalation. `--strict` opts back into a
|
|
302
|
+
non-zero exit when any LOW lands.
|
|
303
|
+
|
|
304
|
+
## Phase 5 — Run (sanity → smoke → CRUD)
|
|
305
|
+
|
|
306
|
+
Wrap multi-run sweeps in a session so `/runs` shows one row:
|
|
307
|
+
|
|
308
|
+
```bash
|
|
309
|
+
zond session start --label "smoke + probes"
|
|
310
|
+
zond run apis/<name>/tests --tag sanity --report json # 5.1 sanity
|
|
311
|
+
zond run apis/<name>/tests --safe --report json # 5.2 smoke (GET-only)
|
|
312
|
+
zond run apis/<name>/tests --tag crud,setup --validate-schema --report json # 5.3 full CRUD
|
|
313
|
+
zond run apis/<name>/tests --tag positive --include 'path:^/emails' # 5.4 narrow
|
|
314
|
+
zond session end
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
**Always pass `--validate-schema` for CRUD** — contract drift is
|
|
318
|
+
invisible without it. `schema_violation` failures are real backend bugs;
|
|
319
|
+
treat them like 5xx.
|
|
320
|
+
|
|
321
|
+
Rate limit: `zond run` defaults to an adaptive limiter (no-op until
|
|
322
|
+
`RateLimit-*` headers appear). Pass `--rate-limit <N>` for a hard cap;
|
|
323
|
+
`--sequential` for old binaries.
|
|
324
|
+
|
|
325
|
+
**Long runs**: `zond run` shows a periodic progress line on stderr when
|
|
326
|
+
stdout/stderr is a TTY (`zond: [42s] 234/10927 steps (2%), 230 req, ~30
|
|
327
|
+
req/s, ETA ~5m`) — overwrites itself in place. Suppressed by `--quiet`
|
|
328
|
+
and on non-interactive output. For huge probe-suites (e.g. GitHub spec
|
|
329
|
+
→ ~10k probes under `--rate-limit 30` = 6+ min), this signals "alive,
|
|
330
|
+
not hung". `zond probe static` itself prints a scale-warning block with
|
|
331
|
+
ETA-at-rate-limit estimates and a `--max-per-endpoint K` sampling hint
|
|
332
|
+
when `totalProbes ≥ 2000`.
|
|
333
|
+
|
|
334
|
+
**Hard cap on requests**: pass `--max-requests N` to cap outgoing HTTP
|
|
335
|
+
calls across the whole run. Remaining steps short-circuit to `skip` with
|
|
336
|
+
`error: "max-requests-cap-reached"`. Each `retry_until` attempt counts as
|
|
337
|
+
one request. Useful for sampling huge probe runs (`--max-requests 500`)
|
|
338
|
+
and for CI time-boxing.
|
|
339
|
+
|
|
340
|
+
`--all`: fold every `apis/<name>/tests/` dir into one `runs.id` per
|
|
341
|
+
invocation (CI shape). CI context (commit_sha, branch, trigger=ci)
|
|
342
|
+
auto-stamped from env vars.
|
|
343
|
+
|
|
344
|
+
### Spec-drift learning tail (`--learn`)
|
|
345
|
+
|
|
346
|
+
After the initial CRUD run, **always** close with `--learn` /
|
|
347
|
+
`--learn-apply`. R09 measured a 28% → 58% pass-coverage jump from this
|
|
348
|
+
phase alone.
|
|
349
|
+
|
|
350
|
+
```bash
|
|
351
|
+
zond run apis/<name>/tests --learn # detect (read-only)
|
|
352
|
+
zond run apis/<name>/tests --learn-apply --learn-target test # rewrite expect.status
|
|
353
|
+
zond run apis/<name>/tests --learn-apply --learn-target drifts # allowlist in tolerated-drifts.yaml
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
**Caveat — `--learn-apply test` weakens assertions, not the code.**
|
|
357
|
+
Diff `apis/<name>/tests/*.yaml` + `tolerated-drifts.yaml` after every
|
|
358
|
+
apply; treat unexpected widenings as a review-blocker.
|
|
359
|
+
|
|
360
|
+
## Phase 6 — Stateful checks (m-20)
|
|
361
|
+
|
|
362
|
+
After Phase 2 annotation is applied, run the cross-call invariants.
|
|
363
|
+
See `zond-checks` for per-check semantics.
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
zond checks run --api <name> --check stateful --report ndjson
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
5 m-20 stateful checks:
|
|
370
|
+
- `cross_call_references` — POST→GET drift (uses readback_diff overlay)
|
|
371
|
+
- `idempotency_replay` — bit-identical replay on `Idempotency-Key`
|
|
372
|
+
- `pagination_invariants` — `?limit=N` + `?after=last_id` non-overlap
|
|
373
|
+
- `lifecycle_transitions` — declared state-machine validity
|
|
374
|
+
- `webhooks` (recipe-based, see `docs/recipes/webhook-receiver.md`)
|
|
375
|
+
|
|
376
|
+
**Iron rule**: don't run `--check stateful` without prior `api annotate`
|
|
377
|
+
review. Defaults catch the obvious; quirks need declared config.
|
|
378
|
+
|
|
379
|
+
## Phase 7 — Proactive bug hunting (probes)
|
|
380
|
+
|
|
381
|
+
```bash
|
|
382
|
+
zond probe static --api <name> # validation + methods (defaults)
|
|
383
|
+
zond probe mass-assignment --api <name> --emit-tests apis/<name>/probes/mass-assignment
|
|
384
|
+
zond probe security ssrf,crlf,open-redirect,prompt-injection --api <name> \
|
|
385
|
+
--emit-tests apis/<name>/probes/security
|
|
386
|
+
zond run apis/<name>/probes --report json
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
Flag: 5xx on null/empty/wrong-type body → missing validation. 2xx on
|
|
390
|
+
undeclared method → contract drift. `is_admin: true` echoed in response
|
|
391
|
+
→ HIGH from mass-assignment.
|
|
392
|
+
|
|
393
|
+
**Body-FK auto-discovery** — `mass-assignment` resolves required
|
|
394
|
+
`*_id`/`*_slug`/`*_uuid`/`*_key` fields by hitting sibling list endpoints.
|
|
395
|
+
If digest still says "unresolved body FKs:" — add to `.env.yaml` manually.
|
|
396
|
+
|
|
397
|
+
**Auto-path-FK** — same logic for path params (`{domain_id}`/`{webhook_id}`).
|
|
398
|
+
Cached per run. Pass `--no-discover` to opt out.
|
|
399
|
+
|
|
400
|
+
### 7.1 — Manual mass-assignment catch-up
|
|
401
|
+
|
|
402
|
+
`mass-assignment` digest splits findings: HIGH / MED / LOW /
|
|
403
|
+
INCONCLUSIVE (no valid body — fixture problem) / INCONCLUSIVE-5XX
|
|
404
|
+
(baseline crashed — already in validation-probe). Sweep INCONCLUSIVE
|
|
405
|
+
with `--emit-template`:
|
|
406
|
+
|
|
407
|
+
```bash
|
|
408
|
+
zond probe mass-assignment --api <name> --emit-template "POST:/<resource>" \
|
|
409
|
+
--output apis/<name>/probes/mass-assignment/<resource>.yaml
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
Emits `create → verify → cleanup` chain pre-filled with classic vectors
|
|
413
|
+
(`is_admin`, `role`, `owner_id`). If a privileged field survives the
|
|
414
|
+
round-trip GET → **HIGH**, file via `report bundle --include case-study`.
|
|
415
|
+
|
|
416
|
+
### 7.2 — Security probes
|
|
417
|
+
|
|
418
|
+
```bash
|
|
419
|
+
zond probe security ssrf,crlf --api <name> --dry-run # first run: which (endpoint, field)
|
|
420
|
+
zond probe security ssrf,crlf,open-redirect,prompt-injection --api <name>
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
Field autodetection: SSRF (`*_url`/`webhook`/`callback`/`format: uri`),
|
|
424
|
+
CRLF (`subject`/`*_prefix`/`name`), open-redirect (`redirect`/`next`).
|
|
425
|
+
|
|
426
|
+
Baseline-OK request first; if baseline ≠ 2xx → `INCONCLUSIVE-BASELINE`,
|
|
427
|
+
attacks skipped (kills 5×404 noise on scope-locked endpoints).
|
|
428
|
+
|
|
429
|
+
Verdict: HIGH (5xx OR payload echoed in 2xx — stored injection),
|
|
430
|
+
LOW (2xx, no echo — verify side-effects manually), OK (4xx).
|
|
431
|
+
|
|
432
|
+
**Cleanup is state-aware**: PUT/PATCH does `GET` → snapshot → attack →
|
|
433
|
+
restore. POST falls back to `DELETE` with eventual-consistency retry.
|
|
434
|
+
Restore failures → `## ⚠️ Cleanup failures` at top of digest.
|
|
435
|
+
|
|
436
|
+
CI exit: non-zero on HIGH > 0 OR `cleanup.error > 0`.
|
|
437
|
+
|
|
438
|
+
### 7.3 — Post-probe hygiene
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
zond prepare-fixtures --api <name> --refresh # drop stale FK ids
|
|
442
|
+
zond cleanup --orphans # retry DELETE for ~/.zond/orphans/
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
Skip only with `probe security --isolated` (isolated mode never attacks
|
|
446
|
+
seeded-fixture endpoints).
|
|
447
|
+
|
|
448
|
+
## Phase 8 — Coverage + spec drift
|
|
449
|
+
|
|
450
|
+
```bash
|
|
451
|
+
zond coverage --api <name> --union session # default for an audit window
|
|
452
|
+
zond coverage --api <name> --union since:1h # last hour
|
|
453
|
+
zond coverage --api <name> --union tag:smoke # tag-filtered
|
|
454
|
+
zond coverage --api <name> --fail-on-coverage 50 # CI gate (use hit-coverage)
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
**Two metrics**: `pass-coverage` (covered2xx — strict, CI gate) vs
|
|
458
|
+
`hit-coverage` (any stored result, breadth proxy). Three-bucket JSON:
|
|
459
|
+
`covered2xx`, `coveredButNon2xx` (hit but failed — usually fixture gap,
|
|
460
|
+
not API bug), `unhit`.
|
|
461
|
+
|
|
462
|
+
**Quality signals to gate CI on** (not raw pass-coverage):
|
|
463
|
+
|
|
464
|
+
| Signal | Command |
|
|
465
|
+
|---|---|
|
|
466
|
+
| Contract drift | `checks run --phase coverage` HIGH count == 0 |
|
|
467
|
+
| Stateful invariants | `checks run --check stateful` HIGH count == 0 |
|
|
468
|
+
| Auth / Authz / injection | `probe security` HIGH count == 0 |
|
|
469
|
+
| Mass-assignment | `probe mass-assignment` HIGH count == 0 |
|
|
470
|
+
| Response conformance | `run --validate-schema` `schema_violation` == 0 |
|
|
471
|
+
| Drift allowlist review | `git diff apis/<name>/tolerated-drifts.yaml` |
|
|
472
|
+
|
|
473
|
+
## Phase 9 — Share findings
|
|
474
|
+
|
|
475
|
+
```bash
|
|
476
|
+
zond report export <run-id> # default: triage/<api>/run-<id>/
|
|
477
|
+
zond report bundle 135..142 -o triage/sweep/ # all artefacts (default): case-study + html + diagnose + index.md
|
|
478
|
+
zond report bundle <run-id> --include case-study,diagnose # subset only (drop html)
|
|
479
|
+
zond report bundle --session <id> -o triage/session/ # group by session
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
Bodies > 8 KB truncated by default; `--no-body-cap` to keep full.
|
|
483
|
+
Existing files at `--output` auto-rotate to `<stem>-vN.<ext>`;
|
|
484
|
+
`--overwrite` to silence. **Always pass `--redact-identity` for
|
|
485
|
+
outbound sharing** (secret-redaction is on by default; identity slugs
|
|
486
|
+
need the extra flag).
|
|
487
|
+
|
|
488
|
+
Offer this proactively after a run surfaces `definitely_bug`
|
|
489
|
+
(5xx / schema violation / mass-assignment 2xx). Skip for `env_issue` /
|
|
490
|
+
`quirk`.
|
|
491
|
+
|
|
492
|
+
## One-shot full audit
|
|
493
|
+
|
|
494
|
+
`zond audit --api <name>` runs the whole pipeline in one shot:
|
|
495
|
+
prepare-fixtures → generate → probe static → session-wrapped run on
|
|
496
|
+
tests+probes → coverage → `audit-report.html`. Each stage prints
|
|
497
|
+
`==> Stage N/M: <name>`; stage failure doesn't stop the rest; final
|
|
498
|
+
exit 1 if any stage failed.
|
|
499
|
+
|
|
500
|
+
```bash
|
|
501
|
+
zond audit --api <name> --dry-run # plan
|
|
502
|
+
zond audit --api <name> # minimal pipeline
|
|
503
|
+
zond audit --api <name> --seed # add prepare-fixtures --cascade --seed
|
|
504
|
+
zond audit --api <name> --with-mass-assignment --with-security
|
|
505
|
+
zond audit --api <name> --out reports/audit-<name>.html
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
`generate` skipped via mtime when `tests/` is fresher than `spec.json`
|
|
509
|
+
(pass `--force` to override).
|
|
510
|
+
|
|
511
|
+
**Gotchas**:
|
|
512
|
+
- Wraps its own session — closes any prior `session start` silently.
|
|
513
|
+
- Can exit 0 on failed stages (parse stdout `Warning: N failed`).
|
|
514
|
+
- `audit-report.html` path not echoed by default — pass `--out`.
|
|
515
|
+
|
|
516
|
+
When NOT to use audit: narrow asks ("fix run X", "why this endpoint
|
|
517
|
+
500s") — walk phases.
|
|
518
|
+
|
|
519
|
+
## Phase S — Single-flow scenarios
|
|
520
|
+
|
|
521
|
+
When the user wants to verify one specific flow (not breadth):
|
|
522
|
+
|
|
523
|
+
```bash
|
|
524
|
+
zond doctor --api <name> # confirm fixtures
|
|
525
|
+
# write apis/<name>/scenarios/<flow-slug>.yaml directly (no scaffold cmd)
|
|
526
|
+
zond session start --label "<short reason>"
|
|
527
|
+
zond run apis/<name>/scenarios/<flow>.yaml --report json
|
|
528
|
+
zond session end
|
|
529
|
+
```
|
|
530
|
+
|
|
531
|
+
YAML grammar (zond runner format — NOT Postman / OpenAPI shape):
|
|
532
|
+
|
|
533
|
+
```yaml
|
|
534
|
+
name: cancel-pending-order
|
|
535
|
+
tags: [scenario, orders]
|
|
536
|
+
base_url: "{{base_url}}"
|
|
537
|
+
headers: { Authorization: "Bearer {{auth_token}}" }
|
|
538
|
+
tests:
|
|
539
|
+
- name: create order
|
|
540
|
+
POST: /workspaces/{{ws_id}}/orders
|
|
541
|
+
json: { amount: 100, currency: USD }
|
|
542
|
+
expect:
|
|
543
|
+
status: 201
|
|
544
|
+
body:
|
|
545
|
+
id: { capture: order_id }
|
|
546
|
+
status: { equals: pending }
|
|
547
|
+
- name: cancel
|
|
548
|
+
POST: /orders/{{order_id}}/cancel
|
|
549
|
+
expect: { status: 200 }
|
|
550
|
+
- name: verify cancelled
|
|
551
|
+
GET: /orders/{{order_id}}
|
|
552
|
+
expect:
|
|
553
|
+
status: 200
|
|
554
|
+
body: { status: { equals: cancelled } }
|
|
555
|
+
- name: cleanup
|
|
556
|
+
DELETE: /orders/{{order_id}}
|
|
557
|
+
always: true # runs even on prior failure
|
|
558
|
+
expect: { status: [200, 204, 404] }
|
|
559
|
+
```
|
|
560
|
+
|
|
561
|
+
Grammar:
|
|
562
|
+
- HTTP verb is a top-level key on the step (`GET:`, `POST:`, …).
|
|
563
|
+
- Body: `json: {...}` / `form: {...}` / `multipart:` / `text:`.
|
|
564
|
+
- Captures: `expect.body.<field>: { capture: <var> }`.
|
|
565
|
+
- Assertions: `equals`, `type`, `exists`, `matches`, `gt`/`lt`,
|
|
566
|
+
`length*`, `each`, `contains_item`, `set_equals`, `oneOf`.
|
|
567
|
+
- `expect.status`: number or array. `oneOf` / `anyOf` string forms NOT
|
|
568
|
+
supported — pass an array directly.
|
|
569
|
+
- `always: true` — runs on prior failure (cleanup).
|
|
570
|
+
- `skip_if: "{{var}} =="` — skip when fixture var unset.
|
|
571
|
+
- Generators: `{{$randomEmail}}`, `{{$uuid}}`, `{{$randomString}}`,
|
|
572
|
+
`{{$randomInt}}`, `{{$randomIsoDate}}`. Full list:
|
|
573
|
+
`zond reference random-helpers`.
|
|
574
|
+
|
|
575
|
+
Hand-crafted scenarios are for **bug repro / post-deploy smoke /
|
|
576
|
+
verifying a fix** — small focused work. For breadth, use Phase 3
|
|
577
|
+
(generate) and the full audit chain.
|
|
578
|
+
|
|
579
|
+
## Diagnose failures
|
|
580
|
+
|
|
581
|
+
```bash
|
|
582
|
+
zond db runs --limit 5 --json
|
|
583
|
+
zond db diagnose <run-id> --json # grouped by root_cause
|
|
584
|
+
zond db run <id> --status 500 --json
|
|
585
|
+
zond db compare <idA> <idB> --json # regression diff
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
`recommended_action` enum (closed):
|
|
589
|
+
- `report_backend_bug` — STOP, surface to user
|
|
590
|
+
- `fix_test_logic` — edit the YAML
|
|
591
|
+
- `fix_auth_config` / `fix_network_config` / `fix_env` — config issues
|
|
592
|
+
- `fix_spec` — edit OpenAPI (from `check spec`)
|
|
593
|
+
- `fix_fixture` — fill `.env.yaml` (from `prepare-fixtures` miss-* or
|
|
594
|
+
inconclusive mass-assignment baseline)
|
|
595
|
+
- `update_spec` — status-drift from `--learn`
|
|
596
|
+
|
|
597
|
+
`cascade` failures collapse under root cause — fix upstream, not
|
|
598
|
+
individually.
|
|
599
|
+
|
|
600
|
+
### Fixing 4xx from stub generators (`fix_test_logic`)
|
|
601
|
+
|
|
602
|
+
When body rejected on format (400/422 + "expected ..."):
|
|
603
|
+
|
|
604
|
+
1. Read failure: `zond db run <id> --status 422 --json`.
|
|
605
|
+
2. **Fixture pack first** — if it's a FK id / verified resource /
|
|
606
|
+
constrained enum, add to `.env.yaml` (Phase 1).
|
|
607
|
+
3. **Typed generator** — swap `{{$randomString}}` for matching format
|
|
608
|
+
helper.
|
|
609
|
+
4. **Hardcoded literal** — when typed generators fail (regex too strict).
|
|
610
|
+
5. **Runtime captures** — for resources the test creates (capture from
|
|
611
|
+
prior `create_*` step or `setup: true` suite). For pre-existing FKs,
|
|
612
|
+
prefer step 2.
|
|
613
|
+
|
|
614
|
+
For deep triage of a specific failing run → hand off to `zond-triage`.
|
|
615
|
+
|
|
616
|
+
## Auth / environments
|
|
617
|
+
|
|
618
|
+
- `apis/<name>/.env.yaml` is both auth and fixture pack — any key
|
|
619
|
+
interpolatable as `{{key}}`. Auto-gitignored.
|
|
620
|
+
- Login-flow tokens: `setup: true` suite captures vars that propagate
|
|
621
|
+
to later suites in the same run.
|
|
622
|
+
- `zond run --env <name>` loads `.env.<name>.yaml`. Discovery walks up
|
|
623
|
+
to workspace root; deeper files override shallower.
|
|
624
|
+
|
|
625
|
+
## File lifecycle
|
|
626
|
+
|
|
627
|
+
Everything zond writes is tracked in `.zond/manifest.json` with
|
|
628
|
+
sha256-at-write. Files whose hash no longer matches are **skipped**
|
|
629
|
+
on clean (user edits stay safe). Re-running a generator overwrites
|
|
630
|
+
the entry — manual edits to generator-owned files are lost on
|
|
631
|
+
regenerate; promote them to a separate suite first.
|
|
632
|
+
|
|
633
|
+
```bash
|
|
634
|
+
zond clean --api <name> # dry-run: lists what would be removed
|
|
635
|
+
zond clean --api <name> --force # delete (preserves probes/)
|
|
636
|
+
zond clean --api <name> --probes --force # also probe suites
|
|
637
|
+
zond clean --all --force # nuclear
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
Triage outputs default to
|
|
641
|
+
`triage/<api>/<run-id>/<command>-<ts>.<ext>`. Don't pass `--output`
|
|
642
|
+
unless the user has a specific destination.
|
|
643
|
+
|
|
644
|
+
## When to hand off
|
|
645
|
+
|
|
646
|
+
- "deep depth-checks", "SARIF", "boundary coverage", "stateful
|
|
647
|
+
invariants details", "annotate aspect XYZ" → `zond-checks`
|
|
648
|
+
- "what failed in last run", "why is it red", "case study from run X"
|
|
649
|
+
→ `zond-triage`
|
|
650
|
+
|
|
651
|
+
If the user is mid-workflow in one of those, don't restart from here.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# zond workspace config
|
|
2
|
+
#
|
|
3
|
+
# The presence of this file marks the workspace root for `zond` walk-up
|
|
4
|
+
# resolution (zond.db, apis/<name>/, .zond/current-api are anchored here).
|
|
5
|
+
|
|
6
|
+
version: 1
|
|
7
|
+
|
|
8
|
+
# Workspace-level defaults. Per-command flags always win; per-API
|
|
9
|
+
# override lives in apis/<name>/.env.yaml as `rateLimit:` / `timeoutMs:`.
|
|
10
|
+
# Resolution: CLI flag > .env.yaml meta > defaults below > built-in fallback.
|
|
11
|
+
#
|
|
12
|
+
# defaults:
|
|
13
|
+
# timeout_ms: 30000 # cleanup / prepare-fixtures / probe / request
|
|
14
|
+
# rate_limit: 5 # `zond run` (number rps, or "auto" for adaptive)
|