@kirrosh/zond 0.14.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -112
- package/README.md +3 -10
- package/package.json +4 -4
- package/src/cli/commands/ci-init.ts +12 -1
- package/src/cli/commands/coverage.ts +21 -1
- package/src/cli/commands/db.ts +121 -0
- package/src/cli/commands/describe.ts +60 -0
- package/src/cli/commands/export.ts +144 -0
- package/src/cli/commands/generate.ts +158 -0
- package/src/cli/commands/guide.ts +127 -0
- package/src/cli/commands/init.ts +57 -0
- package/src/cli/commands/request.ts +57 -0
- package/src/cli/commands/run.ts +74 -14
- package/src/cli/commands/serve.ts +62 -3
- package/src/cli/commands/sync.ts +240 -0
- package/src/cli/commands/validate.ts +18 -2
- package/src/cli/index.ts +258 -17
- package/src/cli/json-envelope.ts +19 -0
- package/src/core/diagnostics/db-analysis.ts +423 -0
- package/src/core/diagnostics/failure-hints.ts +40 -0
- package/src/core/exporter/postman.ts +963 -0
- package/src/core/generator/data-factory.ts +55 -9
- package/src/core/generator/describe.ts +250 -0
- package/src/core/generator/guide-builder.ts +20 -0
- package/src/core/generator/index.ts +1 -1
- package/src/core/generator/openapi-reader.ts +6 -0
- package/src/core/generator/serializer.ts +17 -2
- package/src/core/generator/suite-generator.ts +291 -29
- package/src/core/generator/types.ts +1 -0
- package/src/core/meta/meta-store.ts +78 -0
- package/src/core/meta/types.ts +21 -0
- package/src/core/parser/schema.ts +12 -2
- package/src/core/parser/types.ts +12 -1
- package/src/core/parser/variables.ts +3 -0
- package/src/core/parser/yaml-parser.ts +2 -1
- package/src/core/runner/assertions.ts +44 -20
- package/src/core/runner/execute-run.ts +31 -8
- package/src/core/runner/executor.ts +35 -8
- package/src/core/runner/http-client.ts +1 -1
- package/src/core/runner/send-request.ts +94 -0
- package/src/core/runner/types.ts +2 -0
- package/src/core/sync/spec-differ.ts +38 -0
- package/src/db/queries.ts +4 -2
- package/src/db/schema.ts +11 -3
- package/src/web/views/suites-tab.ts +1 -1
- package/src/cli/commands/mcp.ts +0 -16
- package/src/mcp/descriptions.ts +0 -71
- package/src/mcp/server.ts +0 -45
- package/src/mcp/tools/ci-init.ts +0 -54
- package/src/mcp/tools/coverage-analysis.ts +0 -141
- package/src/mcp/tools/describe-endpoint.ts +0 -242
- package/src/mcp/tools/generate-and-save.ts +0 -202
- package/src/mcp/tools/manage-server.ts +0 -86
- package/src/mcp/tools/query-db.ts +0 -300
- package/src/mcp/tools/run-tests.ts +0 -115
- package/src/mcp/tools/save-test-suite.ts +0 -218
- package/src/mcp/tools/send-request.ts +0 -97
- package/src/mcp/tools/set-work-dir.ts +0 -35
- package/src/mcp/tools/setup-api.ts +0 -88
package/CHANGELOG.md
CHANGED
|
@@ -2,146 +2,166 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
-
## [
|
|
5
|
+
## [Unreleased] — fix/generator-quality-improvements
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
- CLI binary: `apitool` → `zond`
|
|
9
|
-
- Database: `apitool.db` → `zond.db`
|
|
10
|
-
- Environment variable: `APITOOL_AI_KEY` → `ZOND_AI_KEY`
|
|
11
|
-
- MCP server name: `apitool` → `zond`
|
|
12
|
-
- Fresh DB schema (no migrations)
|
|
7
|
+
### Breaking changes
|
|
13
8
|
|
|
14
|
-
|
|
9
|
+
- **MCP layer removed** — `zond mcp` command and `@modelcontextprotocol/sdk` dependency deleted.
|
|
10
|
+
The agent interface is now exclusively the CLI + skills in `skills/`. No migration path needed.
|
|
15
11
|
|
|
16
|
-
|
|
12
|
+
- **`zond migrate` removed** — the migration system was added and then removed in the same branch.
|
|
13
|
+
Format changes in zond are backward-compatible or require a clean `zond generate`.
|
|
17
14
|
|
|
18
|
-
|
|
19
|
-
- **Endpoints** — all OpenAPI spec endpoints with coverage status (passing/failing/no tests/not run), filters by status and method, expandable details with covering test steps, assertions, and response status
|
|
20
|
-
- **Suites** — all YAML test files on disk with run results, expandable step details showing assertions, captures, error messages, and response body for failed steps
|
|
21
|
-
- **Runs** — paginated run history with progress bars, drill-down into run details with JUnit/JSON export
|
|
22
|
-
- **Health strip** — top-of-page overview: coverage donut chart, pass/fail/skip stats, progress bar, environment alert banner
|
|
23
|
-
- **Broken endpoint marking** — `deprecated`, `no_response_schema`, `no_responses_defined`, `required_params_no_examples` warnings shown as badges on endpoints
|
|
24
|
-
- **diagnose_failure** enriched with `failure_type` (api_error / assertion_failed / network_error) and summary counts
|
|
15
|
+
---
|
|
25
16
|
|
|
26
|
-
###
|
|
17
|
+
### Features
|
|
27
18
|
|
|
28
|
-
|
|
19
|
+
#### Generator
|
|
29
20
|
|
|
30
|
-
|
|
21
|
+
- **Sanity suite** (`sanity.yaml`) — `zond generate` now produces a 1-2 step sanity file as the
|
|
22
|
+
first output: an auth step (if the API has auth) + a connectivity probe (healthcheck or first
|
|
23
|
+
simple GET). Run with `--tag sanity` before the full suite to catch `base_url`/auth issues early.
|
|
24
|
+
Skill workflow updated with mandatory Step 3.25.
|
|
31
25
|
|
|
32
|
-
|
|
26
|
+
- **Multipart bodies** — endpoints with `requestBody: multipart/form-data` now generate `multipart:`
|
|
27
|
+
blocks instead of empty `json:`. Binary (`format: binary` / `format: byte`) fields become
|
|
28
|
+
`{ file: ./fixtures/<field>.bin, content_type: application/octet-stream }`.
|
|
33
29
|
|
|
34
|
-
-
|
|
30
|
+
- **Reset endpoint isolation** — `reset`, `flush`, `purge`, `truncate`, `wipe`, `clear-data`,
|
|
31
|
+
`factory-reset` paths now get tags `[system, reset]` instead of `[smoke, unsafe]`, preventing
|
|
32
|
+
them from running during smoke passes and accidentally wiping server state.
|
|
35
33
|
|
|
36
|
-
|
|
34
|
+
- **Logout exclusion from setup suites** — `logout`, `signout`, `invalidate`, `revoke` endpoints
|
|
35
|
+
are no longer included in `setup: true` auth suites. Including them would invalidate the captured
|
|
36
|
+
token for all subsequent suites.
|
|
37
37
|
|
|
38
|
-
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
- URL contains `{{variable}}` → `"URL contains unresolved variable — check variable names in .env.yaml"`
|
|
42
|
-
- error message contains `"URL is invalid"` → `"URL is malformed — likely base_url is empty or invalid"`
|
|
43
|
-
- env hints take priority over generic `statusHint` (401/404/5xx)
|
|
44
|
-
- new top-level `env_issue` field when ALL failures share the same env problem category
|
|
45
|
-
- env file path resolved from `collection.base_dir` — agent sees the exact file to edit
|
|
46
|
-
- **`run_tests` MCP**: always suggests `manage_server(action: 'start')` in `hints` after a run
|
|
47
|
-
- **`generate_tests_guide` / `generate_missing_tests` descriptions**: mention `manage_server` as the next step after saving and running tests
|
|
38
|
+
- **Seed values in smoke path params** — GET smoke steps with path parameters now use concrete seed
|
|
39
|
+
values (from spec `example` field, or `1` for id-like params) instead of unresolved `{{id}}`
|
|
40
|
+
placeholders that cause failures at runtime.
|
|
48
41
|
|
|
49
|
-
|
|
42
|
+
- **Bounded integer generation** — `integer` fields with a `maximum` constraint now generate a
|
|
43
|
+
concrete in-range value instead of `{{$randomInt}}`, which could exceed server-side validation
|
|
44
|
+
limits.
|
|
50
45
|
|
|
51
|
-
|
|
46
|
+
- **ETag auto-injection** — when an endpoint has `412` in its responses or an `If-Match` header
|
|
47
|
+
parameter, the CRUD generator automatically inserts a GET capture step before PUT/PATCH/DELETE
|
|
48
|
+
to capture the ETag, and adds the `If-Match: "{{resource_etag}}"` header to the mutation step.
|
|
52
49
|
|
|
53
|
-
|
|
54
|
-
- **CLI commands** — removed `add-api`, `init`, `collections`, `runs`, `compare`, `doctor`, `update` (available via MCP tools or unnecessary)
|
|
55
|
-
- **Directories** — removed `generated/`, `examples/`, `self-tests/`, `apis/`, `docs/archive/`
|
|
56
|
-
- **Files** — removed `seed-demo.ts`, `BACKLOG.md`, `docs/agent.md`
|
|
57
|
-
|
|
58
|
-
### Added
|
|
59
|
-
|
|
60
|
-
- **Extended YAML test format** — 12 new assertion operators, flow control, and data transforms:
|
|
61
|
-
- **Assertion operators**: `not_equals`, `not_contains`, `gte`, `lte`, `length`, `length_gt/gte/lt/lte`
|
|
62
|
-
- **Array assertions**: `each` (every element matches), `contains_item` (at least one matches), `set_equals` (order-independent comparison)
|
|
63
|
-
- **Flow control**: `skip_if` (conditional skip with expression evaluator), `retry_until` (retry with condition/max_attempts/delay_ms), `for_each` (iterate over array)
|
|
64
|
-
- **Data transforms**: `set` steps with directives — `concat`, `append`, `length`, `get`, `first`, `map_field`
|
|
65
|
-
- **Generator**: `{{$isoTimestamp}}` — ISO 8601 timestamp string
|
|
66
|
-
- **Expression evaluator**: supports `==`, `!=`, `>`, `<`, `>=`, `<=` for skip_if/retry_until conditions
|
|
67
|
-
- Guide-builder YAML cheatsheet updated with all new features
|
|
68
|
-
- Full backward compatibility — all existing tests continue to work unchanged
|
|
69
|
-
|
|
70
|
-
- **MCP feedback improvements**
|
|
71
|
-
- `diagnose_failure` now includes `response_headers` in failure output (e.g. `X-Ably-ErrorMessage`)
|
|
72
|
-
- `generate_tests_guide`: annotates `any`-typed request bodies with a warning comment
|
|
73
|
-
- `generate_tests_guide`: added 204 No Content tips in Practical Tips and Common Mistakes sections
|
|
74
|
-
- `schema-utils.ts`: added `isAnySchema()` helper
|
|
75
|
-
- DB schema v6: `results.response_headers TEXT` column
|
|
76
|
-
|
|
77
|
-
- **M22: MCP-first smart test generation**
|
|
78
|
-
- `generate_tests_guide` MCP tool — returns full API spec with schemas + step-by-step generation algorithm
|
|
79
|
-
- `save_test_suite` MCP tool — validates YAML and saves test files with structured error reporting
|
|
80
|
-
- `explore_api` enhanced — new `includeSchemas` parameter for full request/response body schemas
|
|
81
|
-
- `schema-utils.ts` — extracted `compressSchema()` and `formatParam()` as shared utilities
|
|
82
|
-
- Improved MCP tool descriptions with "when to use" guidance
|
|
50
|
+
#### Executor
|
|
83
51
|
|
|
84
|
-
|
|
52
|
+
- **`set:` on HTTP steps** — `set:` directives on regular HTTP steps are now evaluated before the
|
|
53
|
+
request, pinning generators (e.g., `$uuid`) once so the same value can flow into the request body
|
|
54
|
+
and be reused in subsequent steps.
|
|
55
|
+
|
|
56
|
+
#### Setup suites
|
|
57
|
+
|
|
58
|
+
- **Auth token auto-sharing** — `setup: true` flag on a suite causes it to run before all other
|
|
59
|
+
suites (sequentially). Its captured variables (e.g., `auth_token`) are merged into the environment
|
|
60
|
+
of every subsequent suite automatically. Generated auth suites now include `setup: true`.
|
|
61
|
+
|
|
62
|
+
#### Export
|
|
63
|
+
|
|
64
|
+
- **`zond export postman`** — converts YAML test suites to Postman Collection v2.1 JSON.
|
|
65
|
+
- Full assertion mapping: `status`, `body`, `headers`, `duration` → `pm.test()`/`pm.expect()`
|
|
66
|
+
- Captures → `pm.environment.set()` for cross-request variable sharing
|
|
67
|
+
- `set:` steps → `pm.environment.set()` pre-request scripts on the next HTTP step
|
|
68
|
+
- `skip_if` → `pm.execution.setNextRequest()` pre-request event
|
|
69
|
+
- Optional `--env` flag exports `.env.yaml` as a Postman Environment JSON
|
|
70
|
+
- `each`, `contains_item`, `set_equals` assertions fully translated
|
|
71
|
+
- `type: integer` → `Number.isInteger()` (not `.be.a('number')`)
|
|
72
|
+
- Setup suites sorted first to mirror zond runner behaviour
|
|
73
|
+
- Newman CLI hints embedded in collection description for non-default configs
|
|
74
|
+
|
|
75
|
+
#### Sync
|
|
76
|
+
|
|
77
|
+
- **`zond sync`** — incremental test update command. Compares the current spec against the hash
|
|
78
|
+
stored in `.zond-meta.json`, generates test files only for new endpoints, never overwrites
|
|
79
|
+
existing files. Reports removed endpoints as warnings. Updates `collections.openapi_spec` in
|
|
80
|
+
SQLite automatically.
|
|
81
|
+
|
|
82
|
+
- **`.zond-meta.json`** — metadata file written by `zond generate` and `zond sync`. Stores
|
|
83
|
+
spec URL, SHA-256 hash, and per-file metadata for drift detection.
|
|
84
|
+
|
|
85
|
+
#### Diagnostics
|
|
86
|
+
|
|
87
|
+
- **`recommended_action`** field on every failure in `zond db diagnose --json`:
|
|
88
|
+
`report_backend_bug` / `fix_auth_config` / `fix_test_logic` / `fix_network_config`.
|
|
89
|
+
|
|
90
|
+
- **`agent_directive`** top-level field — when `api_error` count > 0, tells the agent explicitly
|
|
91
|
+
to stop iterating and report the server bug instead of modifying test expectations.
|
|
85
92
|
|
|
86
|
-
-
|
|
93
|
+
- **`cascade_skips`** field — groups skipped tests by the missing capture variable, making
|
|
94
|
+
"5 tests skipped because `createCase` step failed" visible instead of a flat skip list.
|
|
95
|
+
|
|
96
|
+
- **`auth_hint`** — surfaces when ≥30% of tests fail with 401/403, and now mentions
|
|
97
|
+
`setup: true` as the recommended fix.
|
|
98
|
+
|
|
99
|
+
- **Soft delete hint** — when a GET returns `200` with a `status`/`state`/`deleted` field instead
|
|
100
|
+
of the expected `404` (after a DELETE), the diagnostic now surfaces a "likely soft delete" hint
|
|
101
|
+
with a concrete suggestion to assert the status field value.
|
|
87
102
|
|
|
88
103
|
---
|
|
89
104
|
|
|
90
|
-
|
|
105
|
+
### Fixes
|
|
91
106
|
|
|
92
|
-
|
|
107
|
+
#### Parser / runtime
|
|
93
108
|
|
|
94
|
-
-
|
|
95
|
-
|
|
96
|
-
- **Environment selector** — `<select name="env">` dropdown in collection "Run Tests" form
|
|
97
|
-
- **DB queries** — `getEnvironmentById()`, `deleteEnvironment()`, `listEnvironmentRecords()`
|
|
98
|
-
- **Navigation** — "Environments" link in navbar
|
|
99
|
-
- **Improved runs filter** — environment dropdown merges defined environments + run history
|
|
100
|
-
- **Self-documented API** — routes use `@hono/zod-openapi`, `GET /api/openapi.json` serves spec
|
|
101
|
-
- **Incremental generation** — `apitool generate` skips already-covered endpoints
|
|
102
|
-
- **Dogfooding** — integration tests run against apitool's own API
|
|
103
|
-
- **Generator: `additionalProperties`** — Record types generate sample key-value pairs instead of `{}`
|
|
104
|
-
- **CI: typecheck** — `tsc --noEmit` step added to CI pipeline
|
|
109
|
+
- **`expect.headers` now accepts `AssertionRule`** — headers can use `capture:`, `equals:`,
|
|
110
|
+
`type:`, etc. (previously only plain string equality). Enables ETag and other header captures.
|
|
105
111
|
|
|
106
|
-
|
|
112
|
+
- **`filePath` normalized to absolute** — `yaml-parser.ts` now stores absolute paths so
|
|
113
|
+
`multipart: file:` paths resolve correctly regardless of CWD at execution time.
|
|
107
114
|
|
|
108
|
-
-
|
|
115
|
+
- **`multipart:` bodies now reach the HTTP client** — `formData` field added to `HttpRequest`;
|
|
116
|
+
`http-client.ts` sends `formData` when present (previously only `body` was sent, so multipart
|
|
117
|
+
requests were sent empty).
|
|
109
118
|
|
|
110
|
-
|
|
119
|
+
- **`multipart:` variable substitution** — `substituteStep` now processes `multipart:` field
|
|
120
|
+
values, so `{{variables}}` inside multipart blocks are interpolated correctly.
|
|
111
121
|
|
|
112
|
-
-
|
|
113
|
-
|
|
122
|
+
- **Safe mode preserves auth endpoints** — `execute-run.ts` safe mode now keeps
|
|
123
|
+
`login`/`token`/`oauth` endpoints consistent with `run.ts` behaviour.
|
|
114
124
|
|
|
115
|
-
|
|
125
|
+
#### Generator data quality
|
|
116
126
|
|
|
117
|
-
- **
|
|
118
|
-
|
|
127
|
+
- **Nested object serialization** — `serializeValue` in `serializer.ts` now recurses into objects
|
|
128
|
+
instead of calling `String(val)`, fixing `[object Object]` in array item bodies.
|
|
119
129
|
|
|
120
|
-
|
|
130
|
+
- **`format: date`** returns `"2025-01-01"` (date-only), not a full datetime string.
|
|
121
131
|
|
|
122
|
-
|
|
132
|
+
- **`format: uuid`** overrides type — `integer` fields with `format: uuid` now correctly get
|
|
133
|
+
`{{$uuid}}` instead of `{{$randomInt}}`.
|
|
123
134
|
|
|
124
|
-
|
|
135
|
+
#### Skill / documentation
|
|
136
|
+
|
|
137
|
+
- **SKILL.md NEVER rules** — added explicit stop rules for: in-memory auth tokens, ETag, soft
|
|
138
|
+
delete, rate limits, setup suite design, `--tag` without setup tag.
|
|
139
|
+
|
|
140
|
+
- **SKILL.md generator smart behaviors** — documents all generator improvements so agents know
|
|
141
|
+
what to expect from generated output.
|
|
142
|
+
|
|
143
|
+
---
|
|
144
|
+
|
|
145
|
+
### Removed
|
|
146
|
+
|
|
147
|
+
- `src/mcp/` and all MCP tooling (~1900 lines deleted)
|
|
148
|
+
- `zond mcp` CLI command
|
|
149
|
+
- `@modelcontextprotocol/sdk` dependency
|
|
150
|
+
- `zond migrate` command and `src/core/migrations/` module
|
|
151
|
+
- `docs/mcp-guide.md`
|
|
152
|
+
|
|
153
|
+
---
|
|
125
154
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
-
|
|
129
|
-
-
|
|
130
|
-
-
|
|
131
|
-
-
|
|
132
|
-
-
|
|
133
|
-
-
|
|
134
|
-
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
- **CLI commands**:
|
|
140
|
-
- `apitool run <path>` — execute tests with env, reporter, timeout, bail options
|
|
141
|
-
- `apitool validate <path>` — validate YAML test files
|
|
142
|
-
- `apitool generate --from <spec>` — generate tests from OpenAPI
|
|
143
|
-
- `apitool ai-generate --from <spec> --prompt "..."` — AI test generation
|
|
144
|
-
- `apitool serve` — start web dashboard
|
|
145
|
-
- `apitool collections` — list test collections
|
|
146
|
-
- **Multi-auth support** — Basic, Bearer, API Key auth in CLI (`--auth-token`) and WebUI
|
|
147
|
-
- **Standalone binary** — single-file executable via `bun build --compile`
|
|
155
|
+
### Tests
|
|
156
|
+
|
|
157
|
+
- 502 tests total (499 unit + 3 mocked), 0 failures
|
|
158
|
+
- New tests: `suite-generator` — reset tag, smoke seeds, logout filter, ETag injection, multipart
|
|
159
|
+
- New tests: `data-factory` — maximum constraint, `generateMultipartFromSchema`
|
|
160
|
+
- New tests: `serializer` — nested object serialization, `setup: true`
|
|
161
|
+
- New tests: `failure-hints` — soft delete hint, `recommended_action`, `classifyFailure`
|
|
162
|
+
- New tests: `executor` — header capture, `set:` pinning, setup capture propagation
|
|
163
|
+
- New tests: `schema` — `setup: true` round-trip
|
|
164
|
+
- Fixed: `mock.module()` cache pollution — coverage tests moved to `tests/mocked/coverage.ts`
|
|
165
|
+
and run in a separate subprocess via `scripts/run-mocked-tests.ts` (bun#7823, bun#12823)
|
|
166
|
+
- Fixed: `test:unit` script — added `tests/diagnostics/`, corrected `tests/web/` and
|
|
167
|
+
`tests/reporter/` paths
|
package/README.md
CHANGED
|
@@ -17,16 +17,10 @@ Zond reads your OpenAPI spec and gives your AI agent everything it needs to test
|
|
|
17
17
|
|
|
18
18
|
Then say: _"Safely cover the API from openapi.json with tests"_
|
|
19
19
|
|
|
20
|
-
You get
|
|
20
|
+
You get auto-validation hooks and CLI tools — all in one package.
|
|
21
21
|
|
|
22
22
|
<details>
|
|
23
|
-
<summary>Other installation methods (
|
|
24
|
-
|
|
25
|
-
### MCP Server (Cursor, Windsurf, other editors)
|
|
26
|
-
|
|
27
|
-
[](https://cursor.com/en/install-mcp?name=zond&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBraXJyb3NoL3pvbmQiLCJtY3AiXX0K)
|
|
28
|
-
|
|
29
|
-
Or add manually — see [MCP setup guide](docs/mcp-guide.md) for Cursor, Claude Code, and Windsurf config.
|
|
23
|
+
<summary>Other installation methods (CLI, binary)</summary>
|
|
30
24
|
|
|
31
25
|
### CLI / Binary
|
|
32
26
|
|
|
@@ -75,8 +69,7 @@ Claude Code can write pytest from scratch — but it takes 30-60 minutes per flo
|
|
|
75
69
|
|
|
76
70
|
## Documentation
|
|
77
71
|
|
|
78
|
-
- [ZOND.md](ZOND.md) — full CLI
|
|
79
|
-
- [docs/mcp-guide.md](docs/mcp-guide.md) — MCP agent workflow guide
|
|
72
|
+
- [ZOND.md](ZOND.md) — full CLI reference
|
|
80
73
|
- [docs/quickstart.md](docs/quickstart.md) — step-by-step quickstart (RU)
|
|
81
74
|
- [docs/ci.md](docs/ci.md) — CI/CD integration
|
|
82
75
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kirrosh/zond",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "index.ts",
|
|
@@ -26,12 +26,13 @@
|
|
|
26
26
|
"scripts": {
|
|
27
27
|
"zond": "bun run src/cli/index.ts",
|
|
28
28
|
"test": "bun run test:unit && bun run test:mocked",
|
|
29
|
-
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/safe-run.test.ts tests/
|
|
29
|
+
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/diagnostics/ tests/cli/args.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/safe-run.test.ts tests/cli/json-envelope.test.ts tests/cli/describe.test.ts tests/cli/db.test.ts tests/cli/request.test.ts tests/cli/init.test.ts tests/cli/guide.test.ts tests/integration/ tests/web/ tests/reporter/ tests/version-sync.test.ts",
|
|
30
30
|
"test:mocked": "bun run scripts/run-mocked-tests.ts",
|
|
31
31
|
"check": "tsc --noEmit --project tsconfig.json",
|
|
32
32
|
"build": "bun build --compile src/cli/index.ts --outfile zond",
|
|
33
33
|
"version:sync": "bun run scripts/sync-version.ts",
|
|
34
|
-
"postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json
|
|
34
|
+
"postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json",
|
|
35
|
+
"bench:api": "bun benchmarks/api/server.ts"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@types/bun": "latest"
|
|
@@ -42,7 +43,6 @@
|
|
|
42
43
|
"dependencies": {
|
|
43
44
|
"@humanwhocodes/momoa": "^2.0.3",
|
|
44
45
|
"@hono/zod-openapi": "^1.2.2",
|
|
45
|
-
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
46
46
|
"@readme/openapi-parser": "^5.5.0",
|
|
47
47
|
"hono": "^4.12.2",
|
|
48
48
|
"openapi-types": "^12.1.3",
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { resolve, dirname } from "path";
|
|
2
2
|
import { existsSync, mkdirSync, writeFileSync } from "fs";
|
|
3
3
|
import { printSuccess, printError } from "../output.ts";
|
|
4
|
+
import { jsonOk, printJson } from "../json-envelope.ts";
|
|
4
5
|
|
|
5
6
|
export interface CiInitOptions {
|
|
6
7
|
platform?: "github" | "gitlab";
|
|
7
8
|
force: boolean;
|
|
8
9
|
dir?: string;
|
|
10
|
+
json?: boolean;
|
|
9
11
|
}
|
|
10
12
|
|
|
11
13
|
const GH_ACTIONS_TEMPLATE = `name: API Tests
|
|
@@ -155,7 +157,16 @@ export async function ciInitCommand(options: CiInitOptions): Promise<number> {
|
|
|
155
157
|
created = writeIfMissing(targetPath, GITLAB_CI_TEMPLATE, options.force);
|
|
156
158
|
}
|
|
157
159
|
|
|
158
|
-
if (
|
|
160
|
+
if (options.json) {
|
|
161
|
+
const targetPath = platform === "github"
|
|
162
|
+
? resolve(cwd, ".github/workflows/api-tests.yml")
|
|
163
|
+
: resolve(cwd, ".gitlab-ci.yml");
|
|
164
|
+
printJson(jsonOk("ci init", {
|
|
165
|
+
platform,
|
|
166
|
+
filePath: targetPath,
|
|
167
|
+
created,
|
|
168
|
+
}));
|
|
169
|
+
} else if (created) {
|
|
159
170
|
printSuccess("CI workflow created. Commit and push to activate.");
|
|
160
171
|
}
|
|
161
172
|
|
|
@@ -2,12 +2,14 @@ import { readOpenApiSpec, extractEndpoints, scanCoveredEndpoints, filterUncovere
|
|
|
2
2
|
import { getDb } from "../../db/schema.ts";
|
|
3
3
|
import { getResultsByRunId, getRunById } from "../../db/queries.ts";
|
|
4
4
|
import { printError, printSuccess } from "../output.ts";
|
|
5
|
+
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
5
6
|
|
|
6
7
|
export interface CoverageOptions {
|
|
7
8
|
spec: string;
|
|
8
9
|
tests: string;
|
|
9
10
|
failOnCoverage?: number;
|
|
10
11
|
runId?: number;
|
|
12
|
+
json?: boolean;
|
|
11
13
|
}
|
|
12
14
|
|
|
13
15
|
const RESET = "\x1b[0m";
|
|
@@ -145,12 +147,30 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
|
|
|
145
147
|
}
|
|
146
148
|
}
|
|
147
149
|
|
|
150
|
+
if (options.json) {
|
|
151
|
+
const coveredEndpoints = allEndpoints.filter(ep => !uncovered.includes(ep)).map(ep => `${ep.method} ${ep.path}`);
|
|
152
|
+
const uncoveredEndpoints = uncovered.map(ep => `${ep.method} ${ep.path}`);
|
|
153
|
+
printJson(jsonOk("coverage", {
|
|
154
|
+
covered: coveredCount,
|
|
155
|
+
uncovered: uncovered.length,
|
|
156
|
+
total: allEndpoints.length,
|
|
157
|
+
percentage,
|
|
158
|
+
coveredEndpoints,
|
|
159
|
+
uncoveredEndpoints,
|
|
160
|
+
}));
|
|
161
|
+
}
|
|
162
|
+
|
|
148
163
|
if (options.failOnCoverage !== undefined) {
|
|
149
164
|
return percentage < options.failOnCoverage ? 1 : 0;
|
|
150
165
|
}
|
|
151
166
|
return uncovered.length > 0 ? 1 : 0;
|
|
152
167
|
} catch (err) {
|
|
153
|
-
|
|
168
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
169
|
+
if (options.json) {
|
|
170
|
+
printJson(jsonError("coverage", [message]));
|
|
171
|
+
} else {
|
|
172
|
+
printError(message);
|
|
173
|
+
}
|
|
154
174
|
return 2;
|
|
155
175
|
}
|
|
156
176
|
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { getCollections, getRuns, getRunDetail, diagnoseRun, compareRuns } from "../../core/diagnostics/db-analysis.ts";
|
|
2
|
+
import { printError } from "../output.ts";
|
|
3
|
+
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
4
|
+
|
|
5
|
+
export interface DbOptions {
|
|
6
|
+
subcommand: string;
|
|
7
|
+
positional: string[];
|
|
8
|
+
limit?: number;
|
|
9
|
+
verbose?: boolean;
|
|
10
|
+
dbPath?: string;
|
|
11
|
+
json?: boolean;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function dbCommand(options: DbOptions): Promise<number> {
|
|
15
|
+
const { subcommand, positional, json } = options;
|
|
16
|
+
|
|
17
|
+
try {
|
|
18
|
+
switch (subcommand) {
|
|
19
|
+
case "collections": {
|
|
20
|
+
const collections = getCollections(options.dbPath);
|
|
21
|
+
if (json) {
|
|
22
|
+
printJson(jsonOk("db collections", { collections }));
|
|
23
|
+
} else {
|
|
24
|
+
if (collections.length === 0) {
|
|
25
|
+
console.log("No collections found.");
|
|
26
|
+
} else {
|
|
27
|
+
for (const c of collections) {
|
|
28
|
+
console.log(`[${(c as any).id}] ${(c as any).name} — ${(c as any).test_path ?? "no test path"}`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return 0;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
case "runs": {
|
|
36
|
+
const runs = getRuns(options.limit ?? 10, options.dbPath);
|
|
37
|
+
if (json) {
|
|
38
|
+
printJson(jsonOk("db runs", { runs }));
|
|
39
|
+
} else {
|
|
40
|
+
if (runs.length === 0) {
|
|
41
|
+
console.log("No runs found.");
|
|
42
|
+
} else {
|
|
43
|
+
for (const r of runs) {
|
|
44
|
+
const run = r as any;
|
|
45
|
+
const status = run.failed > 0 ? "FAIL" : "PASS";
|
|
46
|
+
console.log(`#${run.id} ${status} ${run.passed}/${run.total} passed (${run.started_at})`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
case "run": {
|
|
54
|
+
const id = parseInt(positional[0] ?? "", 10);
|
|
55
|
+
if (isNaN(id)) {
|
|
56
|
+
const msg = "Missing run ID. Usage: zond db run <id>";
|
|
57
|
+
if (json) printJson(jsonError("db run", [msg]));
|
|
58
|
+
else printError(msg);
|
|
59
|
+
return 2;
|
|
60
|
+
}
|
|
61
|
+
const detail = getRunDetail(id, options.verbose, options.dbPath);
|
|
62
|
+
if (json) {
|
|
63
|
+
printJson(jsonOk("db run", detail));
|
|
64
|
+
} else {
|
|
65
|
+
console.log(JSON.stringify(detail, null, 2));
|
|
66
|
+
}
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
case "diagnose": {
|
|
71
|
+
const id = parseInt(positional[0] ?? "", 10);
|
|
72
|
+
if (isNaN(id)) {
|
|
73
|
+
const msg = "Missing run ID. Usage: zond db diagnose <id>";
|
|
74
|
+
if (json) printJson(jsonError("db diagnose", [msg]));
|
|
75
|
+
else printError(msg);
|
|
76
|
+
return 2;
|
|
77
|
+
}
|
|
78
|
+
const result = diagnoseRun(id, options.verbose, options.dbPath);
|
|
79
|
+
if (json) {
|
|
80
|
+
printJson(jsonOk("db diagnose", result));
|
|
81
|
+
} else {
|
|
82
|
+
console.log(JSON.stringify(result, null, 2));
|
|
83
|
+
}
|
|
84
|
+
return 0;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
case "compare": {
|
|
88
|
+
const idA = parseInt(positional[0] ?? "", 10);
|
|
89
|
+
const idB = parseInt(positional[1] ?? "", 10);
|
|
90
|
+
if (isNaN(idA) || isNaN(idB)) {
|
|
91
|
+
const msg = "Missing run IDs. Usage: zond db compare <idA> <idB>";
|
|
92
|
+
if (json) printJson(jsonError("db compare", [msg]));
|
|
93
|
+
else printError(msg);
|
|
94
|
+
return 2;
|
|
95
|
+
}
|
|
96
|
+
const result = compareRuns(idA, idB, options.dbPath);
|
|
97
|
+
if (json) {
|
|
98
|
+
printJson(jsonOk("db compare", result));
|
|
99
|
+
} else {
|
|
100
|
+
console.log(JSON.stringify(result, null, 2));
|
|
101
|
+
}
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
default: {
|
|
106
|
+
const msg = `Unknown db subcommand: ${subcommand}. Available: collections, runs, run, diagnose, compare`;
|
|
107
|
+
if (json) printJson(jsonError("db", [msg]));
|
|
108
|
+
else printError(msg);
|
|
109
|
+
return 2;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
} catch (err) {
|
|
113
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
114
|
+
if (json) {
|
|
115
|
+
printJson(jsonError(`db ${subcommand}`, [message]));
|
|
116
|
+
} else {
|
|
117
|
+
printError(message);
|
|
118
|
+
}
|
|
119
|
+
return 2;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { describeEndpoint, describeCompact } from "../../core/generator/describe.ts";
|
|
2
|
+
import { printError } from "../output.ts";
|
|
3
|
+
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
4
|
+
|
|
5
|
+
export interface DescribeOptions {
|
|
6
|
+
specPath: string;
|
|
7
|
+
method?: string;
|
|
8
|
+
path?: string;
|
|
9
|
+
compact?: boolean;
|
|
10
|
+
json?: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export async function describeCommand(options: DescribeOptions): Promise<number> {
|
|
14
|
+
try {
|
|
15
|
+
if (options.compact) {
|
|
16
|
+
const endpoints = await describeCompact(options.specPath);
|
|
17
|
+
|
|
18
|
+
if (options.json) {
|
|
19
|
+
printJson(jsonOk("describe", { endpoints }));
|
|
20
|
+
} else {
|
|
21
|
+
for (const ep of endpoints) {
|
|
22
|
+
const parts = [ep.method.padEnd(7), ep.path];
|
|
23
|
+
if (ep.operationId) parts.push(`(${ep.operationId})`);
|
|
24
|
+
if (ep.summary) parts.push(`— ${ep.summary}`);
|
|
25
|
+
if (ep.deprecated) parts.push("[deprecated]");
|
|
26
|
+
console.log(parts.join(" "));
|
|
27
|
+
}
|
|
28
|
+
console.log(`\n${endpoints.length} endpoint(s)`);
|
|
29
|
+
}
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (!options.method || !options.path) {
|
|
34
|
+
const msg = "Missing --method and --path. Use --compact for all endpoints, or specify --method and --path for one.";
|
|
35
|
+
if (options.json) {
|
|
36
|
+
printJson(jsonError("describe", [msg]));
|
|
37
|
+
} else {
|
|
38
|
+
printError(msg);
|
|
39
|
+
}
|
|
40
|
+
return 2;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const result = await describeEndpoint(options.specPath, options.method, options.path);
|
|
44
|
+
|
|
45
|
+
if (options.json) {
|
|
46
|
+
printJson(jsonOk("describe", result));
|
|
47
|
+
} else {
|
|
48
|
+
console.log(JSON.stringify(result, null, 2));
|
|
49
|
+
}
|
|
50
|
+
return 0;
|
|
51
|
+
} catch (err) {
|
|
52
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
53
|
+
if (options.json) {
|
|
54
|
+
printJson(jsonError("describe", [message]));
|
|
55
|
+
} else {
|
|
56
|
+
printError(message);
|
|
57
|
+
}
|
|
58
|
+
return 2;
|
|
59
|
+
}
|
|
60
|
+
}
|