@kirrosh/zond 0.23.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +164 -1
- package/README.md +8 -7
- package/package.json +2 -3
- package/src/CLAUDE.md +112 -0
- package/src/cli/commands/add-api.ts +19 -7
- package/src/cli/commands/api/annotate/index.ts +359 -4
- package/src/cli/commands/api/annotate/lifecycle.ts +1 -1
- package/src/cli/commands/api/annotate/overlay.ts +1 -1
- package/src/cli/commands/api/annotate/pagination.ts +10 -6
- package/src/cli/commands/api/annotate/prompts.ts +39 -2
- package/src/cli/commands/audit.ts +360 -54
- package/src/cli/commands/check.ts +15 -2
- package/src/cli/commands/checks.ts +352 -36
- package/src/cli/commands/cleanup.ts +4 -30
- package/src/cli/commands/coverage.ts +275 -57
- package/src/cli/commands/db.ts +311 -8
- package/src/cli/commands/discover.ts +281 -161
- package/src/cli/commands/doctor.ts +57 -3
- package/src/cli/commands/fixtures.ts +1 -1
- package/src/cli/commands/generate.ts +24 -7
- package/src/cli/commands/init/bootstrap.ts +4 -1
- package/src/cli/commands/init/index.ts +2 -2
- package/src/cli/commands/init/skills.ts +43 -0
- package/src/cli/commands/init/templates/agents.md +12 -3
- package/src/cli/commands/init/templates/skills/warm-up-target.md +122 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +268 -44
- package/src/cli/commands/init/templates/skills/zond-seed.md +114 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +88 -26
- package/src/cli/commands/init/templates/skills/zond.md +274 -64
- package/src/cli/commands/init/templates/zond-config.yml +1 -1
- package/src/cli/commands/prepare-fixtures.ts +14 -52
- package/src/cli/commands/probe/_seed-bodies.ts +52 -0
- package/src/cli/commands/probe/mass-assignment.ts +101 -10
- package/src/cli/commands/probe/security.ts +95 -12
- package/src/cli/commands/probe/webhooks.ts +2 -0
- package/src/cli/commands/probe.ts +87 -11
- package/src/cli/commands/refresh-api.ts +59 -1
- package/src/cli/commands/report-bundle.ts +3 -11
- package/src/cli/commands/request.ts +116 -0
- package/src/cli/commands/run.ts +33 -5
- package/src/cli/commands/schema-from-runs.ts +128 -0
- package/src/cli/commands/secrets.ts +133 -0
- package/src/cli/json-envelope.ts +0 -20
- package/src/cli/json-schemas.ts +51 -0
- package/src/cli/output.ts +17 -1
- package/src/cli/program.ts +5 -4
- package/src/cli/safe-live.ts +24 -0
- package/src/cli/status-filter.ts +0 -10
- package/src/core/audit/persist.ts +183 -0
- package/src/core/checks/budget.ts +59 -0
- package/src/core/checks/checks/cross_call_references.ts +17 -4
- package/src/core/checks/checks/cursor_boundary_fuzzing.ts +219 -0
- package/src/core/checks/checks/idempotency_replay.ts +1 -5
- package/src/core/checks/checks/ignored_auth.ts +44 -1
- package/src/core/checks/checks/index.ts +3 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +169 -26
- package/src/core/checks/checks/negative_data_rejection.ts +119 -16
- package/src/core/checks/checks/not_a_server_error.ts +8 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +47 -18
- package/src/core/checks/checks/pagination_invariants.ts +298 -117
- package/src/core/checks/checks/positive_data_acceptance.ts +1 -4
- package/src/core/checks/checks/status_code_conformance.ts +78 -7
- package/src/core/checks/mode.ts +3 -0
- package/src/core/checks/recommended-action.ts +5 -1
- package/src/core/checks/runner.ts +614 -27
- package/src/core/checks/spec-findings.ts +308 -0
- package/src/core/checks/types.ts +117 -1
- package/src/core/checks/zond-extensions.ts +73 -0
- package/src/core/classifier/recommended-action.ts +35 -6
- package/src/core/coverage/loader.ts +31 -0
- package/src/core/diagnostics/db-analysis.ts +200 -106
- package/src/core/diagnostics/failure-class.ts +21 -1
- package/src/core/diagnostics/failure-hints.ts +4 -208
- package/src/core/diagnostics/suggested-fixes.ts +2 -3
- package/src/core/generator/chunker.ts +1 -8
- package/src/core/generator/data-factory.ts +199 -61
- package/src/core/generator/fixtures-builder.ts +38 -31
- package/src/core/generator/index.ts +0 -2
- package/src/core/generator/openapi-reader.ts +98 -4
- package/src/core/generator/path-param-disambig.ts +30 -4
- package/src/core/generator/resources-builder.ts +276 -26
- package/src/core/generator/schema-utils.ts +22 -0
- package/src/core/generator/suite-generator.ts +168 -15
- package/src/core/generator/types.ts +6 -0
- package/src/core/identity/identity-file.ts +0 -0
- package/src/core/output/README.md +11 -29
- package/src/core/output/index.ts +1 -1
- package/src/core/output/run.ts +0 -35
- package/src/core/output/types.ts +0 -7
- package/src/core/parser/dynamic-values.ts +160 -0
- package/src/core/parser/variables.ts +0 -0
- package/src/core/probe/dry-run-envelope.ts +4 -0
- package/src/core/probe/mass-assignment/classify.ts +175 -0
- package/src/core/probe/mass-assignment/cleanup.ts +52 -0
- package/src/core/probe/mass-assignment/digest.ts +114 -0
- package/src/core/probe/mass-assignment/orchestrator.ts +459 -0
- package/src/core/probe/mass-assignment/regression.ts +141 -0
- package/src/core/probe/mass-assignment/suspects.ts +92 -0
- package/src/core/probe/mass-assignment/types.ts +135 -0
- package/src/core/probe/mass-assignment-probe.ts +23 -1118
- package/src/core/probe/mass-assignment-template.ts +32 -4
- package/src/core/probe/path-discovery.ts +3 -4
- package/src/core/probe/probe-harness.ts +21 -22
- package/src/core/probe/security/baseline.ts +174 -0
- package/src/core/probe/security/classify.ts +341 -0
- package/src/core/probe/security/cleanup.ts +125 -0
- package/src/core/probe/security/detectors.ts +71 -0
- package/src/core/probe/security/digest.ts +104 -0
- package/src/core/probe/security/orchestrator.ts +398 -0
- package/src/core/probe/security/regression.ts +103 -0
- package/src/core/probe/security/types.ts +151 -0
- package/src/core/probe/security-probe-class.ts +8 -2
- package/src/core/probe/security-probe.ts +28 -1449
- package/src/core/probe/shared.ts +26 -0
- package/src/core/probe/webhooks-probe.ts +5 -7
- package/src/core/runner/assertions.ts +1 -1
- package/src/core/runner/executor.ts +3 -18
- package/src/core/runner/form-encode.ts +8 -18
- package/src/core/runner/http-client.ts +38 -1
- package/src/core/runner/preflight-vars.ts +19 -15
- package/src/core/runner/rate-limiter.ts +11 -29
- package/src/core/runner/run-kind.ts +7 -1
- package/src/core/runner/schema-validator.ts +2 -6
- package/src/core/runner/send-request.ts +11 -6
- package/src/core/runner/types.ts +6 -0
- package/src/core/setup-api.ts +53 -15
- package/src/core/severity/index.ts +0 -63
- package/src/core/spec/infer-schema.ts +102 -0
- package/src/core/spec/merge-specs.ts +156 -0
- package/src/core/spec/schema-from-runs.ts +117 -0
- package/src/core/spec/schema-overlay.ts +130 -0
- package/src/core/util/ajv.ts +13 -0
- package/src/core/util/headers.ts +9 -0
- package/src/core/util/url.ts +24 -0
- package/src/core/workspace/fixture-gap-report.ts +84 -0
- package/src/core/workspace/fixture-gaps.ts +71 -0
- package/src/core/workspace/root.ts +13 -11
- package/src/db/migrate.ts +2 -0
- package/src/db/migrations/0002_run_kind_request.sql +59 -0
- package/src/db/queries/collections.ts +2 -2
- package/src/db/queries/results.ts +88 -0
- package/src/db/queries/runs.ts +56 -2
- package/src/db/queries.ts +3 -0
- package/src/db/schema.ts +7 -7
- package/src/cli/commands/bootstrap.ts +0 -710
- package/src/core/anti-fp/bootstrap.ts +0 -34
- package/src/core/anti-fp/index.ts +0 -33
- package/src/core/anti-fp/registry.ts +0 -44
- package/src/core/anti-fp/rules/baseline-echo.ts +0 -74
- package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +0 -52
- package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +0 -38
- package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +0 -35
- package/src/core/anti-fp/rules/schemathesis/index.ts +0 -24
- package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +0 -53
- package/src/core/anti-fp/rules/subscription-gated/index.ts +0 -11
- package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +0 -75
- package/src/core/anti-fp/types.ts +0 -68
- package/src/core/generator/create-body.ts +0 -89
|
@@ -22,6 +22,12 @@ export interface CoverageOptions {
|
|
|
22
22
|
/** ARV-28: list not-covered (and partial) endpoints inline so users
|
|
23
23
|
* don't need `--json | jq` to see what's missing. */
|
|
24
24
|
verbose?: boolean;
|
|
25
|
+
/** ARV-265: which metric block(s) to render.
|
|
26
|
+
* - 'test' — pass/hit coverage only (legacy single-block output)
|
|
27
|
+
* - 'audit' — audit-coverage only (every HTTP touch, with source
|
|
28
|
+
* breakdown). Useful when only `zond checks run` happened.
|
|
29
|
+
* - 'both' (default) — both blocks side-by-side. */
|
|
30
|
+
scope?: "test" | "audit" | "both";
|
|
25
31
|
}
|
|
26
32
|
|
|
27
33
|
const RESET = "\x1b[0m";
|
|
@@ -71,6 +77,10 @@ export interface RowBucket {
|
|
|
71
77
|
/** Latest observed HTTP status across all cells/results on this row, or
|
|
72
78
|
* `null` for unhit / network-error-only rows. */
|
|
73
79
|
lastStatus: number | null;
|
|
80
|
+
/** ARV-379: spec-declared `deprecated` (or text-flagged) for this
|
|
81
|
+
* endpoint. Lets a consumer split "real, closeable gap" from
|
|
82
|
+
* "structurally out of scope by design" without re-reading spec.json. */
|
|
83
|
+
deprecated: boolean;
|
|
74
84
|
}
|
|
75
85
|
|
|
76
86
|
export interface BucketBreakdown {
|
|
@@ -95,6 +105,7 @@ export function bucketRows(matrix: CoverageMatrix): BucketBreakdown {
|
|
|
95
105
|
method: row.method,
|
|
96
106
|
path: row.path,
|
|
97
107
|
lastStatus,
|
|
108
|
+
deprecated: !!row.deprecated,
|
|
98
109
|
};
|
|
99
110
|
if (has2xxPass) covered2xx.push(bucket);
|
|
100
111
|
else if (allResults.length > 0) coveredButNon2xx.push(bucket);
|
|
@@ -103,6 +114,78 @@ export function bucketRows(matrix: CoverageMatrix): BucketBreakdown {
|
|
|
103
114
|
return { covered2xx, coveredButNon2xx, unhit };
|
|
104
115
|
}
|
|
105
116
|
|
|
117
|
+
/**
|
|
118
|
+
* ARV-265: roll up the audit-coverage matrix per producer (`run_kind`).
|
|
119
|
+
* `reached` counts how many spec endpoints have at least one result row
|
|
120
|
+
* across any kind; `totalEvents` is the raw result count (one per HTTP
|
|
121
|
+
* touch). `bySource` keys are the run_kind values that actually appear
|
|
122
|
+
* in `cov.runs` — kinds with zero contribution are omitted from the
|
|
123
|
+
* map so the text reporter doesn't print empty rows.
|
|
124
|
+
*
|
|
125
|
+
* Endpoint attribution uses the same matrix engine output the loader
|
|
126
|
+
* already built (each `MatrixRow` cell carries the results that mapped
|
|
127
|
+
* to that endpoint), but the engine doesn't track which run produced
|
|
128
|
+
* each result. We re-walk the contributing runs and re-load their
|
|
129
|
+
* results to bucket by kind. For the 1184-endpoint github-spec scan
|
|
130
|
+
* this is O(N) over ~1500 result rows — negligible vs the run cost.
|
|
131
|
+
*/
|
|
132
|
+
function computeAuditBreakdown(cov: import("../../core/coverage/loader.ts").CoverageLoadResult): {
|
|
133
|
+
reached: number;
|
|
134
|
+
total: number;
|
|
135
|
+
percentage: number;
|
|
136
|
+
totalEvents: number;
|
|
137
|
+
bySource: Record<string, { endpoints: number; events: number }>;
|
|
138
|
+
} {
|
|
139
|
+
const total = cov.matrix.rows.length;
|
|
140
|
+
// reached: endpoints with any result (cells with at least one entry).
|
|
141
|
+
let reached = 0;
|
|
142
|
+
for (const row of cov.matrix.rows) {
|
|
143
|
+
const cells = Object.values(row.cells);
|
|
144
|
+
if (cells.some((c) => c.results.length > 0)) reached += 1;
|
|
145
|
+
}
|
|
146
|
+
const totalEvents = cov.matrix.rows.reduce(
|
|
147
|
+
(sum, row) => sum + Object.values(row.cells).reduce((s, c) => s + c.results.length, 0),
|
|
148
|
+
0,
|
|
149
|
+
);
|
|
150
|
+
// Re-load per-run results to attribute counts to the producing kind.
|
|
151
|
+
const bySource: Record<string, { endpoints: Set<string>; events: number }> = {};
|
|
152
|
+
for (const run of cov.runs) {
|
|
153
|
+
const kind = run.run_kind;
|
|
154
|
+
const bucket = bySource[kind] ?? { endpoints: new Set<string>(), events: 0 };
|
|
155
|
+
const rows = getResultsByRunId(run.id);
|
|
156
|
+
for (const r of rows) {
|
|
157
|
+
// Skip pre-dispatch skips (max-requests cap, schema-gated) — they have
|
|
158
|
+
// no response and don't represent an actual HTTP touch.
|
|
159
|
+
if (r.response_status == null) continue;
|
|
160
|
+
bucket.events += 1;
|
|
161
|
+
if (typeof r.request_method === "string" && typeof r.request_url === "string") {
|
|
162
|
+
// The result table stores absolute URLs; the matrix engine uses
|
|
163
|
+
// the spec path-template regex to bucket them. For the by-source
|
|
164
|
+
// count we don't need that level of precision — pathname-only is
|
|
165
|
+
// close enough for "did this kind touch endpoint X" telemetry.
|
|
166
|
+
try {
|
|
167
|
+
const u = new URL(r.request_url);
|
|
168
|
+
bucket.endpoints.add(`${r.request_method.toUpperCase()} ${u.pathname}`);
|
|
169
|
+
} catch {
|
|
170
|
+
bucket.endpoints.add(`${r.request_method.toUpperCase()} ${r.request_url}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
bySource[kind] = bucket;
|
|
175
|
+
}
|
|
176
|
+
const flatBySource: Record<string, { endpoints: number; events: number }> = {};
|
|
177
|
+
for (const [k, v] of Object.entries(bySource)) {
|
|
178
|
+
flatBySource[k] = { endpoints: v.endpoints.size, events: v.events };
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
reached,
|
|
182
|
+
total,
|
|
183
|
+
percentage: total === 0 ? 0 : Math.round((reached / total) * 100),
|
|
184
|
+
totalEvents,
|
|
185
|
+
bySource: flatBySource,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
106
189
|
function lastObservedStatus(results: { responseStatus: number | null }[]): number | null {
|
|
107
190
|
for (let i = results.length - 1; i >= 0; i--) {
|
|
108
191
|
const s = results[i]?.responseStatus;
|
|
@@ -133,24 +216,54 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
|
|
|
133
216
|
*/
|
|
134
217
|
async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
|
|
135
218
|
getDb();
|
|
136
|
-
const
|
|
219
|
+
const scope = options.scope ?? "both";
|
|
220
|
+
const wantTest = scope === "test" || scope === "both";
|
|
221
|
+
const wantAudit = scope === "audit" || scope === "both";
|
|
222
|
+
|
|
223
|
+
const loaderArgs = {
|
|
137
224
|
apiName: options.apiName!,
|
|
138
225
|
...(options.sessionId ? { sessionId: options.sessionId } : {}),
|
|
139
226
|
...(options.sinceIso ? { sinceIso: options.sinceIso } : {}),
|
|
140
227
|
...(options.tag ? { tag: options.tag } : {}),
|
|
141
228
|
...(options.runIds && options.runIds.length > 0 ? { runIds: options.runIds } : {}),
|
|
142
229
|
...(options.runId != null ? { runId: options.runId } : {}),
|
|
143
|
-
}
|
|
230
|
+
} as const;
|
|
231
|
+
|
|
232
|
+
// ARV-265: load the matrix twice when scope=both — once for the
|
|
233
|
+
// pass/hit "did our suites land 2xx" metric (`scope: 'test'`), and once
|
|
234
|
+
// for the "did zond touch the endpoint at all" audit metric
|
|
235
|
+
// (`scope: 'audit'`). Loader-level scoping keeps the matrix shape
|
|
236
|
+
// identical so both branches reuse classifyRows / bucketRows.
|
|
237
|
+
const covTest = wantTest ? await loadCoverage({ ...loaderArgs, scope: "test" }) : null;
|
|
238
|
+
const covAudit = wantAudit ? await loadCoverage({ ...loaderArgs, scope: "audit" }) : null;
|
|
239
|
+
// Pick whichever block is present as the canonical "cov" — we still
|
|
240
|
+
// need a matrix to enumerate endpoints + spec warnings.
|
|
241
|
+
const cov = covTest ?? covAudit!;
|
|
144
242
|
const total = cov.matrix.rows.length;
|
|
145
243
|
if (total === 0) {
|
|
146
244
|
printError("No endpoints found in the OpenAPI spec");
|
|
147
245
|
return 1;
|
|
148
246
|
}
|
|
149
247
|
|
|
150
|
-
const { coveredRows, partialRows, uncoveredRows } = classifyRows(
|
|
248
|
+
const { coveredRows, partialRows, uncoveredRows } = classifyRows((covTest ?? covAudit!).matrix);
|
|
151
249
|
const coveredCount = coveredRows.length;
|
|
152
250
|
const percentage = Math.round((coveredCount / total) * 100);
|
|
153
251
|
|
|
252
|
+
// ARV-303: envelope/exit-code contract. When the selector resolved to zero
|
|
253
|
+
// runs (closed session, --session-id with no runs, --union tag with no
|
|
254
|
+
// matches), there is no coverage data — that is the only command-level
|
|
255
|
+
// failure of the matrix path. Everything else (uncovered endpoints remain)
|
|
256
|
+
// is a data point, not a failure, so it must not gate the exit code.
|
|
257
|
+
const noRuns = (covTest ?? covAudit!).runs.length === 0;
|
|
258
|
+
|
|
259
|
+
// ARV-265: per-source breakdown for audit-coverage. Walks each
|
|
260
|
+
// contributing run, groups its results by (METHOD, path-template), and
|
|
261
|
+
// tallies distinct endpoints reached + raw event counts. The match is
|
|
262
|
+
// best-effort (uses the same endpoint regex the matrix engine does) so
|
|
263
|
+
// a run that hit URLs not declared in the spec is still counted by
|
|
264
|
+
// event-count even when no endpoint maps.
|
|
265
|
+
const auditBreakdown = covAudit ? computeAuditBreakdown(covAudit) : null;
|
|
266
|
+
|
|
154
267
|
// TASK-270: two metrics, two intents:
|
|
155
268
|
// pass_coverage = endpoints with at least one passing 2xx (strict — what
|
|
156
269
|
// single-run output has always meant)
|
|
@@ -181,48 +294,64 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
|
|
|
181
294
|
|
|
182
295
|
if (!options.json) {
|
|
183
296
|
const color = useColor();
|
|
297
|
+
const runsForLabel = covTest ?? covAudit!;
|
|
184
298
|
let runLabel: string;
|
|
185
|
-
if (
|
|
186
|
-
runLabel =
|
|
187
|
-
? ` — no runs match --union ${
|
|
299
|
+
if (runsForLabel.runs.length === 0) {
|
|
300
|
+
runLabel = runsForLabel.unionMode
|
|
301
|
+
? ` — no runs match --union ${runsForLabel.unionMode}`
|
|
188
302
|
: " — no runs yet";
|
|
189
303
|
}
|
|
190
|
-
else if (
|
|
304
|
+
else if (runsForLabel.runs.length === 1) runLabel = ` — Run #${runsForLabel.runs[0]!.id}`;
|
|
191
305
|
else {
|
|
192
|
-
const modeLabel =
|
|
193
|
-
runLabel = ` — union${modeLabel} of ${
|
|
306
|
+
const modeLabel = runsForLabel.unionMode ? ` ${runsForLabel.unionMode}` : "";
|
|
307
|
+
runLabel = ` — union${modeLabel} of ${runsForLabel.runs.length} runs (#${runsForLabel.runs.map(r => r.id).join(", #")})`;
|
|
194
308
|
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
309
|
+
if (wantTest) {
|
|
310
|
+
// TASK-270: show both metrics on separate lines so CI/triage scripts
|
|
311
|
+
// and humans can pick the one they care about.
|
|
312
|
+
console.log(`test-coverage`);
|
|
313
|
+
console.log(` Pass-coverage (passing 2xx): ${passCount}/${total} endpoints (${passPct}%)${runLabel}`);
|
|
314
|
+
console.log(` Hit-coverage (any response): ${hitCount}/${total} endpoints (${hitPct}%)`);
|
|
315
|
+
// ARV-265: clarified source line. Previously: "only zond run results — probes not counted".
|
|
316
|
+
// After ARV-265, audit-coverage IS the answer for "did checks run touch X" — point users at it.
|
|
317
|
+
console.log(` ${color ? DIM : ""}(source: \`zond run\` results — for \`checks run\` / \`probe\` touches see audit-coverage below)${color ? RESET : ""}`);
|
|
318
|
+
console.log("");
|
|
204
319
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
320
|
+
if (passing > 0) {
|
|
321
|
+
console.log(` ${color ? GREEN : ""}✅ ${passing} covered (passing 2xx)${color ? RESET : ""}`);
|
|
322
|
+
}
|
|
323
|
+
if (apiError > 0) {
|
|
324
|
+
console.log(` ${color ? YELLOW : ""}⚠️ ${apiError} returning 5xx (possibly broken API)${color ? RESET : ""}`);
|
|
325
|
+
}
|
|
326
|
+
if (testFailed > 0) {
|
|
327
|
+
console.log(` ${color ? RED : ""}❌ ${testFailed} hit endpoint but assertions failed${color ? RESET : ""}`);
|
|
328
|
+
}
|
|
329
|
+
if (partialRows.length > 0 && testFailed === 0) {
|
|
330
|
+
console.log(` ${color ? YELLOW : ""}◐ ${partialRows.length} partial (only non-2xx responses)${color ? RESET : ""}`);
|
|
331
|
+
}
|
|
332
|
+
if (uncoveredRows.length > 0) {
|
|
333
|
+
console.log(` ${color ? DIM : ""}⬜ ${uncoveredRows.length} not covered${color ? RESET : ""}`);
|
|
334
|
+
// ARV-75 (feedback round-03 / F16): when some of the not-covered rows
|
|
335
|
+
// are deprecated, surface the count so a user reading "5% uncovered"
|
|
336
|
+
// can attribute the gap to deprecated endpoints (which generate skips
|
|
337
|
+
// by default) instead of suite regression.
|
|
338
|
+
const deprecatedUnhit = uncoveredRows.filter((r) => r.deprecated).length;
|
|
339
|
+
if (deprecatedUnhit > 0) {
|
|
340
|
+
console.log(` ${color ? DIM : ""}↳ ${deprecatedUnhit} of those are deprecated (skipped by \`zond generate\` unless --include-deprecated)${color ? RESET : ""}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
216
343
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
344
|
+
|
|
345
|
+
if (wantAudit && auditBreakdown) {
|
|
346
|
+
if (wantTest) console.log("");
|
|
347
|
+
console.log(`audit-coverage: ${auditBreakdown.reached}/${auditBreakdown.total} endpoints (${auditBreakdown.percentage}%, ${auditBreakdown.totalEvents} HTTP touches)`);
|
|
348
|
+
console.log(` ${color ? DIM : ""}(any zond producer: run, checks, probe, request, fixture-cascade — ARV-265)${color ? RESET : ""}`);
|
|
349
|
+
const sources = Object.entries(auditBreakdown.bySource).filter(([, v]) => v.events > 0);
|
|
350
|
+
if (sources.length > 0) {
|
|
351
|
+
console.log(` by source:`);
|
|
352
|
+
for (const [kind, v] of sources) {
|
|
353
|
+
console.log(` ${kind.padEnd(8)} ${String(v.endpoints).padStart(4)} endpoints, ${String(v.events).padStart(5)} events`);
|
|
354
|
+
}
|
|
226
355
|
}
|
|
227
356
|
}
|
|
228
357
|
|
|
@@ -264,8 +393,14 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
|
|
|
264
393
|
// fields (covered/uncovered/partial, coveredEndpoints/partialEndpoints/
|
|
265
394
|
// uncoveredEndpoints) are kept as deprecated aliases pending full envelope-
|
|
266
395
|
// policy unification (TASK-184).
|
|
267
|
-
const buckets = bucketRows(
|
|
268
|
-
|
|
396
|
+
const buckets = bucketRows((covTest ?? covAudit!).matrix);
|
|
397
|
+
// ARV-265: dual-metric JSON envelope. `test_coverage` mirrors the
|
|
398
|
+
// pre-ARV-265 single-block shape (kept as the canonical
|
|
399
|
+
// pass/hit pair); `audit_coverage` is additive and only present
|
|
400
|
+
// when scope includes audit. Legacy top-level fields stay populated
|
|
401
|
+
// from the test block when present (back-compat with TASK-280
|
|
402
|
+
// consumers) so existing CI scripts don't break.
|
|
403
|
+
const json: Record<string, unknown> = {
|
|
269
404
|
// Legacy aliases — DO NOT add new consumers; use `totals.*` and
|
|
270
405
|
// `*Endpoints` arrays below instead.
|
|
271
406
|
covered: coveredCount,
|
|
@@ -301,13 +436,52 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
|
|
|
301
436
|
// by zond generate" without re-deriving from the spec.
|
|
302
437
|
deprecated_unhit: uncoveredRows.filter((r) => r.deprecated).length,
|
|
303
438
|
deprecated_total: cov.matrix.rows.filter((r) => r.deprecated).length,
|
|
304
|
-
|
|
439
|
+
// ARV-379: the endpoint keys (method+path) marked deprecated in the
|
|
440
|
+
// spec. Intersect with `uncoveredEndpoints` to split "real, closeable
|
|
441
|
+
// gap" from "deprecated, skip by design" without re-reading spec.json.
|
|
442
|
+
// (RowBucket entries in the *Endpoints arrays also carry a per-entry
|
|
443
|
+
// `deprecated` boolean for the object-shaped consumers.)
|
|
444
|
+
deprecatedEndpoints: cov.matrix.rows.filter((r) => r.deprecated).map((r) => r.endpoint),
|
|
445
|
+
// ARV-265: dual-metric envelope.
|
|
446
|
+
test_coverage: wantTest ? {
|
|
447
|
+
pass: { covered: passCount, total, ratio: total === 0 ? 0 : Number((passCount / total).toFixed(4)) },
|
|
448
|
+
hit: { covered: hitCount, total, ratio: total === 0 ? 0 : Number((hitCount / total).toFixed(4)) },
|
|
449
|
+
runIds: covTest?.runs.map((r) => r.id) ?? [],
|
|
450
|
+
} : null,
|
|
451
|
+
};
|
|
452
|
+
if (auditBreakdown) {
|
|
453
|
+
json.audit_coverage = {
|
|
454
|
+
reached: auditBreakdown.reached,
|
|
455
|
+
total: auditBreakdown.total,
|
|
456
|
+
ratio: auditBreakdown.total === 0 ? 0 : Number((auditBreakdown.reached / auditBreakdown.total).toFixed(4)),
|
|
457
|
+
events: auditBreakdown.totalEvents,
|
|
458
|
+
by_source: auditBreakdown.bySource,
|
|
459
|
+
runIds: covAudit?.runs.map((r) => r.id) ?? [],
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
// ARV-303: surface the zero-runs case as ok:false so the non-zero exit
|
|
463
|
+
// lines up with the envelope shape, instead of an ok:true envelope
|
|
464
|
+
// alongside exit 1.
|
|
465
|
+
if (noRuns) {
|
|
466
|
+
const sel = cov.unionMode ?? "selection";
|
|
467
|
+
printJson(jsonError("coverage", [
|
|
468
|
+
`No runs match ${sel} — coverage cannot be computed against zero runs`,
|
|
469
|
+
]));
|
|
470
|
+
} else {
|
|
471
|
+
printJson(jsonOk("coverage", json));
|
|
472
|
+
}
|
|
305
473
|
}
|
|
306
474
|
|
|
475
|
+
// ARV-303: exit-code contract. ok:false (no runs) ⇒ exit 1. Otherwise the
|
|
476
|
+
// coverage was computed successfully (ok:true) ⇒ exit 0 by default — an
|
|
477
|
+
// orchestrator must be able to tell "coverage ran" from "command failed".
|
|
478
|
+
// "Uncovered endpoints remain" no longer gates the exit on its own; use
|
|
479
|
+
// --fail-on-coverage to opt into a threshold gate.
|
|
480
|
+
if (noRuns) return 1;
|
|
307
481
|
if (options.failOnCoverage !== undefined) {
|
|
308
482
|
return percentage < options.failOnCoverage ? 1 : 0;
|
|
309
483
|
}
|
|
310
|
-
return
|
|
484
|
+
return 0;
|
|
311
485
|
}
|
|
312
486
|
|
|
313
487
|
/**
|
|
@@ -346,6 +520,13 @@ async function runSpecOnlyCoverage(options: CoverageOptions): Promise<number> {
|
|
|
346
520
|
}
|
|
347
521
|
}
|
|
348
522
|
} else {
|
|
523
|
+
// TASK-250 + ARV-303: the spec-only path emits a shape-stable
|
|
524
|
+
// ok:true envelope even when there's no registered API. The
|
|
525
|
+
// legacy contract here is "we parsed the spec, here is 0% with a
|
|
526
|
+
// null runId" — agents and CI scripts depend on the key shape.
|
|
527
|
+
// ARV-303 only tightens the matrix-coverage path (no runs match
|
|
528
|
+
// --union / --session-id), which DID emit ok:true alongside exit
|
|
529
|
+
// 1; the spec-only path was intentionally informational.
|
|
349
530
|
const unhit = allEndpoints.map((ep) => ({
|
|
350
531
|
endpoint: `${ep.method.toUpperCase()} ${ep.path}`,
|
|
351
532
|
method: ep.method.toUpperCase(),
|
|
@@ -366,8 +547,6 @@ async function runSpecOnlyCoverage(options: CoverageOptions): Promise<number> {
|
|
|
366
547
|
covered2xxEndpoints: [],
|
|
367
548
|
coveredButNon2xxEndpoints: [],
|
|
368
549
|
unhitEndpoints: unhit,
|
|
369
|
-
// TASK-270: the spec-only path has no run results, so both metrics
|
|
370
|
-
// are zero. Surface them anyway for shape-stable JSON.
|
|
371
550
|
pass_coverage: { covered: 0, total, ratio: 0 },
|
|
372
551
|
hit_coverage: { covered: 0, total, ratio: 0 },
|
|
373
552
|
}));
|
|
@@ -385,7 +564,7 @@ import { globalJson, resolveApiCollection } from "../resolve.ts";
|
|
|
385
564
|
import { parseInteger, parsePercentage } from "../argv.ts";
|
|
386
565
|
import { getApi } from "../util/api-context.ts";
|
|
387
566
|
import { readCurrentSession } from "../../core/context/session.ts";
|
|
388
|
-
import { listRunsBySession, getLatestRunByCollection, getRunById, findCollectionByNameOrId } from "../../db/queries.ts";
|
|
567
|
+
import { listRunsBySession, getLatestRunByCollection, getRunById, findCollectionByNameOrId, listSessions, getResultsByRunId } from "../../db/queries.ts";
|
|
389
568
|
|
|
390
569
|
/**
|
|
391
570
|
* ARV-55: probe-run classification moved from path-regex heuristic into the
|
|
@@ -476,14 +655,16 @@ export function registerCoverage(program: Command): void {
|
|
|
476
655
|
.command("coverage")
|
|
477
656
|
.description(
|
|
478
657
|
"Analyze API test coverage from stored run results. zond reports two " +
|
|
479
|
-
"
|
|
480
|
-
"
|
|
481
|
-
"
|
|
658
|
+
"metric blocks side-by-side (ARV-265):\n" +
|
|
659
|
+
" test-coverage — pass/hit from `zond run` (regular + probe runs only)\n" +
|
|
660
|
+
" · pass-coverage — endpoint had at least one passing 2xx (strict; CI gate via --fail-on-coverage)\n" +
|
|
661
|
+
" · hit-coverage — endpoint received any response, incl. 5xx / assertion-fail (loose; breadth)\n" +
|
|
662
|
+
" audit-coverage — any HTTP touch from any producer: `run`, `checks run`,\n" +
|
|
663
|
+
" `probe`, `request`, `prepare-fixtures --cascade`. With by-source breakdown.\n" +
|
|
482
664
|
"\n" +
|
|
483
|
-
"
|
|
484
|
-
"
|
|
485
|
-
"
|
|
486
|
-
"execute them via `zond run` to grow coverage.\n" +
|
|
665
|
+
"Use --scope to print just one block: `--scope test` for legacy single-metric\n" +
|
|
666
|
+
"output, `--scope audit` to answer 'did the scan reach the API at all'.\n" +
|
|
667
|
+
"Producers opt out via ZOND_CHECKS_PERSIST=0 (default: persist on).\n" +
|
|
487
668
|
"\n" +
|
|
488
669
|
"Defaults to the latest stored run for the resolved API; pass " +
|
|
489
670
|
"--run-id to pin a specific run, or --union <selector> to combine " +
|
|
@@ -510,10 +691,12 @@ export function registerCoverage(program: Command): void {
|
|
|
510
691
|
" zond run apis/<api>/probes\n" +
|
|
511
692
|
" zond coverage --api <api> --union session\n" +
|
|
512
693
|
"\n" +
|
|
513
|
-
"Exit codes: 0 =
|
|
514
|
-
"
|
|
515
|
-
"
|
|
516
|
-
"--fail-on-coverage
|
|
694
|
+
"Exit codes (ARV-303): 0 = coverage computed successfully (envelope " +
|
|
695
|
+
"ok:true — uncovered endpoints remaining is a data point, not a " +
|
|
696
|
+
"failure); 1 = coverage could not be computed (selector resolved to " +
|
|
697
|
+
"zero runs) or pass-coverage < --fail-on-coverage; 2 = bad input or " +
|
|
698
|
+
"read error. Uncovered endpoints only gate the exit when you opt in " +
|
|
699
|
+
"with --fail-on-coverage; that flag gates pass-coverage, not hit-coverage.",
|
|
517
700
|
)
|
|
518
701
|
.option("--api <name>", "Use API collection (auto-resolves spec; reads stored runs)")
|
|
519
702
|
.option("--spec <path>", "Spec-only fallback when no API is registered (no run results)")
|
|
@@ -526,6 +709,12 @@ export function registerCoverage(program: Command): void {
|
|
|
526
709
|
)
|
|
527
710
|
.option("--db <path>", "Path to SQLite database file")
|
|
528
711
|
.option("--verbose", "List not-covered (and partial) endpoints inline — same data as `--json` but human-readable")
|
|
712
|
+
.addOption(
|
|
713
|
+
new Option(
|
|
714
|
+
"--scope <scope>",
|
|
715
|
+
"ARV-265: which coverage block(s) to print. `test` = pass-coverage/hit-coverage only (legacy single block from `zond run` results). `audit` = audit-coverage only (every HTTP touch from `run` / `checks run` / `probe` / `request` / fixture-cascade, with by-source breakdown). `both` (default) = both side-by-side.",
|
|
716
|
+
).choices(["test", "audit", "both"]).default("both"),
|
|
717
|
+
)
|
|
529
718
|
// ARV-35: `--format json` matches the kubectl/gh/aws-cli convention many
|
|
530
719
|
// users reach for first; until ARV-54 lands a workspace-wide alias layer
|
|
531
720
|
// we accept it locally and forward to `--json`. Other values are rejected
|
|
@@ -566,7 +755,20 @@ export function registerCoverage(program: Command): void {
|
|
|
566
755
|
if (parsed.kind === "session") {
|
|
567
756
|
const current = readCurrentSession();
|
|
568
757
|
if (!current) {
|
|
569
|
-
|
|
758
|
+
// ARV-234: when there's no active session, peek at the most
|
|
759
|
+
// recent one — agents typically hit this right after
|
|
760
|
+
// `session end`. Surface its id in the error so the recovery
|
|
761
|
+
// path is one copy-paste instead of `db sessions`-spelunking.
|
|
762
|
+
let hint = "";
|
|
763
|
+
try {
|
|
764
|
+
const recent = listSessions(1, 0);
|
|
765
|
+
if (recent.length > 0) {
|
|
766
|
+
const r = recent[0]!;
|
|
767
|
+
const endedAt = r.finished_at ? ` (ended ${r.finished_at})` : "";
|
|
768
|
+
hint = ` Most recent session: --session-id ${r.session_id}${endedAt}.`;
|
|
769
|
+
}
|
|
770
|
+
} catch { /* best effort */ }
|
|
771
|
+
printError(`--union session requires an active session (run 'zond session start' first), or pass --session-id <id>.${hint}`);
|
|
570
772
|
process.exitCode = 2;
|
|
571
773
|
return;
|
|
572
774
|
}
|
|
@@ -604,12 +806,14 @@ export function registerCoverage(program: Command): void {
|
|
|
604
806
|
// (the percentage already looked like a regression). Explicit
|
|
605
807
|
// selectors win, --json keeps the envelope untouched.
|
|
606
808
|
const noSelector = !opts.runId && !sessionId && !runIds && !sinceIso && !tag;
|
|
809
|
+
let promotedToSession = false;
|
|
607
810
|
if (apiName && noSelector) {
|
|
608
811
|
const current = readCurrentSession();
|
|
609
812
|
if (current) {
|
|
610
813
|
const sessRuns = listRunsBySession(current.id);
|
|
611
814
|
if (sessRuns.length > 1) {
|
|
612
815
|
sessionId = current.id;
|
|
816
|
+
promotedToSession = true;
|
|
613
817
|
if (!globalJson(cmd)) {
|
|
614
818
|
process.stderr.write(
|
|
615
819
|
`zond: active session has ${sessRuns.length} runs — defaulting to --union session (pass --run-id <N> for a single run).\n`,
|
|
@@ -634,10 +838,23 @@ export function registerCoverage(program: Command): void {
|
|
|
634
838
|
`For combined coverage, wrap your runs in 'zond session start/end' and pass '--union session' here.`;
|
|
635
839
|
process.stderr.write(`zond: ${hint}\n`);
|
|
636
840
|
}
|
|
841
|
+
// ARV-81: parity with the session-promotion footer above —
|
|
842
|
+
// when we *don't* promote to --union session (no session, or
|
|
843
|
+
// session has 1 run), tell the user which run they're seeing
|
|
844
|
+
// so the single-run snapshot can't be mistaken for a regression.
|
|
845
|
+
if (!promotedToSession && !globalJson(cmd)) {
|
|
846
|
+
const regular = getLatestRunByCollection(collection.id, { runKind: "regular" });
|
|
847
|
+
if (regular) {
|
|
848
|
+
process.stderr.write(
|
|
849
|
+
`zond: using latest run #${regular.id}. For union, pass '--union since:<dur>' or '--union runs:<a,b,...>'.\n`,
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
637
853
|
}
|
|
638
854
|
} catch { /* DB inspection is best-effort, don't break coverage */ }
|
|
639
855
|
}
|
|
640
856
|
|
|
857
|
+
const scope = (opts.scope === "test" || opts.scope === "audit") ? opts.scope : "both";
|
|
641
858
|
process.exitCode = await coverageCommand({
|
|
642
859
|
...(apiName ? { apiName } : {}),
|
|
643
860
|
...(spec ? { spec } : {}),
|
|
@@ -649,6 +866,7 @@ export function registerCoverage(program: Command): void {
|
|
|
649
866
|
...(tag ? { tag } : {}),
|
|
650
867
|
json: globalJson(cmd),
|
|
651
868
|
verbose: opts.verbose === true,
|
|
869
|
+
scope,
|
|
652
870
|
});
|
|
653
871
|
});
|
|
654
872
|
}
|