@kirrosh/zond 0.23.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/CHANGELOG.md +164 -1
  2. package/README.md +8 -7
  3. package/package.json +2 -3
  4. package/src/CLAUDE.md +112 -0
  5. package/src/cli/commands/add-api.ts +19 -7
  6. package/src/cli/commands/api/annotate/index.ts +359 -4
  7. package/src/cli/commands/api/annotate/lifecycle.ts +1 -1
  8. package/src/cli/commands/api/annotate/overlay.ts +1 -1
  9. package/src/cli/commands/api/annotate/pagination.ts +10 -6
  10. package/src/cli/commands/api/annotate/prompts.ts +39 -2
  11. package/src/cli/commands/audit.ts +360 -54
  12. package/src/cli/commands/check.ts +15 -2
  13. package/src/cli/commands/checks.ts +352 -36
  14. package/src/cli/commands/cleanup.ts +4 -30
  15. package/src/cli/commands/coverage.ts +275 -57
  16. package/src/cli/commands/db.ts +311 -8
  17. package/src/cli/commands/discover.ts +281 -161
  18. package/src/cli/commands/doctor.ts +57 -3
  19. package/src/cli/commands/fixtures.ts +1 -1
  20. package/src/cli/commands/generate.ts +24 -7
  21. package/src/cli/commands/init/bootstrap.ts +4 -1
  22. package/src/cli/commands/init/index.ts +2 -2
  23. package/src/cli/commands/init/skills.ts +43 -0
  24. package/src/cli/commands/init/templates/agents.md +12 -3
  25. package/src/cli/commands/init/templates/skills/warm-up-target.md +122 -0
  26. package/src/cli/commands/init/templates/skills/zond-checks.md +268 -44
  27. package/src/cli/commands/init/templates/skills/zond-seed.md +114 -0
  28. package/src/cli/commands/init/templates/skills/zond-triage.md +88 -26
  29. package/src/cli/commands/init/templates/skills/zond.md +274 -64
  30. package/src/cli/commands/init/templates/zond-config.yml +1 -1
  31. package/src/cli/commands/prepare-fixtures.ts +14 -52
  32. package/src/cli/commands/probe/_seed-bodies.ts +52 -0
  33. package/src/cli/commands/probe/mass-assignment.ts +101 -10
  34. package/src/cli/commands/probe/security.ts +95 -12
  35. package/src/cli/commands/probe/webhooks.ts +2 -0
  36. package/src/cli/commands/probe.ts +87 -11
  37. package/src/cli/commands/refresh-api.ts +59 -1
  38. package/src/cli/commands/report-bundle.ts +3 -11
  39. package/src/cli/commands/request.ts +116 -0
  40. package/src/cli/commands/run.ts +33 -5
  41. package/src/cli/commands/schema-from-runs.ts +128 -0
  42. package/src/cli/commands/secrets.ts +133 -0
  43. package/src/cli/json-envelope.ts +0 -20
  44. package/src/cli/json-schemas.ts +51 -0
  45. package/src/cli/output.ts +17 -1
  46. package/src/cli/program.ts +5 -4
  47. package/src/cli/safe-live.ts +24 -0
  48. package/src/cli/status-filter.ts +0 -10
  49. package/src/core/audit/persist.ts +183 -0
  50. package/src/core/checks/budget.ts +59 -0
  51. package/src/core/checks/checks/cross_call_references.ts +17 -4
  52. package/src/core/checks/checks/cursor_boundary_fuzzing.ts +219 -0
  53. package/src/core/checks/checks/idempotency_replay.ts +1 -5
  54. package/src/core/checks/checks/ignored_auth.ts +44 -1
  55. package/src/core/checks/checks/index.ts +3 -0
  56. package/src/core/checks/checks/lifecycle_transitions.ts +169 -26
  57. package/src/core/checks/checks/negative_data_rejection.ts +119 -16
  58. package/src/core/checks/checks/not_a_server_error.ts +8 -0
  59. package/src/core/checks/checks/open_cors_on_sensitive.ts +47 -18
  60. package/src/core/checks/checks/pagination_invariants.ts +298 -117
  61. package/src/core/checks/checks/positive_data_acceptance.ts +1 -4
  62. package/src/core/checks/checks/status_code_conformance.ts +78 -7
  63. package/src/core/checks/mode.ts +3 -0
  64. package/src/core/checks/recommended-action.ts +5 -1
  65. package/src/core/checks/runner.ts +614 -27
  66. package/src/core/checks/spec-findings.ts +308 -0
  67. package/src/core/checks/types.ts +117 -1
  68. package/src/core/checks/zond-extensions.ts +73 -0
  69. package/src/core/classifier/recommended-action.ts +35 -6
  70. package/src/core/coverage/loader.ts +31 -0
  71. package/src/core/diagnostics/db-analysis.ts +200 -106
  72. package/src/core/diagnostics/failure-class.ts +21 -1
  73. package/src/core/diagnostics/failure-hints.ts +4 -208
  74. package/src/core/diagnostics/suggested-fixes.ts +2 -3
  75. package/src/core/generator/chunker.ts +1 -8
  76. package/src/core/generator/data-factory.ts +199 -61
  77. package/src/core/generator/fixtures-builder.ts +38 -31
  78. package/src/core/generator/index.ts +0 -2
  79. package/src/core/generator/openapi-reader.ts +98 -4
  80. package/src/core/generator/path-param-disambig.ts +30 -4
  81. package/src/core/generator/resources-builder.ts +276 -26
  82. package/src/core/generator/schema-utils.ts +22 -0
  83. package/src/core/generator/suite-generator.ts +168 -15
  84. package/src/core/generator/types.ts +6 -0
  85. package/src/core/identity/identity-file.ts +0 -0
  86. package/src/core/output/README.md +11 -29
  87. package/src/core/output/index.ts +1 -1
  88. package/src/core/output/run.ts +0 -35
  89. package/src/core/output/types.ts +0 -7
  90. package/src/core/parser/dynamic-values.ts +160 -0
  91. package/src/core/parser/variables.ts +0 -0
  92. package/src/core/probe/dry-run-envelope.ts +4 -0
  93. package/src/core/probe/mass-assignment/classify.ts +175 -0
  94. package/src/core/probe/mass-assignment/cleanup.ts +52 -0
  95. package/src/core/probe/mass-assignment/digest.ts +114 -0
  96. package/src/core/probe/mass-assignment/orchestrator.ts +459 -0
  97. package/src/core/probe/mass-assignment/regression.ts +141 -0
  98. package/src/core/probe/mass-assignment/suspects.ts +92 -0
  99. package/src/core/probe/mass-assignment/types.ts +135 -0
  100. package/src/core/probe/mass-assignment-probe.ts +23 -1118
  101. package/src/core/probe/mass-assignment-template.ts +32 -4
  102. package/src/core/probe/path-discovery.ts +3 -4
  103. package/src/core/probe/probe-harness.ts +21 -22
  104. package/src/core/probe/security/baseline.ts +174 -0
  105. package/src/core/probe/security/classify.ts +341 -0
  106. package/src/core/probe/security/cleanup.ts +125 -0
  107. package/src/core/probe/security/detectors.ts +71 -0
  108. package/src/core/probe/security/digest.ts +104 -0
  109. package/src/core/probe/security/orchestrator.ts +398 -0
  110. package/src/core/probe/security/regression.ts +103 -0
  111. package/src/core/probe/security/types.ts +151 -0
  112. package/src/core/probe/security-probe-class.ts +8 -2
  113. package/src/core/probe/security-probe.ts +28 -1449
  114. package/src/core/probe/shared.ts +26 -0
  115. package/src/core/probe/webhooks-probe.ts +5 -7
  116. package/src/core/runner/assertions.ts +1 -1
  117. package/src/core/runner/executor.ts +3 -18
  118. package/src/core/runner/form-encode.ts +8 -18
  119. package/src/core/runner/http-client.ts +38 -1
  120. package/src/core/runner/preflight-vars.ts +19 -15
  121. package/src/core/runner/rate-limiter.ts +11 -29
  122. package/src/core/runner/run-kind.ts +7 -1
  123. package/src/core/runner/schema-validator.ts +2 -6
  124. package/src/core/runner/send-request.ts +11 -6
  125. package/src/core/runner/types.ts +6 -0
  126. package/src/core/setup-api.ts +53 -15
  127. package/src/core/severity/index.ts +0 -63
  128. package/src/core/spec/infer-schema.ts +102 -0
  129. package/src/core/spec/merge-specs.ts +156 -0
  130. package/src/core/spec/schema-from-runs.ts +117 -0
  131. package/src/core/spec/schema-overlay.ts +130 -0
  132. package/src/core/util/ajv.ts +13 -0
  133. package/src/core/util/headers.ts +9 -0
  134. package/src/core/util/url.ts +24 -0
  135. package/src/core/workspace/fixture-gap-report.ts +84 -0
  136. package/src/core/workspace/fixture-gaps.ts +71 -0
  137. package/src/core/workspace/root.ts +13 -11
  138. package/src/db/migrate.ts +2 -0
  139. package/src/db/migrations/0002_run_kind_request.sql +59 -0
  140. package/src/db/queries/collections.ts +2 -2
  141. package/src/db/queries/results.ts +88 -0
  142. package/src/db/queries/runs.ts +56 -2
  143. package/src/db/queries.ts +3 -0
  144. package/src/db/schema.ts +7 -7
  145. package/src/cli/commands/bootstrap.ts +0 -710
  146. package/src/core/anti-fp/bootstrap.ts +0 -34
  147. package/src/core/anti-fp/index.ts +0 -33
  148. package/src/core/anti-fp/registry.ts +0 -44
  149. package/src/core/anti-fp/rules/baseline-echo.ts +0 -74
  150. package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +0 -52
  151. package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +0 -38
  152. package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +0 -35
  153. package/src/core/anti-fp/rules/schemathesis/index.ts +0 -24
  154. package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +0 -53
  155. package/src/core/anti-fp/rules/subscription-gated/index.ts +0 -11
  156. package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +0 -75
  157. package/src/core/anti-fp/types.ts +0 -68
  158. package/src/core/generator/create-body.ts +0 -89
@@ -22,6 +22,12 @@ export interface CoverageOptions {
22
22
  /** ARV-28: list not-covered (and partial) endpoints inline so users
23
23
  * don't need `--json | jq` to see what's missing. */
24
24
  verbose?: boolean;
25
+ /** ARV-265: which metric block(s) to render.
26
+ * - 'test' — pass/hit coverage only (legacy single-block output)
27
+ * - 'audit' — audit-coverage only (every HTTP touch, with source
28
+ * breakdown). Useful when only `zond checks run` happened.
29
+ * - 'both' (default) — both blocks side-by-side. */
30
+ scope?: "test" | "audit" | "both";
25
31
  }
26
32
 
27
33
  const RESET = "\x1b[0m";
@@ -71,6 +77,10 @@ export interface RowBucket {
71
77
  /** Latest observed HTTP status across all cells/results on this row, or
72
78
  * `null` for unhit / network-error-only rows. */
73
79
  lastStatus: number | null;
80
+ /** ARV-379: spec-declared `deprecated` (or text-flagged) for this
81
+ * endpoint. Lets a consumer split "real, closeable gap" from
82
+ * "structurally out of scope by design" without re-reading spec.json. */
83
+ deprecated: boolean;
74
84
  }
75
85
 
76
86
  export interface BucketBreakdown {
@@ -95,6 +105,7 @@ export function bucketRows(matrix: CoverageMatrix): BucketBreakdown {
95
105
  method: row.method,
96
106
  path: row.path,
97
107
  lastStatus,
108
+ deprecated: !!row.deprecated,
98
109
  };
99
110
  if (has2xxPass) covered2xx.push(bucket);
100
111
  else if (allResults.length > 0) coveredButNon2xx.push(bucket);
@@ -103,6 +114,78 @@ export function bucketRows(matrix: CoverageMatrix): BucketBreakdown {
103
114
  return { covered2xx, coveredButNon2xx, unhit };
104
115
  }
105
116
 
117
+ /**
118
+ * ARV-265: roll up the audit-coverage matrix per producer (`run_kind`).
119
+ * `reached` counts how many spec endpoints have at least one result row
120
+ * across any kind; `totalEvents` is the raw result count (one per HTTP
121
+ * touch). `bySource` keys are the run_kind values that actually appear
122
+ * in `cov.runs` — kinds with zero contribution are omitted from the
123
+ * map so the text reporter doesn't print empty rows.
124
+ *
125
+ * Endpoint attribution uses the same matrix engine output the loader
126
+ * already built (each `MatrixRow` cell carries the results that mapped
127
+ * to that endpoint), but the engine doesn't track which run produced
128
+ * each result. We re-walk the contributing runs and re-load their
129
+ * results to bucket by kind. For the 1184-endpoint github-spec scan
130
+ * this is O(N) over ~1500 result rows — negligible vs the run cost.
131
+ */
132
+ function computeAuditBreakdown(cov: import("../../core/coverage/loader.ts").CoverageLoadResult): {
133
+ reached: number;
134
+ total: number;
135
+ percentage: number;
136
+ totalEvents: number;
137
+ bySource: Record<string, { endpoints: number; events: number }>;
138
+ } {
139
+ const total = cov.matrix.rows.length;
140
+ // reached: endpoints with any result (cells with at least one entry).
141
+ let reached = 0;
142
+ for (const row of cov.matrix.rows) {
143
+ const cells = Object.values(row.cells);
144
+ if (cells.some((c) => c.results.length > 0)) reached += 1;
145
+ }
146
+ const totalEvents = cov.matrix.rows.reduce(
147
+ (sum, row) => sum + Object.values(row.cells).reduce((s, c) => s + c.results.length, 0),
148
+ 0,
149
+ );
150
+ // Re-load per-run results to attribute counts to the producing kind.
151
+ const bySource: Record<string, { endpoints: Set<string>; events: number }> = {};
152
+ for (const run of cov.runs) {
153
+ const kind = run.run_kind;
154
+ const bucket = bySource[kind] ?? { endpoints: new Set<string>(), events: 0 };
155
+ const rows = getResultsByRunId(run.id);
156
+ for (const r of rows) {
157
+ // Skip pre-dispatch skips (max-requests cap, schema-gated) — they have
158
+ // no response and don't represent an actual HTTP touch.
159
+ if (r.response_status == null) continue;
160
+ bucket.events += 1;
161
+ if (typeof r.request_method === "string" && typeof r.request_url === "string") {
162
+ // The result table stores absolute URLs; the matrix engine uses
163
+ // the spec path-template regex to bucket them. For the by-source
164
+ // count we don't need that level of precision — pathname-only is
165
+ // close enough for "did this kind touch endpoint X" telemetry.
166
+ try {
167
+ const u = new URL(r.request_url);
168
+ bucket.endpoints.add(`${r.request_method.toUpperCase()} ${u.pathname}`);
169
+ } catch {
170
+ bucket.endpoints.add(`${r.request_method.toUpperCase()} ${r.request_url}`);
171
+ }
172
+ }
173
+ }
174
+ bySource[kind] = bucket;
175
+ }
176
+ const flatBySource: Record<string, { endpoints: number; events: number }> = {};
177
+ for (const [k, v] of Object.entries(bySource)) {
178
+ flatBySource[k] = { endpoints: v.endpoints.size, events: v.events };
179
+ }
180
+ return {
181
+ reached,
182
+ total,
183
+ percentage: total === 0 ? 0 : Math.round((reached / total) * 100),
184
+ totalEvents,
185
+ bySource: flatBySource,
186
+ };
187
+ }
188
+
106
189
  function lastObservedStatus(results: { responseStatus: number | null }[]): number | null {
107
190
  for (let i = results.length - 1; i >= 0; i--) {
108
191
  const s = results[i]?.responseStatus;
@@ -133,24 +216,54 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
133
216
  */
134
217
  async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
135
218
  getDb();
136
- const cov = await loadCoverage({
219
+ const scope = options.scope ?? "both";
220
+ const wantTest = scope === "test" || scope === "both";
221
+ const wantAudit = scope === "audit" || scope === "both";
222
+
223
+ const loaderArgs = {
137
224
  apiName: options.apiName!,
138
225
  ...(options.sessionId ? { sessionId: options.sessionId } : {}),
139
226
  ...(options.sinceIso ? { sinceIso: options.sinceIso } : {}),
140
227
  ...(options.tag ? { tag: options.tag } : {}),
141
228
  ...(options.runIds && options.runIds.length > 0 ? { runIds: options.runIds } : {}),
142
229
  ...(options.runId != null ? { runId: options.runId } : {}),
143
- });
230
+ } as const;
231
+
232
+ // ARV-265: load the matrix twice when scope=both — once for the
233
+ // pass/hit "did our suites land 2xx" metric (`scope: 'test'`), and once
234
+ // for the "did zond touch the endpoint at all" audit metric
235
+ // (`scope: 'audit'`). Loader-level scoping keeps the matrix shape
236
+ // identical so both branches reuse classifyRows / bucketRows.
237
+ const covTest = wantTest ? await loadCoverage({ ...loaderArgs, scope: "test" }) : null;
238
+ const covAudit = wantAudit ? await loadCoverage({ ...loaderArgs, scope: "audit" }) : null;
239
+ // Pick whichever block is present as the canonical "cov" — we still
240
+ // need a matrix to enumerate endpoints + spec warnings.
241
+ const cov = covTest ?? covAudit!;
144
242
  const total = cov.matrix.rows.length;
145
243
  if (total === 0) {
146
244
  printError("No endpoints found in the OpenAPI spec");
147
245
  return 1;
148
246
  }
149
247
 
150
- const { coveredRows, partialRows, uncoveredRows } = classifyRows(cov.matrix);
248
+ const { coveredRows, partialRows, uncoveredRows } = classifyRows((covTest ?? covAudit!).matrix);
151
249
  const coveredCount = coveredRows.length;
152
250
  const percentage = Math.round((coveredCount / total) * 100);
153
251
 
252
+ // ARV-303: envelope/exit-code contract. When the selector resolved to zero
253
+ // runs (closed session, --session-id with no runs, --union tag with no
254
+ // matches), there is no coverage data — that is the only command-level
255
+ // failure of the matrix path. Everything else (uncovered endpoints remain)
256
+ // is a data point, not a failure, so it must not gate the exit code.
257
+ const noRuns = (covTest ?? covAudit!).runs.length === 0;
258
+
259
+ // ARV-265: per-source breakdown for audit-coverage. Walks each
260
+ // contributing run, groups its results by (METHOD, path-template), and
261
+ // tallies distinct endpoints reached + raw event counts. The match is
262
+ // best-effort (uses the same endpoint regex the matrix engine does) so
263
+ // a run that hit URLs not declared in the spec is still counted by
264
+ // event-count even when no endpoint maps.
265
+ const auditBreakdown = covAudit ? computeAuditBreakdown(covAudit) : null;
266
+
154
267
  // TASK-270: two metrics, two intents:
155
268
  // pass_coverage = endpoints with at least one passing 2xx (strict — what
156
269
  // single-run output has always meant)
@@ -181,48 +294,64 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
181
294
 
182
295
  if (!options.json) {
183
296
  const color = useColor();
297
+ const runsForLabel = covTest ?? covAudit!;
184
298
  let runLabel: string;
185
- if (cov.runs.length === 0) {
186
- runLabel = cov.unionMode
187
- ? ` — no runs match --union ${cov.unionMode}`
299
+ if (runsForLabel.runs.length === 0) {
300
+ runLabel = runsForLabel.unionMode
301
+ ? ` — no runs match --union ${runsForLabel.unionMode}`
188
302
  : " — no runs yet";
189
303
  }
190
- else if (cov.runs.length === 1) runLabel = ` — Run #${cov.runs[0]!.id}`;
304
+ else if (runsForLabel.runs.length === 1) runLabel = ` — Run #${runsForLabel.runs[0]!.id}`;
191
305
  else {
192
- const modeLabel = cov.unionMode ? ` ${cov.unionMode}` : "";
193
- runLabel = ` — union${modeLabel} of ${cov.runs.length} runs (#${cov.runs.map(r => r.id).join(", #")})`;
306
+ const modeLabel = runsForLabel.unionMode ? ` ${runsForLabel.unionMode}` : "";
307
+ runLabel = ` — union${modeLabel} of ${runsForLabel.runs.length} runs (#${runsForLabel.runs.map(r => r.id).join(", #")})`;
194
308
  }
195
- // TASK-270: show both metrics on separate lines so CI/triage scripts
196
- // and humans can pick the one they care about.
197
- console.log(`Pass-coverage (passing 2xx): ${passCount}/${total} endpoints (${passPct}%)${runLabel}`);
198
- console.log(`Hit-coverage (any response): ${hitCount}/${total} endpoints (${hitPct}%)`);
199
- // ARV-19: explicit gap-disclosure so users running `zond checks run`
200
- // alongside don't assume probes contribute. Only `zond run` results
201
- // land in the run table that coverage reads from.
202
- console.log(` ${color ? DIM : ""}(source: \`zond run\` results only — \`zond checks run\` probes are not counted)${color ? RESET : ""}`);
203
- console.log("");
309
+ if (wantTest) {
310
+ // TASK-270: show both metrics on separate lines so CI/triage scripts
311
+ // and humans can pick the one they care about.
312
+ console.log(`test-coverage`);
313
+ console.log(` Pass-coverage (passing 2xx): ${passCount}/${total} endpoints (${passPct}%)${runLabel}`);
314
+ console.log(` Hit-coverage (any response): ${hitCount}/${total} endpoints (${hitPct}%)`);
315
+ // ARV-265: clarified source line. Previously: "only zond run results probes not counted".
316
+ // After ARV-265, audit-coverage IS the answer for "did checks run touch X" point users at it.
317
+ console.log(` ${color ? DIM : ""}(source: \`zond run\` results — for \`checks run\` / \`probe\` touches see audit-coverage below)${color ? RESET : ""}`);
318
+ console.log("");
204
319
 
205
- if (passing > 0) {
206
- console.log(` ${color ? GREEN : ""}✅ ${passing} covered (passing 2xx)${color ? RESET : ""}`);
207
- }
208
- if (apiError > 0) {
209
- console.log(` ${color ? YELLOW : ""}⚠️ ${apiError} returning 5xx (possibly broken API)${color ? RESET : ""}`);
210
- }
211
- if (testFailed > 0) {
212
- console.log(` ${color ? RED : ""}❌ ${testFailed} hit endpoint but assertions failed${color ? RESET : ""}`);
213
- }
214
- if (partialRows.length > 0 && testFailed === 0) {
215
- console.log(` ${color ? YELLOW : ""}◐ ${partialRows.length} partial (only non-2xx responses)${color ? RESET : ""}`);
320
+ if (passing > 0) {
321
+ console.log(` ${color ? GREEN : ""}✅ ${passing} covered (passing 2xx)${color ? RESET : ""}`);
322
+ }
323
+ if (apiError > 0) {
324
+ console.log(` ${color ? YELLOW : ""}⚠️ ${apiError} returning 5xx (possibly broken API)${color ? RESET : ""}`);
325
+ }
326
+ if (testFailed > 0) {
327
+ console.log(` ${color ? RED : ""}❌ ${testFailed} hit endpoint but assertions failed${color ? RESET : ""}`);
328
+ }
329
+ if (partialRows.length > 0 && testFailed === 0) {
330
+ console.log(` ${color ? YELLOW : ""}◐ ${partialRows.length} partial (only non-2xx responses)${color ? RESET : ""}`);
331
+ }
332
+ if (uncoveredRows.length > 0) {
333
+ console.log(` ${color ? DIM : ""}⬜ ${uncoveredRows.length} not covered${color ? RESET : ""}`);
334
+ // ARV-75 (feedback round-03 / F16): when some of the not-covered rows
335
+ // are deprecated, surface the count so a user reading "5% uncovered"
336
+ // can attribute the gap to deprecated endpoints (which generate skips
337
+ // by default) instead of suite regression.
338
+ const deprecatedUnhit = uncoveredRows.filter((r) => r.deprecated).length;
339
+ if (deprecatedUnhit > 0) {
340
+ console.log(` ${color ? DIM : ""}↳ ${deprecatedUnhit} of those are deprecated (skipped by \`zond generate\` unless --include-deprecated)${color ? RESET : ""}`);
341
+ }
342
+ }
216
343
  }
217
- if (uncoveredRows.length > 0) {
218
- console.log(` ${color ? DIM : ""}⬜ ${uncoveredRows.length} not covered${color ? RESET : ""}`);
219
- // ARV-75 (feedback round-03 / F16): when some of the not-covered rows
220
- // are deprecated, surface the count so a user reading "5% uncovered"
221
- // can attribute the gap to deprecated endpoints (which generate skips
222
- // by default) instead of suite regression.
223
- const deprecatedUnhit = uncoveredRows.filter((r) => r.deprecated).length;
224
- if (deprecatedUnhit > 0) {
225
- console.log(` ${color ? DIM : ""}↳ ${deprecatedUnhit} of those are deprecated (skipped by \`zond generate\` unless --include-deprecated)${color ? RESET : ""}`);
344
+
345
+ if (wantAudit && auditBreakdown) {
346
+ if (wantTest) console.log("");
347
+ console.log(`audit-coverage: ${auditBreakdown.reached}/${auditBreakdown.total} endpoints (${auditBreakdown.percentage}%, ${auditBreakdown.totalEvents} HTTP touches)`);
348
+ console.log(` ${color ? DIM : ""}(any zond producer: run, checks, probe, request, fixture-cascade — ARV-265)${color ? RESET : ""}`);
349
+ const sources = Object.entries(auditBreakdown.bySource).filter(([, v]) => v.events > 0);
350
+ if (sources.length > 0) {
351
+ console.log(` by source:`);
352
+ for (const [kind, v] of sources) {
353
+ console.log(` ${kind.padEnd(8)} ${String(v.endpoints).padStart(4)} endpoints, ${String(v.events).padStart(5)} events`);
354
+ }
226
355
  }
227
356
  }
228
357
 
@@ -264,8 +393,14 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
264
393
  // fields (covered/uncovered/partial, coveredEndpoints/partialEndpoints/
265
394
  // uncoveredEndpoints) are kept as deprecated aliases pending full envelope-
266
395
  // policy unification (TASK-184).
267
- const buckets = bucketRows(cov.matrix);
268
- printJson(jsonOk("coverage", {
396
+ const buckets = bucketRows((covTest ?? covAudit!).matrix);
397
+ // ARV-265: dual-metric JSON envelope. `test_coverage` mirrors the
398
+ // pre-ARV-265 single-block shape (kept as the canonical
399
+ // pass/hit pair); `audit_coverage` is additive and only present
400
+ // when scope includes audit. Legacy top-level fields stay populated
401
+ // from the test block when present (back-compat with TASK-280
402
+ // consumers) so existing CI scripts don't break.
403
+ const json: Record<string, unknown> = {
269
404
  // Legacy aliases — DO NOT add new consumers; use `totals.*` and
270
405
  // `*Endpoints` arrays below instead.
271
406
  covered: coveredCount,
@@ -301,13 +436,52 @@ async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
301
436
  // by zond generate" without re-deriving from the spec.
302
437
  deprecated_unhit: uncoveredRows.filter((r) => r.deprecated).length,
303
438
  deprecated_total: cov.matrix.rows.filter((r) => r.deprecated).length,
304
- }));
439
+ // ARV-379: the endpoint keys (method+path) marked deprecated in the
440
+ // spec. Intersect with `uncoveredEndpoints` to split "real, closeable
441
+ // gap" from "deprecated, skip by design" without re-reading spec.json.
442
+ // (RowBucket entries in the *Endpoints arrays also carry a per-entry
443
+ // `deprecated` boolean for the object-shaped consumers.)
444
+ deprecatedEndpoints: cov.matrix.rows.filter((r) => r.deprecated).map((r) => r.endpoint),
445
+ // ARV-265: dual-metric envelope.
446
+ test_coverage: wantTest ? {
447
+ pass: { covered: passCount, total, ratio: total === 0 ? 0 : Number((passCount / total).toFixed(4)) },
448
+ hit: { covered: hitCount, total, ratio: total === 0 ? 0 : Number((hitCount / total).toFixed(4)) },
449
+ runIds: covTest?.runs.map((r) => r.id) ?? [],
450
+ } : null,
451
+ };
452
+ if (auditBreakdown) {
453
+ json.audit_coverage = {
454
+ reached: auditBreakdown.reached,
455
+ total: auditBreakdown.total,
456
+ ratio: auditBreakdown.total === 0 ? 0 : Number((auditBreakdown.reached / auditBreakdown.total).toFixed(4)),
457
+ events: auditBreakdown.totalEvents,
458
+ by_source: auditBreakdown.bySource,
459
+ runIds: covAudit?.runs.map((r) => r.id) ?? [],
460
+ };
461
+ }
462
+ // ARV-303: surface the zero-runs case as ok:false so the non-zero exit
463
+ // lines up with the envelope shape, instead of an ok:true envelope
464
+ // alongside exit 1.
465
+ if (noRuns) {
466
+ const sel = cov.unionMode ?? "selection";
467
+ printJson(jsonError("coverage", [
468
+ `No runs match ${sel} — coverage cannot be computed against zero runs`,
469
+ ]));
470
+ } else {
471
+ printJson(jsonOk("coverage", json));
472
+ }
305
473
  }
306
474
 
475
+ // ARV-303: exit-code contract. ok:false (no runs) ⇒ exit 1. Otherwise the
476
+ // coverage was computed successfully (ok:true) ⇒ exit 0 by default — an
477
+ // orchestrator must be able to tell "coverage ran" from "command failed".
478
+ // "Uncovered endpoints remain" no longer gates the exit on its own; use
479
+ // --fail-on-coverage to opt into a threshold gate.
480
+ if (noRuns) return 1;
307
481
  if (options.failOnCoverage !== undefined) {
308
482
  return percentage < options.failOnCoverage ? 1 : 0;
309
483
  }
310
- return uncoveredRows.length > 0 ? 1 : 0;
484
+ return 0;
311
485
  }
312
486
 
313
487
  /**
@@ -346,6 +520,13 @@ async function runSpecOnlyCoverage(options: CoverageOptions): Promise<number> {
346
520
  }
347
521
  }
348
522
  } else {
523
+ // TASK-250 + ARV-303: the spec-only path emits a shape-stable
524
+ // ok:true envelope even when there's no registered API. The
525
+ // legacy contract here is "we parsed the spec, here is 0% with a
526
+ // null runId" — agents and CI scripts depend on the key shape.
527
+ // ARV-303 only tightens the matrix-coverage path (no runs match
528
+ // --union / --session-id), which DID emit ok:true alongside exit
529
+ // 1; the spec-only path was intentionally informational.
349
530
  const unhit = allEndpoints.map((ep) => ({
350
531
  endpoint: `${ep.method.toUpperCase()} ${ep.path}`,
351
532
  method: ep.method.toUpperCase(),
@@ -366,8 +547,6 @@ async function runSpecOnlyCoverage(options: CoverageOptions): Promise<number> {
366
547
  covered2xxEndpoints: [],
367
548
  coveredButNon2xxEndpoints: [],
368
549
  unhitEndpoints: unhit,
369
- // TASK-270: the spec-only path has no run results, so both metrics
370
- // are zero. Surface them anyway for shape-stable JSON.
371
550
  pass_coverage: { covered: 0, total, ratio: 0 },
372
551
  hit_coverage: { covered: 0, total, ratio: 0 },
373
552
  }));
@@ -385,7 +564,7 @@ import { globalJson, resolveApiCollection } from "../resolve.ts";
385
564
  import { parseInteger, parsePercentage } from "../argv.ts";
386
565
  import { getApi } from "../util/api-context.ts";
387
566
  import { readCurrentSession } from "../../core/context/session.ts";
388
- import { listRunsBySession, getLatestRunByCollection, getRunById, findCollectionByNameOrId } from "../../db/queries.ts";
567
+ import { listRunsBySession, getLatestRunByCollection, getRunById, findCollectionByNameOrId, listSessions, getResultsByRunId } from "../../db/queries.ts";
389
568
 
390
569
  /**
391
570
  * ARV-55: probe-run classification moved from path-regex heuristic into the
@@ -476,14 +655,16 @@ export function registerCoverage(program: Command): void {
476
655
  .command("coverage")
477
656
  .description(
478
657
  "Analyze API test coverage from stored run results. zond reports two " +
479
- "metrics side-by-side (TASK-270):\n" +
480
- " pass-coverage — endpoint had at least one passing 2xx response (strict; what CI usually wants)\n" +
481
- " hit-coverage — endpoint received any response at all, including 5xx and assertion failures (loose; for breadth audits)\n" +
658
+ "metric blocks side-by-side (ARV-265):\n" +
659
+ " test-coverage — pass/hit from `zond run` (regular + probe runs only)\n" +
660
+ " · pass-coverage — endpoint had at least one passing 2xx (strict; CI gate via --fail-on-coverage)\n" +
661
+ " · hit-coverage — endpoint received any response, incl. 5xx / assertion-fail (loose; breadth)\n" +
662
+ " audit-coverage — any HTTP touch from any producer: `run`, `checks run`,\n" +
663
+ " `probe`, `request`, `prepare-fixtures --cascade`. With by-source breakdown.\n" +
482
664
  "\n" +
483
- "Source: only `zond run` results are aggregated (ARV-19). `zond checks " +
484
- "run` probes hit the API but are not stored as run results, so they " +
485
- "don't move either metric generate suites with `zond generate` and " +
486
- "execute them via `zond run` to grow coverage.\n" +
665
+ "Use --scope to print just one block: `--scope test` for legacy single-metric\n" +
666
+ "output, `--scope audit` to answer 'did the scan reach the API at all'.\n" +
667
+ "Producers opt out via ZOND_CHECKS_PERSIST=0 (default: persist on).\n" +
487
668
  "\n" +
488
669
  "Defaults to the latest stored run for the resolved API; pass " +
489
670
  "--run-id to pin a specific run, or --union <selector> to combine " +
@@ -510,10 +691,12 @@ export function registerCoverage(program: Command): void {
510
691
  " zond run apis/<api>/probes\n" +
511
692
  " zond coverage --api <api> --union session\n" +
512
693
  "\n" +
513
- "Exit codes: 0 = every endpoint covered (or pass-coverage ≥ " +
514
- "--fail-on-coverage when set); 1 = uncovered endpoints remain (or " +
515
- "pass-coverage < --fail-on-coverage); 2 = bad input or read error. " +
516
- "--fail-on-coverage gates pass-coverage, not hit-coverage.",
694
+ "Exit codes (ARV-303): 0 = coverage computed successfully (envelope " +
695
+ "ok:true uncovered endpoints remaining is a data point, not a " +
696
+ "failure); 1 = coverage could not be computed (selector resolved to " +
697
+ "zero runs) or pass-coverage < --fail-on-coverage; 2 = bad input or " +
698
+ "read error. Uncovered endpoints only gate the exit when you opt in " +
699
+ "with --fail-on-coverage; that flag gates pass-coverage, not hit-coverage.",
517
700
  )
518
701
  .option("--api <name>", "Use API collection (auto-resolves spec; reads stored runs)")
519
702
  .option("--spec <path>", "Spec-only fallback when no API is registered (no run results)")
@@ -526,6 +709,12 @@ export function registerCoverage(program: Command): void {
526
709
  )
527
710
  .option("--db <path>", "Path to SQLite database file")
528
711
  .option("--verbose", "List not-covered (and partial) endpoints inline — same data as `--json` but human-readable")
712
+ .addOption(
713
+ new Option(
714
+ "--scope <scope>",
715
+ "ARV-265: which coverage block(s) to print. `test` = pass-coverage/hit-coverage only (legacy single block from `zond run` results). `audit` = audit-coverage only (every HTTP touch from `run` / `checks run` / `probe` / `request` / fixture-cascade, with by-source breakdown). `both` (default) = both side-by-side.",
716
+ ).choices(["test", "audit", "both"]).default("both"),
717
+ )
529
718
  // ARV-35: `--format json` matches the kubectl/gh/aws-cli convention many
530
719
  // users reach for first; until ARV-54 lands a workspace-wide alias layer
531
720
  // we accept it locally and forward to `--json`. Other values are rejected
@@ -566,7 +755,20 @@ export function registerCoverage(program: Command): void {
566
755
  if (parsed.kind === "session") {
567
756
  const current = readCurrentSession();
568
757
  if (!current) {
569
- printError("--union session requires an active session (run 'zond session start' first), or pass --session-id <id>.");
758
+ // ARV-234: when there's no active session, peek at the most
759
+ // recent one — agents typically hit this right after
760
+ // `session end`. Surface its id in the error so the recovery
761
+ // path is one copy-paste instead of `db sessions`-spelunking.
762
+ let hint = "";
763
+ try {
764
+ const recent = listSessions(1, 0);
765
+ if (recent.length > 0) {
766
+ const r = recent[0]!;
767
+ const endedAt = r.finished_at ? ` (ended ${r.finished_at})` : "";
768
+ hint = ` Most recent session: --session-id ${r.session_id}${endedAt}.`;
769
+ }
770
+ } catch { /* best effort */ }
771
+ printError(`--union session requires an active session (run 'zond session start' first), or pass --session-id <id>.${hint}`);
570
772
  process.exitCode = 2;
571
773
  return;
572
774
  }
@@ -604,12 +806,14 @@ export function registerCoverage(program: Command): void {
604
806
  // (the percentage already looked like a regression). Explicit
605
807
  // selectors win, --json keeps the envelope untouched.
606
808
  const noSelector = !opts.runId && !sessionId && !runIds && !sinceIso && !tag;
809
+ let promotedToSession = false;
607
810
  if (apiName && noSelector) {
608
811
  const current = readCurrentSession();
609
812
  if (current) {
610
813
  const sessRuns = listRunsBySession(current.id);
611
814
  if (sessRuns.length > 1) {
612
815
  sessionId = current.id;
816
+ promotedToSession = true;
613
817
  if (!globalJson(cmd)) {
614
818
  process.stderr.write(
615
819
  `zond: active session has ${sessRuns.length} runs — defaulting to --union session (pass --run-id <N> for a single run).\n`,
@@ -634,10 +838,23 @@ export function registerCoverage(program: Command): void {
634
838
  `For combined coverage, wrap your runs in 'zond session start/end' and pass '--union session' here.`;
635
839
  process.stderr.write(`zond: ${hint}\n`);
636
840
  }
841
+ // ARV-81: parity with the session-promotion footer above —
842
+ // when we *don't* promote to --union session (no session, or
843
+ // session has 1 run), tell the user which run they're seeing
844
+ // so the single-run snapshot can't be mistaken for a regression.
845
+ if (!promotedToSession && !globalJson(cmd)) {
846
+ const regular = getLatestRunByCollection(collection.id, { runKind: "regular" });
847
+ if (regular) {
848
+ process.stderr.write(
849
+ `zond: using latest run #${regular.id}. For union, pass '--union since:<dur>' or '--union runs:<a,b,...>'.\n`,
850
+ );
851
+ }
852
+ }
637
853
  }
638
854
  } catch { /* DB inspection is best-effort, don't break coverage */ }
639
855
  }
640
856
 
857
+ const scope = (opts.scope === "test" || opts.scope === "audit") ? opts.scope : "both";
641
858
  process.exitCode = await coverageCommand({
642
859
  ...(apiName ? { apiName } : {}),
643
860
  ...(spec ? { spec } : {}),
@@ -649,6 +866,7 @@ export function registerCoverage(program: Command): void {
649
866
  ...(tag ? { tag } : {}),
650
867
  json: globalJson(cmd),
651
868
  verbose: opts.verbose === true,
869
+ scope,
652
870
  });
653
871
  });
654
872
  }