@kirrosh/zond 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +758 -3
- package/README.md +78 -15
- package/package.json +17 -10
- package/src/cli/argv.ts +122 -0
- package/src/cli/commands/add-api.ts +134 -0
- package/src/cli/commands/api/annotate/idempotency.ts +59 -0
- package/src/cli/commands/api/annotate/index.ts +525 -0
- package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
- package/src/cli/commands/api/annotate/overlay.ts +206 -0
- package/src/cli/commands/api/annotate/pagination.ts +60 -0
- package/src/cli/commands/api/annotate/prompts.ts +183 -0
- package/src/cli/commands/api/annotate/readback.ts +58 -0
- package/src/cli/commands/api/annotate/resources.ts +91 -0
- package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
- package/src/cli/commands/audit.ts +480 -0
- package/src/cli/commands/bootstrap.ts +710 -0
- package/src/cli/commands/catalog.ts +35 -0
- package/src/cli/commands/check.ts +348 -0
- package/src/cli/commands/checks.ts +756 -0
- package/src/cli/commands/ci-init.ts +55 -6
- package/src/cli/commands/clean.ts +212 -0
- package/src/cli/commands/cleanup.ts +262 -0
- package/src/cli/commands/completions.ts +192 -0
- package/src/cli/commands/coverage.ts +605 -132
- package/src/cli/commands/db.ts +180 -8
- package/src/cli/commands/describe.ts +37 -2
- package/src/cli/commands/discover.ts +1236 -0
- package/src/cli/commands/doctor.ts +607 -0
- package/src/cli/commands/fixtures.ts +402 -0
- package/src/cli/commands/generate.ts +420 -47
- package/src/cli/commands/init/agents-md.ts +61 -0
- package/src/cli/commands/init/bootstrap.ts +108 -0
- package/src/cli/commands/init/index.ts +244 -0
- package/src/cli/commands/init/skills.ts +98 -0
- package/src/cli/commands/init/templates/agents.md +77 -0
- package/src/cli/commands/init/templates/markdown.d.ts +4 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +397 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +210 -0
- package/src/cli/commands/init/templates/skills/zond.md +651 -0
- package/src/cli/commands/init/templates/zond-config.yml +14 -0
- package/src/cli/commands/prepare-fixtures.ts +135 -0
- package/src/cli/commands/probe/mass-assignment.ts +503 -0
- package/src/cli/commands/probe/security.ts +454 -0
- package/src/cli/commands/probe/static.ts +255 -0
- package/src/cli/commands/probe/webhooks.ts +161 -0
- package/src/cli/commands/probe.ts +459 -0
- package/src/cli/commands/reference.ts +87 -0
- package/src/cli/commands/refresh-api.ts +169 -0
- package/src/cli/commands/remove-api.ts +150 -0
- package/src/cli/commands/report-bundle.ts +318 -0
- package/src/cli/commands/report.ts +241 -0
- package/src/cli/commands/request.ts +379 -4
- package/src/cli/commands/run.ts +911 -33
- package/src/cli/commands/session.ts +244 -0
- package/src/cli/commands/use.ts +74 -0
- package/src/cli/index.ts +36 -607
- package/src/cli/json-envelope.ts +112 -3
- package/src/cli/json-schemas.ts +263 -0
- package/src/cli/program.ts +218 -0
- package/src/cli/resolve.ts +105 -0
- package/src/cli/status-filter.ts +124 -0
- package/src/cli/util/api-context.ts +85 -0
- package/src/cli/version.ts +8 -0
- package/src/core/anti-fp/bootstrap.ts +34 -0
- package/src/core/anti-fp/index.ts +33 -0
- package/src/core/anti-fp/registry.ts +44 -0
- package/src/core/anti-fp/rules/baseline-echo.ts +74 -0
- package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +52 -0
- package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +38 -0
- package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +35 -0
- package/src/core/anti-fp/rules/schemathesis/index.ts +24 -0
- package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +53 -0
- package/src/core/anti-fp/rules/subscription-gated/index.ts +11 -0
- package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +75 -0
- package/src/core/anti-fp/types.ts +68 -0
- package/src/core/checks/checks/_crud-helpers.ts +133 -0
- package/src/core/checks/checks/_negative_mutator.ts +133 -0
- package/src/core/checks/checks/_readback-helpers.ts +133 -0
- package/src/core/checks/checks/content_type_conformance.ts +39 -0
- package/src/core/checks/checks/cross_call_references.ts +134 -0
- package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
- package/src/core/checks/checks/idempotency_replay.ts +246 -0
- package/src/core/checks/checks/ignored_auth.ts +211 -0
- package/src/core/checks/checks/index.ts +65 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +273 -0
- package/src/core/checks/checks/missing_required_header.ts +40 -0
- package/src/core/checks/checks/negative_data_rejection.ts +45 -0
- package/src/core/checks/checks/not_a_server_error.ts +27 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +131 -0
- package/src/core/checks/checks/pagination_invariants.ts +238 -0
- package/src/core/checks/checks/positive_data_acceptance.ts +36 -0
- package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
- package/src/core/checks/checks/response_headers_conformance.ts +74 -0
- package/src/core/checks/checks/response_schema_conformance.ts +30 -0
- package/src/core/checks/checks/status_code_conformance.ts +61 -0
- package/src/core/checks/checks/unsupported_method.ts +63 -0
- package/src/core/checks/checks/use_after_free.ts +78 -0
- package/src/core/checks/index.ts +30 -0
- package/src/core/checks/mode.ts +79 -0
- package/src/core/checks/recommended-action.ts +64 -0
- package/src/core/checks/registry.ts +78 -0
- package/src/core/checks/runner.ts +874 -0
- package/src/core/checks/sarif.ts +230 -0
- package/src/core/checks/stateful.ts +121 -0
- package/src/core/checks/types.ts +189 -0
- package/src/core/classifier/recommended-action.ts +222 -0
- package/src/core/context/current.ts +51 -0
- package/src/core/context/session.ts +78 -0
- package/src/core/coverage/loader.ts +185 -0
- package/src/core/coverage/reasons.ts +300 -0
- package/src/core/diagnostics/db-analysis.ts +161 -12
- package/src/core/diagnostics/failure-class.ts +120 -0
- package/src/core/diagnostics/failure-hints.ts +212 -9
- package/src/core/diagnostics/spec-pointer.ts +99 -0
- package/src/core/diagnostics/suggested-fixes.ts +156 -0
- package/src/core/exporter/case-study/index.ts +270 -0
- package/src/core/exporter/curl.ts +40 -0
- package/src/core/exporter/exporter.ts +48 -0
- package/src/core/exporter/html-report/escape.ts +24 -0
- package/src/core/exporter/html-report/index.ts +479 -0
- package/src/core/exporter/html-report/script.ts +100 -0
- package/src/core/exporter/html-report/styles.ts +408 -0
- package/src/core/generator/chunker.ts +53 -15
- package/src/core/generator/coverage-phase.ts +0 -0
- package/src/core/generator/create-body.ts +89 -0
- package/src/core/generator/data-factory.ts +490 -33
- package/src/core/generator/describe.ts +1 -1
- package/src/core/generator/fixtures-builder.ts +325 -0
- package/src/core/generator/index.ts +7 -5
- package/src/core/generator/openapi-reader.ts +55 -3
- package/src/core/generator/path-param-disambig.ts +114 -0
- package/src/core/generator/resources-builder.ts +648 -0
- package/src/core/generator/schema-utils.ts +11 -3
- package/src/core/generator/serializer.ts +114 -15
- package/src/core/generator/suite-generator.ts +484 -77
- package/src/core/generator/types.ts +8 -0
- package/src/core/identity/identity-file.ts +129 -0
- package/src/core/lint/affects.ts +28 -0
- package/src/core/lint/config.ts +96 -0
- package/src/core/lint/format.ts +42 -0
- package/src/core/lint/index.ts +94 -0
- package/src/core/lint/reporter.ts +128 -0
- package/src/core/lint/rules/consistency.ts +158 -0
- package/src/core/lint/rules/heuristics.ts +97 -0
- package/src/core/lint/rules/strictness.ts +109 -0
- package/src/core/lint/types.ts +96 -0
- package/src/core/lint/walker.ts +248 -0
- package/src/core/meta/meta-store.ts +6 -73
- package/src/core/output/README.md +91 -0
- package/src/core/output/index.ts +13 -0
- package/src/core/output/run.ts +126 -0
- package/src/core/output/types.ts +129 -0
- package/src/core/parser/env-interpolation.ts +104 -0
- package/src/core/parser/filter.ts +57 -0
- package/src/core/parser/schema.ts +132 -5
- package/src/core/parser/types.ts +29 -2
- package/src/core/parser/variables.ts +0 -0
- package/src/core/parser/yaml-parser.ts +108 -13
- package/src/core/probe/bootstrap.ts +34 -0
- package/src/core/probe/dry-run-envelope.ts +57 -0
- package/src/core/probe/mass-assignment-probe-class.ts +198 -0
- package/src/core/probe/mass-assignment-probe.ts +1122 -0
- package/src/core/probe/mass-assignment-template.ts +212 -0
- package/src/core/probe/method-probe.ts +164 -0
- package/src/core/probe/method-shared.ts +69 -0
- package/src/core/probe/negative-probe.ts +691 -0
- package/src/core/probe/orphan-tracker.ts +188 -0
- package/src/core/probe/path-discovery.ts +440 -0
- package/src/core/probe/probe-harness.ts +120 -0
- package/src/core/probe/registry.ts +89 -0
- package/src/core/probe/runner.ts +136 -0
- package/src/core/probe/security-probe-class.ts +201 -0
- package/src/core/probe/security-probe.ts +1453 -0
- package/src/core/probe/shared.ts +505 -0
- package/src/core/probe/static-probe-class.ts +125 -0
- package/src/core/probe/types.ts +165 -0
- package/src/core/probe/verdict-aggregator.ts +33 -0
- package/src/core/probe/webhooks-probe.ts +284 -0
- package/src/core/reporter/console.ts +69 -4
- package/src/core/reporter/index.ts +2 -3
- package/src/core/reporter/json.ts +15 -2
- package/src/core/reporter/junit.ts +27 -12
- package/src/core/reporter/ndjson.ts +37 -0
- package/src/core/reporter/types.ts +3 -0
- package/src/core/runner/assertions.ts +62 -2
- package/src/core/runner/async-pool.ts +108 -0
- package/src/core/runner/auth-path.ts +8 -0
- package/src/core/runner/ci-context.ts +72 -0
- package/src/core/runner/executor.ts +391 -52
- package/src/core/runner/form-encode.ts +51 -0
- package/src/core/runner/http-client.ts +115 -7
- package/src/core/runner/learn-drift.ts +293 -0
- package/src/core/runner/preflight-vars.ts +149 -0
- package/src/core/runner/progress-tracker.ts +73 -0
- package/src/core/runner/rate-limiter.ts +203 -0
- package/src/core/runner/run-kind.ts +39 -0
- package/src/core/runner/schema-validator.ts +312 -0
- package/src/core/runner/send-request.ts +153 -20
- package/src/core/runner/types.ts +38 -0
- package/src/core/secrets/registry.ts +164 -0
- package/src/core/secrets/secrets-file.ts +115 -0
- package/src/core/selectors/operation-filter.ts +144 -0
- package/src/core/setup-api.ts +419 -17
- package/src/core/severity/category.ts +94 -0
- package/src/core/severity/index.ts +121 -0
- package/src/core/spec/layers.ts +154 -0
- package/src/core/util/format-eta.ts +21 -0
- package/src/core/utils.ts +5 -1
- package/src/core/workspace/config.ts +129 -0
- package/src/core/workspace/manifest.ts +283 -0
- package/src/core/workspace/output-rotation.ts +62 -0
- package/src/core/workspace/root.ts +94 -0
- package/src/core/workspace/triage-path.ts +87 -0
- package/src/db/lint-runs.ts +47 -0
- package/src/db/migrate.ts +126 -0
- package/src/db/migrations/0001_run_kind.sql +25 -0
- package/src/db/migrations/sql.d.ts +4 -0
- package/src/db/queries/collections.ts +133 -0
- package/src/db/queries/coverage.ts +9 -0
- package/src/db/queries/dashboard.ts +59 -0
- package/src/db/queries/results.ts +128 -0
- package/src/db/queries/runs.ts +235 -0
- package/src/db/queries/sessions.ts +42 -0
- package/src/db/queries/settings.ts +28 -0
- package/src/db/queries/types.ts +172 -0
- package/src/db/queries.ts +72 -802
- package/src/db/schema.ts +179 -48
- package/src/cli/commands/export.ts +0 -144
- package/src/cli/commands/guide.ts +0 -127
- package/src/cli/commands/init.ts +0 -57
- package/src/cli/commands/serve.ts +0 -81
- package/src/cli/commands/sync.ts +0 -269
- package/src/cli/commands/update.ts +0 -189
- package/src/cli/commands/validate.ts +0 -34
- package/src/core/exporter/postman.ts +0 -963
- package/src/core/generator/guide-builder.ts +0 -253
- package/src/core/meta/types.ts +0 -21
- package/src/core/parser/index.ts +0 -21
- package/src/core/runner/execute-run.ts +0 -132
- package/src/core/runner/index.ts +0 -12
- package/src/core/sync/spec-differ.ts +0 -38
- package/src/web/data/collection-state.ts +0 -362
- package/src/web/routes/api.ts +0 -314
- package/src/web/routes/dashboard.ts +0 -350
- package/src/web/routes/runs.ts +0 -64
- package/src/web/schemas.ts +0 -121
- package/src/web/server.ts +0 -134
- package/src/web/static/htmx.min.cjs +0 -1
- package/src/web/static/style.css +0 -1148
- package/src/web/views/endpoints-tab.ts +0 -174
- package/src/web/views/explorer-tab.ts +0 -402
- package/src/web/views/health-strip.ts +0 -92
- package/src/web/views/layout.ts +0 -48
- package/src/web/views/results.ts +0 -210
- package/src/web/views/runs-tab.ts +0 -126
- package/src/web/views/suites-tab.ts +0 -181
|
@@ -1,15 +1,27 @@
|
|
|
1
|
-
import { readOpenApiSpec, extractEndpoints,
|
|
1
|
+
import { readOpenApiSpec, extractEndpoints, analyzeEndpoints } from "../../core/generator/index.ts";
|
|
2
2
|
import { getDb } from "../../db/schema.ts";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { loadCoverage } from "../../core/coverage/loader.ts";
|
|
4
|
+
import type { CoverageMatrix, MatrixRow } from "../../core/coverage/reasons.ts";
|
|
5
|
+
import { printError } from "../output.ts";
|
|
5
6
|
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
6
7
|
|
|
7
8
|
export interface CoverageOptions {
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
apiName?: string;
|
|
10
|
+
spec?: string;
|
|
10
11
|
failOnCoverage?: number;
|
|
11
12
|
runId?: number;
|
|
13
|
+
/** TASK-255: union across multiple runs. */
|
|
14
|
+
runIds?: number[];
|
|
15
|
+
/** TASK-255: union across all runs in a session (filtered to the API). */
|
|
16
|
+
sessionId?: string;
|
|
17
|
+
/** TASK-274: union across all runs of the API started after this ISO ts. */
|
|
18
|
+
sinceIso?: string;
|
|
19
|
+
/** TASK-274: union across all runs of the API tagged <tag>. */
|
|
20
|
+
tag?: string;
|
|
12
21
|
json?: boolean;
|
|
22
|
+
/** ARV-28: list not-covered (and partial) endpoints inline so users
|
|
23
|
+
* don't need `--json | jq` to see what's missing. */
|
|
24
|
+
verbose?: boolean;
|
|
13
25
|
}
|
|
14
26
|
|
|
15
27
|
const RESET = "\x1b[0m";
|
|
@@ -22,160 +34,621 @@ function useColor(): boolean {
|
|
|
22
34
|
return process.stdout.isTTY ?? false;
|
|
23
35
|
}
|
|
24
36
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
37
|
+
interface CoverageBreakdown {
|
|
38
|
+
coveredRows: MatrixRow[];
|
|
39
|
+
partialRows: MatrixRow[];
|
|
40
|
+
uncoveredRows: MatrixRow[];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function classifyRows(matrix: CoverageMatrix): CoverageBreakdown {
|
|
44
|
+
const coveredRows: MatrixRow[] = [];
|
|
45
|
+
const partialRows: MatrixRow[] = [];
|
|
46
|
+
const uncoveredRows: MatrixRow[] = [];
|
|
47
|
+
for (const row of matrix.rows) {
|
|
48
|
+
const cells = Object.values(row.cells);
|
|
49
|
+
if (cells.some((c) => c.status === "covered")) coveredRows.push(row);
|
|
50
|
+
else if (cells.some((c) => c.status === "partial")) partialRows.push(row);
|
|
51
|
+
else uncoveredRows.push(row);
|
|
30
52
|
}
|
|
53
|
+
return { coveredRows, partialRows, uncoveredRows };
|
|
31
54
|
}
|
|
32
55
|
|
|
33
|
-
|
|
34
|
-
|
|
56
|
+
/**
|
|
57
|
+
* TASK-280: row-level pass/fail classification used by both the text and JSON
|
|
58
|
+
* outputs so they share a single source of truth.
|
|
59
|
+
*
|
|
60
|
+
* covered2xx — at least one stored result on this endpoint was a
|
|
61
|
+
* passing 2xx (matches `✅ N covered (passing 2xx)`)
|
|
62
|
+
* coveredButNon2xx — endpoint was hit but never produced a 2xx pass
|
|
63
|
+
* (5xx, 4xx, assertion failure — anything that landed
|
|
64
|
+
* a response or generated an `error`)
|
|
65
|
+
* unhit — no stored results at all on this endpoint
|
|
66
|
+
*/
|
|
67
|
+
export interface RowBucket {
|
|
68
|
+
endpoint: string;
|
|
69
|
+
method: string;
|
|
70
|
+
path: string;
|
|
71
|
+
/** Latest observed HTTP status across all cells/results on this row, or
|
|
72
|
+
* `null` for unhit / network-error-only rows. */
|
|
73
|
+
lastStatus: number | null;
|
|
74
|
+
}
|
|
35
75
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
76
|
+
export interface BucketBreakdown {
|
|
77
|
+
covered2xx: RowBucket[];
|
|
78
|
+
coveredButNon2xx: RowBucket[];
|
|
79
|
+
unhit: RowBucket[];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function bucketRows(matrix: CoverageMatrix): BucketBreakdown {
|
|
83
|
+
const covered2xx: RowBucket[] = [];
|
|
84
|
+
const coveredButNon2xx: RowBucket[] = [];
|
|
85
|
+
const unhit: RowBucket[] = [];
|
|
86
|
+
for (const row of matrix.rows) {
|
|
87
|
+
const cells = Object.values(row.cells);
|
|
88
|
+
const allResults = cells.flatMap(c => c.results);
|
|
89
|
+
const has2xxPass = allResults.some(
|
|
90
|
+
r => r.status === "pass" && r.responseStatus != null && r.responseStatus >= 200 && r.responseStatus < 300,
|
|
91
|
+
);
|
|
92
|
+
const lastStatus = lastObservedStatus(allResults);
|
|
93
|
+
const bucket: RowBucket = {
|
|
94
|
+
endpoint: row.endpoint,
|
|
95
|
+
method: row.method,
|
|
96
|
+
path: row.path,
|
|
97
|
+
lastStatus,
|
|
98
|
+
};
|
|
99
|
+
if (has2xxPass) covered2xx.push(bucket);
|
|
100
|
+
else if (allResults.length > 0) coveredButNon2xx.push(bucket);
|
|
101
|
+
else unhit.push(bucket);
|
|
102
|
+
}
|
|
103
|
+
return { covered2xx, coveredButNon2xx, unhit };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function lastObservedStatus(results: { responseStatus: number | null }[]): number | null {
|
|
107
|
+
for (let i = results.length - 1; i >= 0; i--) {
|
|
108
|
+
const s = results[i]?.responseStatus;
|
|
109
|
+
if (typeof s === "number") return s;
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
39
113
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
114
|
+
export async function coverageCommand(options: CoverageOptions): Promise<number> {
|
|
115
|
+
try {
|
|
116
|
+
if (options.apiName) {
|
|
117
|
+
return await runMatrixCoverage(options);
|
|
43
118
|
}
|
|
119
|
+
return await runSpecOnlyCoverage(options);
|
|
120
|
+
} catch (err) {
|
|
121
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
122
|
+
if (options.json) printJson(jsonError("coverage", [message]));
|
|
123
|
+
else printError(message);
|
|
124
|
+
return 2;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Matrix-based path: an endpoint is "covered" iff some result on it landed
|
|
130
|
+
* a 2xx pass. Driven by the latest stored run (or `--runId`). This is the
|
|
131
|
+
* answer to "did we actually exercise the endpoint", which is what
|
|
132
|
+
* `--fail-on-coverage` gates in CI.
|
|
133
|
+
*/
|
|
134
|
+
async function runMatrixCoverage(options: CoverageOptions): Promise<number> {
|
|
135
|
+
getDb();
|
|
136
|
+
const cov = await loadCoverage({
|
|
137
|
+
apiName: options.apiName!,
|
|
138
|
+
...(options.sessionId ? { sessionId: options.sessionId } : {}),
|
|
139
|
+
...(options.sinceIso ? { sinceIso: options.sinceIso } : {}),
|
|
140
|
+
...(options.tag ? { tag: options.tag } : {}),
|
|
141
|
+
...(options.runIds && options.runIds.length > 0 ? { runIds: options.runIds } : {}),
|
|
142
|
+
...(options.runId != null ? { runId: options.runId } : {}),
|
|
143
|
+
});
|
|
144
|
+
const total = cov.matrix.rows.length;
|
|
145
|
+
if (total === 0) {
|
|
146
|
+
printError("No endpoints found in the OpenAPI spec");
|
|
147
|
+
return 1;
|
|
148
|
+
}
|
|
44
149
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
150
|
+
const { coveredRows, partialRows, uncoveredRows } = classifyRows(cov.matrix);
|
|
151
|
+
const coveredCount = coveredRows.length;
|
|
152
|
+
const percentage = Math.round((coveredCount / total) * 100);
|
|
153
|
+
|
|
154
|
+
// TASK-270: two metrics, two intents:
|
|
155
|
+
// pass_coverage = endpoints with at least one passing 2xx (strict — what
|
|
156
|
+
// single-run output has always meant)
|
|
157
|
+
// hit_coverage = endpoints we touched at all (loose — what `--union`
|
|
158
|
+
// used to silently mean)
|
|
159
|
+
// Show both so the reader doesn't have to re-derive the difference.
|
|
160
|
+
const passCount = coveredRows.length;
|
|
161
|
+
const hitCount = coveredRows.length + partialRows.length;
|
|
162
|
+
const passPct = Math.round((passCount / total) * 100);
|
|
163
|
+
const hitPct = Math.round((hitCount / total) * 100);
|
|
164
|
+
|
|
165
|
+
let passing = 0;
|
|
166
|
+
let apiError = 0;
|
|
167
|
+
let testFailed = 0;
|
|
168
|
+
for (const row of [...coveredRows, ...partialRows]) {
|
|
169
|
+
const cells = Object.values(row.cells);
|
|
170
|
+
const has5xx = cells.some((c) =>
|
|
171
|
+
c.results.some((r) => r.responseStatus != null && r.responseStatus >= 500),
|
|
172
|
+
);
|
|
173
|
+
const has2xxPass = cells.some((c) =>
|
|
174
|
+
c.results.some((r) => r.status === "pass" && r.responseStatus != null && r.responseStatus >= 200 && r.responseStatus < 300),
|
|
175
|
+
);
|
|
176
|
+
const hasFail = cells.some((c) => c.results.some((r) => r.status !== "pass"));
|
|
177
|
+
if (has5xx) apiError++;
|
|
178
|
+
else if (has2xxPass) passing++;
|
|
179
|
+
else if (hasFail) testFailed++;
|
|
180
|
+
}
|
|
49
181
|
|
|
182
|
+
if (!options.json) {
|
|
50
183
|
const color = useColor();
|
|
184
|
+
let runLabel: string;
|
|
185
|
+
if (cov.runs.length === 0) {
|
|
186
|
+
runLabel = cov.unionMode
|
|
187
|
+
? ` — no runs match --union ${cov.unionMode}`
|
|
188
|
+
: " — no runs yet";
|
|
189
|
+
}
|
|
190
|
+
else if (cov.runs.length === 1) runLabel = ` — Run #${cov.runs[0]!.id}`;
|
|
191
|
+
else {
|
|
192
|
+
const modeLabel = cov.unionMode ? ` ${cov.unionMode}` : "";
|
|
193
|
+
runLabel = ` — union${modeLabel} of ${cov.runs.length} runs (#${cov.runs.map(r => r.id).join(", #")})`;
|
|
194
|
+
}
|
|
195
|
+
// TASK-270: show both metrics on separate lines so CI/triage scripts
|
|
196
|
+
// and humans can pick the one they care about.
|
|
197
|
+
console.log(`Pass-coverage (passing 2xx): ${passCount}/${total} endpoints (${passPct}%)${runLabel}`);
|
|
198
|
+
console.log(`Hit-coverage (any response): ${hitCount}/${total} endpoints (${hitPct}%)`);
|
|
199
|
+
// ARV-19: explicit gap-disclosure so users running `zond checks run`
|
|
200
|
+
// alongside don't assume probes contribute. Only `zond run` results
|
|
201
|
+
// land in the run table that coverage reads from.
|
|
202
|
+
console.log(` ${color ? DIM : ""}(source: \`zond run\` results only — \`zond checks run\` probes are not counted)${color ? RESET : ""}`);
|
|
203
|
+
console.log("");
|
|
51
204
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
205
|
+
if (passing > 0) {
|
|
206
|
+
console.log(` ${color ? GREEN : ""}✅ ${passing} covered (passing 2xx)${color ? RESET : ""}`);
|
|
207
|
+
}
|
|
208
|
+
if (apiError > 0) {
|
|
209
|
+
console.log(` ${color ? YELLOW : ""}⚠️ ${apiError} returning 5xx (possibly broken API)${color ? RESET : ""}`);
|
|
210
|
+
}
|
|
211
|
+
if (testFailed > 0) {
|
|
212
|
+
console.log(` ${color ? RED : ""}❌ ${testFailed} hit endpoint but assertions failed${color ? RESET : ""}`);
|
|
213
|
+
}
|
|
214
|
+
if (partialRows.length > 0 && testFailed === 0) {
|
|
215
|
+
console.log(` ${color ? YELLOW : ""}◐ ${partialRows.length} partial (only non-2xx responses)${color ? RESET : ""}`);
|
|
216
|
+
}
|
|
217
|
+
if (uncoveredRows.length > 0) {
|
|
218
|
+
console.log(` ${color ? DIM : ""}⬜ ${uncoveredRows.length} not covered${color ? RESET : ""}`);
|
|
219
|
+
// ARV-75 (feedback round-03 / F16): when some of the not-covered rows
|
|
220
|
+
// are deprecated, surface the count so a user reading "5% uncovered"
|
|
221
|
+
// can attribute the gap to deprecated endpoints (which generate skips
|
|
222
|
+
// by default) instead of suite regression.
|
|
223
|
+
const deprecatedUnhit = uncoveredRows.filter((r) => r.deprecated).length;
|
|
224
|
+
if (deprecatedUnhit > 0) {
|
|
225
|
+
console.log(` ${color ? DIM : ""}↳ ${deprecatedUnhit} of those are deprecated (skipped by \`zond generate\` unless --include-deprecated)${color ? RESET : ""}`);
|
|
63
226
|
}
|
|
227
|
+
}
|
|
64
228
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (r.request_method === ep.method && regex.test(normalizedUrl)) {
|
|
78
|
-
const key = `${ep.method} ${ep.path}`;
|
|
79
|
-
const existing = endpointStatus.get(key);
|
|
80
|
-
|
|
81
|
-
if (r.response_status !== null && r.response_status >= 500) {
|
|
82
|
-
endpointStatus.set(key, "api_error");
|
|
83
|
-
} else if (r.status === "fail" || r.status === "error") {
|
|
84
|
-
if (existing !== "api_error") {
|
|
85
|
-
endpointStatus.set(key, "test_failed");
|
|
86
|
-
}
|
|
87
|
-
} else if (!existing) {
|
|
88
|
-
endpointStatus.set(key, "passing");
|
|
89
|
-
}
|
|
90
|
-
break;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
229
|
+
// ARV-28: --verbose lists not-covered endpoints (and partial) inline,
|
|
230
|
+
// so users don't have to pipe through `--json | jq` for that detail.
|
|
231
|
+
if (options.verbose && (uncoveredRows.length > 0 || partialRows.length > 0)) {
|
|
232
|
+
if (partialRows.length > 0) {
|
|
233
|
+
console.log("");
|
|
234
|
+
console.log(`${color ? YELLOW : ""}Partial (only non-2xx responses):${color ? RESET : ""}`);
|
|
235
|
+
for (const row of partialRows) console.log(` ◐ ${row.endpoint}`);
|
|
236
|
+
}
|
|
237
|
+
if (uncoveredRows.length > 0) {
|
|
238
|
+
console.log("");
|
|
239
|
+
console.log(`${color ? DIM : ""}Not covered:${color ? RESET : ""}`);
|
|
240
|
+
for (const row of uncoveredRows) console.log(` ⬜ ${row.endpoint}`);
|
|
93
241
|
}
|
|
242
|
+
}
|
|
94
243
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
244
|
+
if (cov.matrix.totals.byReason["no-fixtures"] > 0 || cov.matrix.totals.byReason["auth-scope-mismatch"] > 0) {
|
|
245
|
+
console.log("");
|
|
246
|
+
if (cov.matrix.totals.byReason["no-fixtures"] > 0) {
|
|
247
|
+
// TASK-41: clarify what "blocked by no-fixtures" means — these are
|
|
248
|
+
// endpoints whose smoke/CRUD suite is *generated* but `skip_if`-gated
|
|
249
|
+
// on an empty path-param fixture. The fix is a one-shot env edit, not
|
|
250
|
+
// suite regeneration. Point users at the actual remedy.
|
|
251
|
+
console.log(
|
|
252
|
+
` ${color ? DIM : ""}↳ ${cov.matrix.totals.byReason["no-fixtures"]} ` +
|
|
253
|
+
`cells blocked by no-fixtures (suite generated, awaiting IDs in .env.yaml — ` +
|
|
254
|
+
`run \`zond prepare-fixtures --api ${cov.apiName}\` or seed manually).${color ? RESET : ""}`,
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
if (cov.matrix.totals.byReason["auth-scope-mismatch"] > 0) {
|
|
258
|
+
console.log(` ${color ? DIM : ""}↳ ${cov.matrix.totals.byReason["auth-scope-mismatch"]} cells blocked by auth-scope-mismatch${color ? RESET : ""}`);
|
|
99
259
|
}
|
|
100
260
|
}
|
|
261
|
+
} else {
|
|
262
|
+
// TASK-280: emit explicit covered2xx / coveredButNon2xx / unhit buckets
|
|
263
|
+
// so JSON consumers see the same breakdown as the text reporter. Legacy
|
|
264
|
+
// fields (covered/uncovered/partial, coveredEndpoints/partialEndpoints/
|
|
265
|
+
// uncoveredEndpoints) are kept as deprecated aliases pending full envelope-
|
|
266
|
+
// policy unification (TASK-184).
|
|
267
|
+
const buckets = bucketRows(cov.matrix);
|
|
268
|
+
printJson(jsonOk("coverage", {
|
|
269
|
+
// Legacy aliases — DO NOT add new consumers; use `totals.*` and
|
|
270
|
+
// `*Endpoints` arrays below instead.
|
|
271
|
+
covered: coveredCount,
|
|
272
|
+
uncovered: uncoveredRows.length,
|
|
273
|
+
partial: partialRows.length,
|
|
274
|
+
total,
|
|
275
|
+
percentage,
|
|
276
|
+
runId: cov.run?.id ?? null,
|
|
277
|
+
runIds: cov.runs.map((r) => r.id),
|
|
278
|
+
union_mode: cov.unionMode,
|
|
279
|
+
coveredEndpoints: coveredRows.map((r) => r.endpoint),
|
|
280
|
+
partialEndpoints: partialRows.map((r) => r.endpoint),
|
|
281
|
+
uncoveredEndpoints: uncoveredRows.map((r) => r.endpoint),
|
|
282
|
+
// Canonical buckets (TASK-280).
|
|
283
|
+
totals: {
|
|
284
|
+
all: total,
|
|
285
|
+
covered2xx: buckets.covered2xx.length,
|
|
286
|
+
coveredButNon2xx: buckets.coveredButNon2xx.length,
|
|
287
|
+
unhit: buckets.unhit.length,
|
|
288
|
+
},
|
|
289
|
+
// TASK-270: explicit twin metrics — pass_coverage is the strict
|
|
290
|
+
// "does the test land a 2xx", hit_coverage is the loose "did we
|
|
291
|
+
// touch the endpoint at all". Both expressed as endpoint-count and
|
|
292
|
+
// 0..1 ratio so CI scripts don't re-derive them from the buckets.
|
|
293
|
+
pass_coverage: { covered: passCount, total, ratio: total === 0 ? 0 : Number((passCount / total).toFixed(4)) },
|
|
294
|
+
hit_coverage: { covered: hitCount, total, ratio: total === 0 ? 0 : Number((hitCount / total).toFixed(4)) },
|
|
295
|
+
covered2xxEndpoints: buckets.covered2xx,
|
|
296
|
+
coveredButNon2xxEndpoints: buckets.coveredButNon2xx,
|
|
297
|
+
unhitEndpoints: buckets.unhit,
|
|
298
|
+
// ARV-75 (F16): expose deprecated-endpoint counts so CI / agents can
|
|
299
|
+
// distinguish "we missed coverage on a live endpoint" from "the
|
|
300
|
+
// remaining uncovered rows are spec-deprecated and explicitly skipped
|
|
301
|
+
// by zond generate" without re-deriving from the spec.
|
|
302
|
+
deprecated_unhit: uncoveredRows.filter((r) => r.deprecated).length,
|
|
303
|
+
deprecated_total: cov.matrix.rows.filter((r) => r.deprecated).length,
|
|
304
|
+
}));
|
|
305
|
+
}
|
|
101
306
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
307
|
+
if (options.failOnCoverage !== undefined) {
|
|
308
|
+
return percentage < options.failOnCoverage ? 1 : 0;
|
|
309
|
+
}
|
|
310
|
+
return uncoveredRows.length > 0 ? 1 : 0;
|
|
311
|
+
}
|
|
106
312
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
313
|
+
/**
|
|
314
|
+
* Legacy fallback: when neither `--api` nor a current API is set, report
|
|
315
|
+
* spec-only stats. Without a registered collection there is no run to read,
|
|
316
|
+
* so no endpoint is callable-covered — surface that explicitly instead of
|
|
317
|
+
* silently scanning YAML and over-reporting.
|
|
318
|
+
*/
|
|
319
|
+
async function runSpecOnlyCoverage(options: CoverageOptions): Promise<number> {
|
|
320
|
+
if (!options.spec) {
|
|
321
|
+
const msg = "Need --api <name> (preferred) or --spec <path>. Coverage is computed against stored run results.";
|
|
322
|
+
if (options.json) printJson(jsonError("coverage", [msg]));
|
|
323
|
+
else printError(msg);
|
|
324
|
+
return 2;
|
|
325
|
+
}
|
|
326
|
+
const doc = await readOpenApiSpec(options.spec);
|
|
327
|
+
const allEndpoints = extractEndpoints(doc);
|
|
328
|
+
if (allEndpoints.length === 0) {
|
|
329
|
+
printError("No endpoints found in the OpenAPI spec");
|
|
330
|
+
return 1;
|
|
331
|
+
}
|
|
332
|
+
const total = allEndpoints.length;
|
|
333
|
+
const percentage = 0;
|
|
334
|
+
|
|
335
|
+
if (!options.json) {
|
|
336
|
+
const color = useColor();
|
|
337
|
+
console.log(`Coverage: 0/${total} endpoints (0%) — no API registered`);
|
|
338
|
+
console.log("");
|
|
339
|
+
console.log(` ${color ? DIM : ""}Register the spec with \`zond add api --spec <path>\` to track run results.${color ? RESET : ""}`);
|
|
340
|
+
const warnings = analyzeEndpoints(allEndpoints);
|
|
341
|
+
if (warnings.length > 0) {
|
|
342
|
+
console.log("");
|
|
343
|
+
console.log(`${color ? YELLOW : ""}Spec warnings:${color ? RESET : ""}`);
|
|
344
|
+
for (const w of warnings) {
|
|
345
|
+
console.log(` ${color ? YELLOW : ""}⚠${color ? RESET : ""} ${w.method.padEnd(7)} ${w.path}: ${w.warnings.join(", ")}`);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
} else {
|
|
349
|
+
const unhit = allEndpoints.map((ep) => ({
|
|
350
|
+
endpoint: `${ep.method.toUpperCase()} ${ep.path}`,
|
|
351
|
+
method: ep.method.toUpperCase(),
|
|
352
|
+
path: ep.path,
|
|
353
|
+
lastStatus: null,
|
|
354
|
+
}));
|
|
355
|
+
printJson(jsonOk("coverage", {
|
|
356
|
+
covered: 0,
|
|
357
|
+
uncovered: total,
|
|
358
|
+
partial: 0,
|
|
359
|
+
total,
|
|
360
|
+
percentage,
|
|
361
|
+
runId: null,
|
|
362
|
+
coveredEndpoints: [],
|
|
363
|
+
partialEndpoints: [],
|
|
364
|
+
uncoveredEndpoints: unhit.map(u => u.endpoint),
|
|
365
|
+
totals: { all: total, covered2xx: 0, coveredButNon2xx: 0, unhit: total },
|
|
366
|
+
covered2xxEndpoints: [],
|
|
367
|
+
coveredButNon2xxEndpoints: [],
|
|
368
|
+
unhitEndpoints: unhit,
|
|
369
|
+
// TASK-270: the spec-only path has no run results, so both metrics
|
|
370
|
+
// are zero. Surface them anyway for shape-stable JSON.
|
|
371
|
+
pass_coverage: { covered: 0, total, ratio: 0 },
|
|
372
|
+
hit_coverage: { covered: 0, total, ratio: 0 },
|
|
373
|
+
}));
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
if (options.failOnCoverage !== undefined) {
|
|
377
|
+
return percentage < options.failOnCoverage ? 1 : 0;
|
|
378
|
+
}
|
|
379
|
+
return 1;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
import type { Command } from "commander";
|
|
383
|
+
import { Option } from "commander";
|
|
384
|
+
import { globalJson, resolveApiCollection } from "../resolve.ts";
|
|
385
|
+
import { parseInteger, parsePercentage } from "../argv.ts";
|
|
386
|
+
import { getApi } from "../util/api-context.ts";
|
|
387
|
+
import { readCurrentSession } from "../../core/context/session.ts";
|
|
388
|
+
import { listRunsBySession, getLatestRunByCollection, getRunById, findCollectionByNameOrId } from "../../db/queries.ts";
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* ARV-55: probe-run classification moved from path-regex heuristic into the
|
|
392
|
+
* persisted `runs.run_kind` column. Coverage's default loader query already
|
|
393
|
+
* filters `run_kind = 'regular'`, so this helper is no longer the gate — it
|
|
394
|
+
* just powers the human-readable warning when the *latest* run (regardless
|
|
395
|
+
* of kind) happens to be a probe-only one, which still surprises users.
|
|
396
|
+
*/
|
|
397
|
+
export function isProbeOnlyRun(runId: number): boolean {
|
|
398
|
+
const run = getRunById(runId);
|
|
399
|
+
return run?.run_kind === "probe";
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export type UnionSpec =
|
|
403
|
+
| { kind: "session" }
|
|
404
|
+
| { kind: "since"; durationMs: number; raw: string }
|
|
405
|
+
| { kind: "tag"; name: string }
|
|
406
|
+
| { kind: "runIds"; ids: number[] };
|
|
407
|
+
|
|
408
|
+
const DURATION_RE = /^(\d+)\s*(s|m|h|d)$/i;
|
|
409
|
+
const UNIT_MS: Record<string, number> = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 };
|
|
410
|
+
|
|
411
|
+
/** TASK-274: parse a duration like `30m`, `2h`, `7d`. Throws on bad input. */
|
|
412
|
+
export function parseDuration(value: string): number {
|
|
413
|
+
const m = value.trim().match(DURATION_RE);
|
|
414
|
+
if (!m) {
|
|
415
|
+
throw new Error(
|
|
416
|
+
`Invalid duration '${value}' — expected '<N><unit>' where unit is s/m/h/d (e.g. '30m', '24h', '7d').`,
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
const n = Number(m[1]!);
|
|
420
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
421
|
+
throw new Error(`Invalid duration '${value}' — must be a positive integer.`);
|
|
422
|
+
}
|
|
423
|
+
return n * UNIT_MS[m[2]!.toLowerCase()]!;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** TASK-255 + TASK-274: parse `--union <selector>`. Supported forms:
|
|
427
|
+
* - `session` — all runs in the active/specified session
|
|
428
|
+
* - `since:<dur>` (e.g. `since:24h`)— runs of this collection started after now-dur
|
|
429
|
+
* - `tag:<name>` — runs of this collection whose stored tags include <name>
|
|
430
|
+
* - `runs:A,B,C` or bare `A,B,C` — explicit list of run IDs (back-compat)
|
|
431
|
+
* Exported for unit tests. */
|
|
432
|
+
export function parseUnion(value: string): UnionSpec {
|
|
433
|
+
const v = value.trim();
|
|
434
|
+
if (v.length === 0) {
|
|
435
|
+
throw new Error(
|
|
436
|
+
"--union expects 'session', 'since:<dur>', 'tag:<name>', or 'runs:<id1,id2,…>' (also accepts a bare comma-separated id list).",
|
|
437
|
+
);
|
|
438
|
+
}
|
|
439
|
+
const lower = v.toLowerCase();
|
|
440
|
+
if (lower === "session") return { kind: "session" };
|
|
441
|
+
|
|
442
|
+
if (lower.startsWith("since:")) {
|
|
443
|
+
const raw = v.slice("since:".length).trim();
|
|
444
|
+
if (!raw) throw new Error("--union since:<dur> needs a duration (e.g. 'since:24h').");
|
|
445
|
+
return { kind: "since", durationMs: parseDuration(raw), raw };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
if (lower.startsWith("tag:")) {
|
|
449
|
+
const name = v.slice("tag:".length).trim();
|
|
450
|
+
if (!name) throw new Error("--union tag:<name> needs a tag (e.g. 'tag:smoke').");
|
|
451
|
+
return { kind: "tag", name };
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// `runs:` prefix is the documented form; bare comma list kept for
|
|
455
|
+
// back-compat with the original TASK-255 surface (`--union 58,59`).
|
|
456
|
+
const idsRaw = lower.startsWith("runs:") ? v.slice("runs:".length) : v;
|
|
457
|
+
const ids = idsRaw.split(",").map((s) => s.trim()).filter((s) => s.length > 0).map((s) => {
|
|
458
|
+
const n = Number(s);
|
|
459
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
460
|
+
throw new Error(
|
|
461
|
+
`--union expects 'session', 'since:<dur>', 'tag:<name>', or 'runs:<id1,id2,…>' — got '${value}'.`,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
return n;
|
|
465
|
+
});
|
|
466
|
+
if (ids.length === 0) {
|
|
467
|
+
throw new Error(
|
|
468
|
+
"--union expects 'session', 'since:<dur>', 'tag:<name>', or 'runs:<id1,id2,…>'.",
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
return { kind: "runIds", ids };
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
export function registerCoverage(program: Command): void {
|
|
475
|
+
program
|
|
476
|
+
.command("coverage")
|
|
477
|
+
.description(
|
|
478
|
+
"Analyze API test coverage from stored run results. zond reports two " +
|
|
479
|
+
"metrics side-by-side (TASK-270):\n" +
|
|
480
|
+
" pass-coverage — endpoint had at least one passing 2xx response (strict; what CI usually wants)\n" +
|
|
481
|
+
" hit-coverage — endpoint received any response at all, including 5xx and assertion failures (loose; for breadth audits)\n" +
|
|
482
|
+
"\n" +
|
|
483
|
+
"Source: only `zond run` results are aggregated (ARV-19). `zond checks " +
|
|
484
|
+
"run` probes hit the API but are not stored as run results, so they " +
|
|
485
|
+
"don't move either metric — generate suites with `zond generate` and " +
|
|
486
|
+
"execute them via `zond run` to grow coverage.\n" +
|
|
487
|
+
"\n" +
|
|
488
|
+
"Defaults to the latest stored run for the resolved API; pass " +
|
|
489
|
+
"--run-id to pin a specific run, or --union <selector> to combine " +
|
|
490
|
+
"multiple runs.\n" +
|
|
491
|
+
"\n" +
|
|
492
|
+
"--union selectors:\n" +
|
|
493
|
+
" session Active session (or --session-id <id>) — every run in the\n" +
|
|
494
|
+
" session is folded in; use this for the\n" +
|
|
495
|
+
" tests-run + probes-run pattern from one\n" +
|
|
496
|
+
" `zond session start` block.\n" +
|
|
497
|
+
" since:<dur> Time-window across the API (1h, 24h, 7d, 30m). Folds\n" +
|
|
498
|
+
" every run started within the window — handy for CI\n" +
|
|
499
|
+
" 'last-day coverage' aggregates spanning multiple sessions.\n" +
|
|
500
|
+
" tag:<name> Every run whose stored tags include <name>. Tags come\n" +
|
|
501
|
+
" from suite-level `tags:` plus any explicit `--tag <x>`\n" +
|
|
502
|
+
" on `zond run`. Useful for slicing by class\n" +
|
|
503
|
+
" (e.g. `tag:smoke`, `tag:negative`).\n" +
|
|
504
|
+
" runs:<id1,id2,...> Explicit list of run IDs (e.g. release-vs-release).\n" +
|
|
505
|
+
" A bare `<id1,id2,...>` is also accepted for back-compat.\n" +
|
|
506
|
+
"\n" +
|
|
507
|
+
"Recipe (session form):\n" +
|
|
508
|
+
" zond session start --label combined\n" +
|
|
509
|
+
" zond run apis/<api>/tests\n" +
|
|
510
|
+
" zond run apis/<api>/probes\n" +
|
|
511
|
+
" zond coverage --api <api> --union session\n" +
|
|
512
|
+
"\n" +
|
|
513
|
+
"Exit codes: 0 = every endpoint covered (or pass-coverage ≥ " +
|
|
514
|
+
"--fail-on-coverage when set); 1 = uncovered endpoints remain (or " +
|
|
515
|
+
"pass-coverage < --fail-on-coverage); 2 = bad input or read error. " +
|
|
516
|
+
"--fail-on-coverage gates pass-coverage, not hit-coverage.",
|
|
517
|
+
)
|
|
518
|
+
.option("--api <name>", "Use API collection (auto-resolves spec; reads stored runs)")
|
|
519
|
+
.option("--spec <path>", "Spec-only fallback when no API is registered (no run results)")
|
|
520
|
+
.option("--fail-on-coverage <N>", "Exit 1 when coverage percentage is below N (0–100)", parsePercentage)
|
|
521
|
+
.option("--run-id <number>", "Pin to a specific run instead of the latest", parseInteger("--run-id"))
|
|
522
|
+
.option("--session-id <id>", "Union all runs in this session (filtered to the chosen API)")
|
|
523
|
+
.option(
|
|
524
|
+
"--union <selector>",
|
|
525
|
+
"Combine multiple runs. Selector: 'session', 'since:<dur>' (e.g. since:24h), 'tag:<name>', or 'runs:<id1,id2,…>' (bare comma-list also accepted)",
|
|
526
|
+
)
|
|
527
|
+
.option("--db <path>", "Path to SQLite database file")
|
|
528
|
+
.option("--verbose", "List not-covered (and partial) endpoints inline — same data as `--json` but human-readable")
|
|
529
|
+
// ARV-35: `--format json` matches the kubectl/gh/aws-cli convention many
|
|
530
|
+
// users reach for first; until ARV-54 lands a workspace-wide alias layer
|
|
531
|
+
// we accept it locally and forward to `--json`. Other values are rejected
|
|
532
|
+
// (no markdown reporter on coverage) so typos still fail loud.
|
|
533
|
+
.addOption(
|
|
534
|
+
new Option("--format <fmt>", "Alias for --json (parity with kubectl/gh/aws-cli)").choices(["json"]),
|
|
535
|
+
)
|
|
536
|
+
.action(async (opts, cmd: Command) => {
|
|
537
|
+
if (opts.format === "json") opts.json = true;
|
|
538
|
+
// ARV-53: only walk the --api chain when --spec wasn't provided —
|
|
539
|
+
// an explicit spec disables the current-API fallback (coverage's
|
|
540
|
+
// legacy mode supports bare-spec usage).
|
|
541
|
+
const apiFlag = opts.spec ? (opts.api as string | undefined) : getApi(cmd, opts);
|
|
542
|
+
let apiName: string | undefined;
|
|
543
|
+
let spec: string | undefined = opts.spec;
|
|
544
|
+
|
|
545
|
+
if (apiFlag) {
|
|
546
|
+
const resolved = resolveApiCollection(apiFlag, opts.db);
|
|
547
|
+
if ("error" in resolved) {
|
|
548
|
+
printError(resolved.error);
|
|
549
|
+
process.exitCode = resolved.error.startsWith("Failed") ? 2 : 1;
|
|
550
|
+
return;
|
|
118
551
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
console.log("");
|
|
552
|
+
apiName = apiFlag;
|
|
553
|
+
if (!spec && resolved.spec) spec = resolved.spec;
|
|
554
|
+
}
|
|
123
555
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
556
|
+
// Resolve --union and --session-id. --session-id wins; --union session
|
|
557
|
+
// resolves via .zond/current-session; --union since:/tag:/runs: are
|
|
558
|
+
// routed via discrete loader options so the loader can do one DB query.
|
|
559
|
+
let sessionId: string | undefined = opts.sessionId;
|
|
560
|
+
let runIds: number[] | undefined;
|
|
561
|
+
let sinceIso: string | undefined;
|
|
562
|
+
let tag: string | undefined;
|
|
563
|
+
if (opts.union) {
|
|
564
|
+
try {
|
|
565
|
+
const parsed = parseUnion(opts.union as string);
|
|
566
|
+
if (parsed.kind === "session") {
|
|
567
|
+
const current = readCurrentSession();
|
|
568
|
+
if (!current) {
|
|
569
|
+
printError("--union session requires an active session (run 'zond session start' first), or pass --session-id <id>.");
|
|
570
|
+
process.exitCode = 2;
|
|
571
|
+
return;
|
|
130
572
|
}
|
|
573
|
+
sessionId = current.id;
|
|
574
|
+
} else if (parsed.kind === "since") {
|
|
575
|
+
// Anchor the window at "now minus dur" — coverage CLI is
|
|
576
|
+
// wall-clock-driven, the loader just sees the resolved ISO.
|
|
577
|
+
sinceIso = new Date(Date.now() - parsed.durationMs).toISOString();
|
|
578
|
+
} else if (parsed.kind === "tag") {
|
|
579
|
+
tag = parsed.name;
|
|
580
|
+
} else {
|
|
581
|
+
runIds = parsed.ids;
|
|
131
582
|
}
|
|
132
|
-
|
|
583
|
+
} catch (err) {
|
|
584
|
+
printError(err instanceof Error ? err.message : String(err));
|
|
585
|
+
process.exitCode = 2;
|
|
586
|
+
return;
|
|
133
587
|
}
|
|
588
|
+
}
|
|
134
589
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
590
|
+
// since:/tag: only make sense with an API resolved (they query the
|
|
591
|
+
// collection's run history). Fail fast rather than silently scoping to
|
|
592
|
+
// every collection.
|
|
593
|
+
if ((sinceIso || tag) && !apiName) {
|
|
594
|
+
printError("--union since:/tag: requires an API. Pass --api <name> or set the current API.");
|
|
595
|
+
process.exitCode = 2;
|
|
596
|
+
return;
|
|
142
597
|
}
|
|
143
598
|
|
|
144
|
-
//
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
599
|
+
// ARV-71 (feedback round-02 / F12): when --api X is set and a zond
|
|
600
|
+
// session is active with more than one run, auto-promote the default
|
|
601
|
+
// to `--union session`. The pre-ARV-71 behaviour ("latest run only")
|
|
602
|
+
// misreads as a coverage regression every time a user runs a partial
|
|
603
|
+
// suite mid-session, and the previous stderr hint was easy to miss
|
|
604
|
+
// (the percentage already looked like a regression). Explicit
|
|
605
|
+
// selectors win, --json keeps the envelope untouched.
|
|
606
|
+
const noSelector = !opts.runId && !sessionId && !runIds && !sinceIso && !tag;
|
|
607
|
+
if (apiName && noSelector) {
|
|
608
|
+
const current = readCurrentSession();
|
|
609
|
+
if (current) {
|
|
610
|
+
const sessRuns = listRunsBySession(current.id);
|
|
611
|
+
if (sessRuns.length > 1) {
|
|
612
|
+
sessionId = current.id;
|
|
613
|
+
if (!globalJson(cmd)) {
|
|
614
|
+
process.stderr.write(
|
|
615
|
+
`zond: active session has ${sessRuns.length} runs — defaulting to --union session (pass --run-id <N> for a single run).\n`,
|
|
616
|
+
);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
151
619
|
}
|
|
620
|
+
// ARV-41: warn when the latest run is probe-only — otherwise
|
|
621
|
+
// `zond coverage` right after `zond run apis/<api>/probes/...`
|
|
622
|
+
// looks like a regression vs the prior smoke/CRUD run.
|
|
623
|
+
try {
|
|
624
|
+
const collection = findCollectionByNameOrId(apiName);
|
|
625
|
+
if (collection) {
|
|
626
|
+
// ARV-55: peek at the absolute latest run (`runKind: 'any'`).
|
|
627
|
+
// Coverage's default loader query already skips probe runs
|
|
628
|
+
// via `run_kind = 'regular'`, so the user won't see a
|
|
629
|
+
// regression — but if their *most recent* invocation was a
|
|
630
|
+
// probe-only run, the inline warning keeps it visible.
|
|
631
|
+
const latest = getLatestRunByCollection(collection.id, { runKind: "any" });
|
|
632
|
+
if (latest && latest.run_kind === "probe") {
|
|
633
|
+
const hint = `Latest run #${latest.id} only executed probe suites — coverage falls back to the prior smoke/CRUD run. ` +
|
|
634
|
+
`For combined coverage, wrap your runs in 'zond session start/end' and pass '--union session' here.`;
|
|
635
|
+
process.stderr.write(`zond: ${hint}\n`);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
} catch { /* DB inspection is best-effort, don't break coverage */ }
|
|
152
639
|
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
if (options.json) {
|
|
156
|
-
const coveredEndpoints = allEndpoints.filter(ep => !uncovered.includes(ep)).map(ep => `${ep.method} ${ep.path}`);
|
|
157
|
-
const uncoveredEndpoints = uncovered.map(ep => `${ep.method} ${ep.path}`);
|
|
158
|
-
printJson(jsonOk("coverage", {
|
|
159
|
-
covered: coveredCount,
|
|
160
|
-
uncovered: uncovered.length,
|
|
161
|
-
total: allEndpoints.length,
|
|
162
|
-
percentage,
|
|
163
|
-
coveredEndpoints,
|
|
164
|
-
uncoveredEndpoints,
|
|
165
|
-
}));
|
|
166
|
-
}
|
|
167
640
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
641
|
+
process.exitCode = await coverageCommand({
|
|
642
|
+
...(apiName ? { apiName } : {}),
|
|
643
|
+
...(spec ? { spec } : {}),
|
|
644
|
+
failOnCoverage: opts.failOnCoverage,
|
|
645
|
+
runId: opts.runId,
|
|
646
|
+
...(runIds ? { runIds } : {}),
|
|
647
|
+
...(sessionId ? { sessionId } : {}),
|
|
648
|
+
...(sinceIso ? { sinceIso } : {}),
|
|
649
|
+
...(tag ? { tag } : {}),
|
|
650
|
+
json: globalJson(cmd),
|
|
651
|
+
verbose: opts.verbose === true,
|
|
652
|
+
});
|
|
653
|
+
});
|
|
181
654
|
}
|