@kirrosh/zond 0.22.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +811 -0
- package/README.md +59 -6
- package/package.json +9 -7
- package/src/CLAUDE.md +112 -0
- package/src/cli/argv.ts +122 -0
- package/src/cli/commands/add-api.ts +146 -0
- package/src/cli/commands/api/annotate/idempotency.ts +59 -0
- package/src/cli/commands/api/annotate/index.ts +880 -0
- package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
- package/src/cli/commands/api/annotate/overlay.ts +206 -0
- package/src/cli/commands/api/annotate/pagination.ts +64 -0
- package/src/cli/commands/api/annotate/prompts.ts +220 -0
- package/src/cli/commands/api/annotate/readback.ts +58 -0
- package/src/cli/commands/api/annotate/resources.ts +91 -0
- package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
- package/src/cli/commands/audit.ts +786 -0
- package/src/cli/commands/catalog.ts +35 -0
- package/src/cli/commands/check.ts +361 -0
- package/src/cli/commands/checks.ts +1072 -0
- package/src/cli/commands/ci-init.ts +43 -0
- package/src/cli/commands/clean.ts +212 -0
- package/src/cli/commands/cleanup.ts +236 -0
- package/src/cli/commands/completions.ts +16 -0
- package/src/cli/commands/coverage.ts +823 -132
- package/src/cli/commands/db.ts +486 -12
- package/src/cli/commands/describe.ts +37 -2
- package/src/cli/commands/discover.ts +1356 -0
- package/src/cli/commands/doctor.ts +661 -0
- package/src/cli/commands/fixtures.ts +402 -0
- package/src/cli/commands/generate.ts +438 -47
- package/src/cli/commands/init/bootstrap.ts +34 -2
- package/src/cli/commands/{init.ts → init/index.ts} +99 -5
- package/src/cli/commands/init/skills.ts +99 -3
- package/src/cli/commands/init/templates/agents.md +77 -64
- package/src/cli/commands/init/templates/skills/warm-up-target.md +122 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +621 -0
- package/src/cli/commands/init/templates/skills/zond-seed.md +114 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +272 -0
- package/src/cli/commands/init/templates/skills/zond.md +802 -125
- package/src/cli/commands/init/templates/zond-config.yml +8 -9
- package/src/cli/commands/prepare-fixtures.ts +97 -0
- package/src/cli/commands/probe/_seed-bodies.ts +52 -0
- package/src/cli/commands/probe/mass-assignment.ts +594 -0
- package/src/cli/commands/probe/security.ts +537 -0
- package/src/cli/commands/probe/static.ts +255 -0
- package/src/cli/commands/probe/webhooks.ts +163 -0
- package/src/cli/commands/probe.ts +535 -0
- package/src/cli/commands/reference.ts +87 -0
- package/src/cli/commands/refresh-api.ts +227 -0
- package/src/cli/commands/remove-api.ts +150 -0
- package/src/cli/commands/report-bundle.ts +310 -0
- package/src/cli/commands/report.ts +241 -0
- package/src/cli/commands/request.ts +495 -4
- package/src/cli/commands/run.ts +870 -53
- package/src/cli/commands/schema-from-runs.ts +128 -0
- package/src/cli/commands/secrets.ts +133 -0
- package/src/cli/commands/session.ts +244 -0
- package/src/cli/commands/use.ts +18 -1
- package/src/cli/index.ts +20 -3
- package/src/cli/json-envelope.ts +92 -3
- package/src/cli/json-schemas.ts +314 -0
- package/src/cli/output.ts +17 -1
- package/src/cli/program.ts +199 -635
- package/src/cli/resolve.ts +105 -0
- package/src/cli/safe-live.ts +24 -0
- package/src/cli/status-filter.ts +114 -0
- package/src/cli/util/api-context.ts +85 -0
- package/src/cli/version.ts +5 -0
- package/src/core/audit/persist.ts +183 -0
- package/src/core/checks/budget.ts +59 -0
- package/src/core/checks/checks/_crud-helpers.ts +133 -0
- package/src/core/checks/checks/_negative_mutator.ts +133 -0
- package/src/core/checks/checks/_readback-helpers.ts +133 -0
- package/src/core/checks/checks/content_type_conformance.ts +39 -0
- package/src/core/checks/checks/cross_call_references.ts +147 -0
- package/src/core/checks/checks/cursor_boundary_fuzzing.ts +219 -0
- package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
- package/src/core/checks/checks/idempotency_replay.ts +242 -0
- package/src/core/checks/checks/ignored_auth.ts +254 -0
- package/src/core/checks/checks/index.ts +68 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +416 -0
- package/src/core/checks/checks/missing_required_header.ts +40 -0
- package/src/core/checks/checks/negative_data_rejection.ts +148 -0
- package/src/core/checks/checks/not_a_server_error.ts +35 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +160 -0
- package/src/core/checks/checks/pagination_invariants.ts +419 -0
- package/src/core/checks/checks/positive_data_acceptance.ts +33 -0
- package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
- package/src/core/checks/checks/response_headers_conformance.ts +74 -0
- package/src/core/checks/checks/response_schema_conformance.ts +30 -0
- package/src/core/checks/checks/status_code_conformance.ts +132 -0
- package/src/core/checks/checks/unsupported_method.ts +63 -0
- package/src/core/checks/checks/use_after_free.ts +78 -0
- package/src/core/checks/index.ts +30 -0
- package/src/core/checks/mode.ts +82 -0
- package/src/core/checks/recommended-action.ts +68 -0
- package/src/core/checks/registry.ts +78 -0
- package/src/core/checks/runner.ts +1461 -0
- package/src/core/checks/sarif.ts +230 -0
- package/src/core/checks/spec-findings.ts +308 -0
- package/src/core/checks/stateful.ts +121 -0
- package/src/core/checks/types.ts +305 -0
- package/src/core/checks/zond-extensions.ts +73 -0
- package/src/core/classifier/recommended-action.ts +251 -0
- package/src/core/context/current.ts +22 -6
- package/src/core/context/session.ts +78 -0
- package/src/core/coverage/loader.ts +216 -0
- package/src/core/coverage/reasons.ts +300 -0
- package/src/core/diagnostics/db-analysis.ts +293 -59
- package/src/core/diagnostics/failure-class.ts +140 -0
- package/src/core/diagnostics/failure-hints.ts +88 -89
- package/src/core/diagnostics/spec-pointer.ts +99 -0
- package/src/core/diagnostics/suggested-fixes.ts +155 -0
- package/src/core/exporter/case-study/index.ts +270 -0
- package/src/core/exporter/curl.ts +40 -0
- package/src/core/exporter/exporter.ts +48 -0
- package/src/core/exporter/html-report/escape.ts +24 -0
- package/src/core/exporter/html-report/index.ts +479 -0
- package/src/core/exporter/html-report/script.ts +100 -0
- package/src/core/exporter/html-report/styles.ts +408 -0
- package/src/core/generator/chunker.ts +38 -19
- package/src/core/generator/coverage-phase.ts +0 -0
- package/src/core/generator/data-factory.ts +586 -22
- package/src/core/generator/describe.ts +1 -1
- package/src/core/generator/fixtures-builder.ts +332 -0
- package/src/core/generator/index.ts +5 -5
- package/src/core/generator/openapi-reader.ts +135 -7
- package/src/core/generator/path-param-disambig.ts +140 -0
- package/src/core/generator/resources-builder.ts +898 -0
- package/src/core/generator/schema-utils.ts +33 -3
- package/src/core/generator/serializer.ts +103 -13
- package/src/core/generator/suite-generator.ts +583 -122
- package/src/core/generator/types.ts +14 -0
- package/src/core/identity/identity-file.ts +0 -0
- package/src/core/lint/affects.ts +28 -0
- package/src/core/lint/config.ts +96 -0
- package/src/core/lint/format.ts +42 -0
- package/src/core/lint/index.ts +94 -0
- package/src/core/lint/reporter.ts +128 -0
- package/src/core/lint/rules/consistency.ts +158 -0
- package/src/core/lint/rules/heuristics.ts +97 -0
- package/src/core/lint/rules/strictness.ts +109 -0
- package/src/core/lint/types.ts +96 -0
- package/src/core/lint/walker.ts +248 -0
- package/src/core/meta/meta-store.ts +6 -73
- package/src/core/output/README.md +73 -0
- package/src/core/output/index.ts +13 -0
- package/src/core/output/run.ts +91 -0
- package/src/core/output/types.ts +122 -0
- package/src/core/parser/dynamic-values.ts +160 -0
- package/src/core/parser/env-interpolation.ts +104 -0
- package/src/core/parser/filter.ts +57 -0
- package/src/core/parser/schema.ts +129 -4
- package/src/core/parser/types.ts +19 -1
- package/src/core/parser/variables.ts +0 -0
- package/src/core/parser/yaml-parser.ts +58 -12
- package/src/core/probe/bootstrap.ts +34 -0
- package/src/core/probe/dry-run-envelope.ts +61 -0
- package/src/core/probe/mass-assignment/classify.ts +175 -0
- package/src/core/probe/mass-assignment/cleanup.ts +52 -0
- package/src/core/probe/mass-assignment/digest.ts +114 -0
- package/src/core/probe/mass-assignment/orchestrator.ts +459 -0
- package/src/core/probe/mass-assignment/regression.ts +141 -0
- package/src/core/probe/mass-assignment/suspects.ts +92 -0
- package/src/core/probe/mass-assignment/types.ts +135 -0
- package/src/core/probe/mass-assignment-probe-class.ts +198 -0
- package/src/core/probe/mass-assignment-probe.ts +27 -0
- package/src/core/probe/mass-assignment-template.ts +240 -0
- package/src/core/probe/method-probe.ts +43 -76
- package/src/core/probe/method-shared.ts +69 -0
- package/src/core/probe/negative-probe.ts +183 -149
- package/src/core/probe/orphan-tracker.ts +188 -0
- package/src/core/probe/path-discovery.ts +439 -0
- package/src/core/probe/probe-harness.ts +119 -0
- package/src/core/probe/registry.ts +89 -0
- package/src/core/probe/runner.ts +136 -0
- package/src/core/probe/security/baseline.ts +174 -0
- package/src/core/probe/security/classify.ts +341 -0
- package/src/core/probe/security/cleanup.ts +125 -0
- package/src/core/probe/security/detectors.ts +71 -0
- package/src/core/probe/security/digest.ts +104 -0
- package/src/core/probe/security/orchestrator.ts +398 -0
- package/src/core/probe/security/regression.ts +103 -0
- package/src/core/probe/security/types.ts +151 -0
- package/src/core/probe/security-probe-class.ts +207 -0
- package/src/core/probe/security-probe.ts +32 -0
- package/src/core/probe/shared.ts +531 -0
- package/src/core/probe/static-probe-class.ts +125 -0
- package/src/core/probe/types.ts +165 -0
- package/src/core/probe/verdict-aggregator.ts +33 -0
- package/src/core/probe/webhooks-probe.ts +282 -0
- package/src/core/reporter/console.ts +41 -2
- package/src/core/reporter/index.ts +2 -3
- package/src/core/reporter/json.ts +11 -1
- package/src/core/reporter/junit.ts +27 -12
- package/src/core/reporter/ndjson.ts +37 -0
- package/src/core/reporter/types.ts +3 -0
- package/src/core/runner/assertions.ts +59 -2
- package/src/core/runner/async-pool.ts +108 -0
- package/src/core/runner/auth-path.ts +8 -0
- package/src/core/runner/ci-context.ts +72 -0
- package/src/core/runner/executor.ts +265 -36
- package/src/core/runner/form-encode.ts +41 -0
- package/src/core/runner/http-client.ts +112 -2
- package/src/core/runner/learn-drift.ts +293 -0
- package/src/core/runner/preflight-vars.ts +153 -0
- package/src/core/runner/progress-tracker.ts +73 -0
- package/src/core/runner/rate-limiter.ts +87 -33
- package/src/core/runner/run-kind.ts +45 -0
- package/src/core/runner/schema-validator.ts +308 -0
- package/src/core/runner/send-request.ts +158 -20
- package/src/core/runner/types.ts +44 -0
- package/src/core/secrets/registry.ts +164 -0
- package/src/core/secrets/secrets-file.ts +115 -0
- package/src/core/selectors/operation-filter.ts +144 -0
- package/src/core/setup-api.ts +457 -20
- package/src/core/severity/category.ts +94 -0
- package/src/core/severity/index.ts +58 -0
- package/src/core/spec/infer-schema.ts +102 -0
- package/src/core/spec/layers.ts +154 -0
- package/src/core/spec/merge-specs.ts +156 -0
- package/src/core/spec/schema-from-runs.ts +117 -0
- package/src/core/spec/schema-overlay.ts +130 -0
- package/src/core/util/ajv.ts +13 -0
- package/src/core/util/format-eta.ts +21 -0
- package/src/core/util/headers.ts +9 -0
- package/src/core/util/url.ts +24 -0
- package/src/core/utils.ts +5 -1
- package/src/core/workspace/config.ts +129 -0
- package/src/core/workspace/fixture-gap-report.ts +84 -0
- package/src/core/workspace/fixture-gaps.ts +71 -0
- package/src/core/workspace/manifest.ts +283 -0
- package/src/core/workspace/output-rotation.ts +62 -0
- package/src/core/workspace/root.ts +13 -11
- package/src/core/workspace/triage-path.ts +87 -0
- package/src/db/lint-runs.ts +47 -0
- package/src/db/migrate.ts +128 -0
- package/src/db/migrations/0001_run_kind.sql +25 -0
- package/src/db/migrations/0002_run_kind_request.sql +59 -0
- package/src/db/migrations/sql.d.ts +4 -0
- package/src/db/queries/collections.ts +133 -0
- package/src/db/queries/coverage.ts +9 -0
- package/src/db/queries/dashboard.ts +59 -0
- package/src/db/queries/results.ts +216 -0
- package/src/db/queries/runs.ts +289 -0
- package/src/db/queries/sessions.ts +42 -0
- package/src/db/queries/settings.ts +28 -0
- package/src/db/queries/types.ts +172 -0
- package/src/db/queries.ts +75 -802
- package/src/db/schema.ts +178 -50
- package/src/cli/commands/export.ts +0 -144
- package/src/cli/commands/guide.ts +0 -127
- package/src/cli/commands/init/templates/skills/scenarios.md +0 -97
- package/src/cli/commands/probe-methods.ts +0 -108
- package/src/cli/commands/probe-validation.ts +0 -124
- package/src/cli/commands/serve.ts +0 -114
- package/src/cli/commands/sync.ts +0 -268
- package/src/cli/commands/update.ts +0 -189
- package/src/cli/commands/validate.ts +0 -34
- package/src/core/diagnostics/render-md.ts +0 -112
- package/src/core/exporter/postman.ts +0 -963
- package/src/core/generator/guide-builder.ts +0 -253
- package/src/core/meta/types.ts +0 -19
- package/src/core/parser/index.ts +0 -21
- package/src/core/runner/execute-run.ts +0 -132
- package/src/core/runner/index.ts +0 -12
- package/src/core/sync/spec-differ.ts +0 -38
- package/src/web/data/collection-state.ts +0 -362
- package/src/web/routes/api.ts +0 -314
- package/src/web/routes/dashboard.ts +0 -350
- package/src/web/routes/runs.ts +0 -64
- package/src/web/schemas.ts +0 -121
- package/src/web/server.ts +0 -134
- package/src/web/static/htmx.min.cjs +0 -1
- package/src/web/static/style.css +0 -1148
- package/src/web/views/endpoints-tab.ts +0 -174
- package/src/web/views/explorer-tab.ts +0 -402
- package/src/web/views/health-strip.ts +0 -92
- package/src/web/views/layout.ts +0 -48
- package/src/web/views/results.ts +0 -210
- package/src/web/views/runs-tab.ts +0 -126
- package/src/web/views/suites-tab.ts +0 -181
|
@@ -0,0 +1,1072 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `zond checks` umbrella — schemathesis-style depth checks framework
|
|
3
|
+
* (m-15 ARV-1). Two subcommands today:
|
|
4
|
+
*
|
|
5
|
+
* zond checks list — emit the registered check catalog so
|
|
6
|
+
* agents can discover what's available.
|
|
7
|
+
* zond checks run — execute the active checks against a
|
|
8
|
+
* live API and emit findings.
|
|
9
|
+
*
|
|
10
|
+
* Built-in checks register themselves on import via `core/checks` —
|
|
11
|
+
* adding a new check (ARV-2/3/4) doesn't require touching this file.
|
|
12
|
+
*/
|
|
13
|
+
import { writeFileSync, readFileSync, openSync, writeSync, closeSync } from "node:fs";
|
|
14
|
+
import { resolve as resolvePath, relative as relativePath } from "node:path";
|
|
15
|
+
import type { Command } from "commander";
|
|
16
|
+
|
|
17
|
+
import { listChecks, runChecks } from "../../core/checks/index.ts";
|
|
18
|
+
import { listStatefulChecks } from "../../core/checks/stateful.ts";
|
|
19
|
+
import { resolveBudget, isBudget, BUDGETS, type Budget } from "../../core/checks/budget.ts";
|
|
20
|
+
import { generateSarifReport } from "../../core/checks/sarif.ts";
|
|
21
|
+
import { emitToStdout, nowIso } from "../../core/reporter/ndjson.ts";
|
|
22
|
+
import { parseWorkers } from "../../core/runner/async-pool.ts";
|
|
23
|
+
import { createAdaptiveRateLimiter, createRateLimiter, type RateLimiter } from "../../core/runner/rate-limiter.ts";
|
|
24
|
+
import { compileOperationFilter } from "../../core/selectors/operation-filter.ts";
|
|
25
|
+
import { resolveSpecArg, globalJson, resolveApiCollection } from "../resolve.ts";
|
|
26
|
+
import { readResourceMap } from "./discover.ts";
|
|
27
|
+
import type { ReadbackDiffConfig, IdempotencyConfig, PaginationConfig, LifecycleConfig, SeedBodyConfig } from "../../core/generator/resources-builder.ts";
|
|
28
|
+
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
29
|
+
import { printError, printSuccess } from "../output.ts";
|
|
30
|
+
import { SAFE_HELP, LIVE_HELP, resolveLive } from "../safe-live.ts";
|
|
31
|
+
import { loadEnvironment } from "../../core/parser/variables.ts";
|
|
32
|
+
import { readFixtureGaps, gapIndex } from "../../core/workspace/fixture-gaps.ts";
|
|
33
|
+
import { getApi } from "../util/api-context.ts";
|
|
34
|
+
import { VERSION } from "../version.ts";
|
|
35
|
+
import { resolveOutput, OutputSpecError, type OutputSpec, type ResolvedOutput } from "../../core/output/index.ts";
|
|
36
|
+
import type { RunChecksResult } from "../../core/checks/index.ts";
|
|
37
|
+
import type { ChecksCaseEvent } from "../../core/checks/runner.ts";
|
|
38
|
+
import { beginAuditRun, finalizeAuditRun, checksPersistEnabled, type AuditCaseRecord } from "../../core/audit/persist.ts";
|
|
39
|
+
import { readCurrentSession } from "../../core/context/session.ts";
|
|
40
|
+
import { getDb } from "../../db/schema.ts";
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* ARV-118 (m-19): typed declaration of every `--report` / `--output` /
|
|
44
|
+
* `--json` combination `zond checks run` supports. Replaces the inline
|
|
45
|
+
* `--report sarif|ndjson` parser + the legacy `--ndjson` boolean + the
|
|
46
|
+
* mutual-exclusion checks that produced ARV-63, ARV-83, ARV-97.
|
|
47
|
+
*
|
|
48
|
+
* - `console` (default) — human-readable text on stdout.
|
|
49
|
+
* - `json` — `{ok, command, data}` envelope (`--json` opts into this).
|
|
50
|
+
* - `ndjson` — streamed events on stdout, one JSON object per line.
|
|
51
|
+
* `--report ndjson --output <path>` redirects the stream to file
|
|
52
|
+
* (ARV-97 — no more silent drop).
|
|
53
|
+
* - `sarif` — SARIF v2.1.0 for GitHub Code Scanning, default file
|
|
54
|
+
* `zond-checks.sarif` when `--output` is omitted.
|
|
55
|
+
* - `markdown` — short human-readable summary (file via `--output`
|
|
56
|
+
* or stdout otherwise).
|
|
57
|
+
*/
|
|
58
|
+
export const CHECKS_OUTPUT_SPEC: OutputSpec<RunChecksResult> = {
|
|
59
|
+
command: "checks run",
|
|
60
|
+
defaultFormat: "console",
|
|
61
|
+
formats: {
|
|
62
|
+
console: { defaultChannel: "stdout", description: "Human-readable summary (default)" },
|
|
63
|
+
json: { defaultChannel: "stdout", envelopeWrap: true, envelopeSchemaFile: "checksRunData.schema.json", description: "JSON envelope ({ok, command, data})" },
|
|
64
|
+
ndjson: { defaultChannel: "stdout", description: "Stream events on stdout (check_start | check_result | finding | summary)" },
|
|
65
|
+
sarif: { defaultChannel: "file", defaultFilename: "zond-checks.sarif", description: "SARIF v2.1.0 for GitHub Code Scanning" },
|
|
66
|
+
markdown: { defaultChannel: "stdout", description: "Short markdown summary of findings" },
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
interface ChecksListOptions {
|
|
71
|
+
json?: boolean;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function checksListAction(_args: unknown, cmd: Command): Promise<void> {
|
|
75
|
+
const opts = cmd.opts<ChecksListOptions>();
|
|
76
|
+
const json = opts.json === true || globalJson(cmd);
|
|
77
|
+
const catalog = [
|
|
78
|
+
...listChecks().map((c) => ({
|
|
79
|
+
id: c.id,
|
|
80
|
+
severity: c.severity,
|
|
81
|
+
default_expected: c.defaultExpected,
|
|
82
|
+
references: c.references,
|
|
83
|
+
phase: "response" as const,
|
|
84
|
+
})),
|
|
85
|
+
...listStatefulChecks().map((c) => ({
|
|
86
|
+
id: c.id,
|
|
87
|
+
severity: c.severity,
|
|
88
|
+
default_expected: c.defaultExpected,
|
|
89
|
+
references: c.references,
|
|
90
|
+
phase: c.phase,
|
|
91
|
+
})),
|
|
92
|
+
].sort((a, b) => a.id.localeCompare(b.id));
|
|
93
|
+
|
|
94
|
+
if (json) {
|
|
95
|
+
printJson(jsonOk("checks list", { checks: catalog }));
|
|
96
|
+
} else {
|
|
97
|
+
printSuccess(`${catalog.length} check(s) registered`);
|
|
98
|
+
for (const c of catalog) {
|
|
99
|
+
console.log(` ${c.id} [${c.severity}] — ${c.default_expected}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
process.exit(0);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
interface ChecksRunOptions {
|
|
106
|
+
api?: string;
|
|
107
|
+
spec?: string;
|
|
108
|
+
baseUrl?: string;
|
|
109
|
+
check?: string[];
|
|
110
|
+
excludeCheck?: string[];
|
|
111
|
+
timeout?: number;
|
|
112
|
+
db?: string;
|
|
113
|
+
json?: boolean;
|
|
114
|
+
authHeader?: string[];
|
|
115
|
+
bootstrapCleanupFailed?: boolean;
|
|
116
|
+
report?: string;
|
|
117
|
+
output?: string;
|
|
118
|
+
phase?: string;
|
|
119
|
+
allowX00?: boolean;
|
|
120
|
+
mode?: string;
|
|
121
|
+
include?: string[];
|
|
122
|
+
exclude?: string[];
|
|
123
|
+
workers?: string;
|
|
124
|
+
rateLimit?: string;
|
|
125
|
+
verbose?: boolean;
|
|
126
|
+
strict405?: boolean;
|
|
127
|
+
strict401?: boolean;
|
|
128
|
+
maxRequests?: number;
|
|
129
|
+
/** ARV-342: operation-window for bounded/resumable sweeps of large specs. */
|
|
130
|
+
maxOps?: number;
|
|
131
|
+
skipOps?: number;
|
|
132
|
+
budget?: string;
|
|
133
|
+
showSuppressed?: boolean;
|
|
134
|
+
/** ARV-308: `--no-fail-on-findings` sets this to false (commander default
|
|
135
|
+
* true) — keep exit 0 even when HIGH/CRITICAL findings exist so an
|
|
136
|
+
* orchestrator can distinguish "found drift" from "command failed". */
|
|
137
|
+
failOnFindings?: boolean;
|
|
138
|
+
/** ARV-308: `--advisory` alias for --no-fail-on-findings. */
|
|
139
|
+
advisory?: boolean;
|
|
140
|
+
/** ARV-299: safe/live parity with `audit`. Default safe — mutating
|
|
141
|
+
* stateful create-chains self-skip; `--live` runs them. */
|
|
142
|
+
safe?: boolean;
|
|
143
|
+
live?: boolean;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function parseAuthHeaders(values: string[] | undefined): Record<string, string> {
|
|
147
|
+
const out: Record<string, string> = {};
|
|
148
|
+
for (const raw of values ?? []) {
|
|
149
|
+
const idx = raw.indexOf(":");
|
|
150
|
+
if (idx <= 0) continue;
|
|
151
|
+
const name = raw.slice(0, idx).trim();
|
|
152
|
+
const value = raw.slice(idx + 1).trim();
|
|
153
|
+
if (name) out[name] = value;
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** ARV-141: lift filled fixtures from `apis/<name>/.env.yaml` so the runner
|
|
159
|
+
* can substitute them into path-params. Keeps the result string-only (drops
|
|
160
|
+
* numeric/object values silently — they can't be URL-encoded path segments
|
|
161
|
+
* anyway) and skips obvious placeholders so a TODO-string doesn't masquerade
|
|
162
|
+
* as a real id and produce phantom-200s. */
|
|
163
|
+
/**
|
|
164
|
+
* ARV-169: load per-resource overrides for stateful checks. Reads
|
|
165
|
+
* `apis/<name>/.api-resources.yaml` (+ `.local.yaml` overlay through
|
|
166
|
+
* `readResourceMap`) and surfaces each resource's `readback_diff`
|
|
167
|
+
* block keyed by resource name. Returns undefined when no API context
|
|
168
|
+
* is in scope (raw `--spec` invocation without a registered API) so
|
|
169
|
+
* the runner falls back to default ignore patterns.
|
|
170
|
+
*/
|
|
171
|
+
async function deriveResourceConfigsFromApi(
|
|
172
|
+
apiName: string | undefined,
|
|
173
|
+
dbPath: string | undefined,
|
|
174
|
+
): Promise<Map<string, ResourceConfigEntry> | undefined> {
|
|
175
|
+
if (!apiName) return undefined;
|
|
176
|
+
const col = resolveApiCollection(apiName, dbPath);
|
|
177
|
+
if ("error" in col) return undefined;
|
|
178
|
+
if (!col.baseDir) return undefined;
|
|
179
|
+
const map = await readResourceMap(col.baseDir);
|
|
180
|
+
if (!map) return undefined;
|
|
181
|
+
const out = new Map<string, ResourceConfigEntry>();
|
|
182
|
+
for (const r of map.resources) {
|
|
183
|
+
if (!r.readback_diff && !r.idempotency && !r.pagination && !r.lifecycle && !r.seed_body) continue;
|
|
184
|
+
const entry: ResourceConfigEntry = {};
|
|
185
|
+
if (r.seed_body) {
|
|
186
|
+
entry.seedBody = {
|
|
187
|
+
contentType: r.seed_body.content_type,
|
|
188
|
+
body: r.seed_body.body,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
if (r.readback_diff) {
|
|
192
|
+
entry.readbackDiff = {
|
|
193
|
+
ignoreFields: r.readback_diff.ignore_fields,
|
|
194
|
+
writeToReadMap: r.readback_diff.write_to_read_map,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
if (r.idempotency) {
|
|
198
|
+
entry.idempotency = {
|
|
199
|
+
header: r.idempotency.header,
|
|
200
|
+
scope: r.idempotency.scope,
|
|
201
|
+
ignoreResponseFields: r.idempotency.ignore_response_fields,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
if (r.pagination) {
|
|
205
|
+
entry.pagination = {
|
|
206
|
+
type: r.pagination.type,
|
|
207
|
+
cursorParam: r.pagination.cursor_param,
|
|
208
|
+
cursorField: r.pagination.cursor_field,
|
|
209
|
+
hasMoreField: r.pagination.has_more_field,
|
|
210
|
+
limitParam: r.pagination.limit_param,
|
|
211
|
+
defaultLimit: r.pagination.default_limit,
|
|
212
|
+
itemsField: r.pagination.items_field,
|
|
213
|
+
pageParam: r.pagination.page_param,
|
|
214
|
+
startPage: r.pagination.start_page,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
if (r.lifecycle) {
|
|
218
|
+
entry.lifecycle = {
|
|
219
|
+
field: r.lifecycle.field,
|
|
220
|
+
states: r.lifecycle.states,
|
|
221
|
+
transitions: r.lifecycle.transitions,
|
|
222
|
+
actions: Object.fromEntries(
|
|
223
|
+
Object.entries(r.lifecycle.actions).map(([name, a]) => [name, {
|
|
224
|
+
endpoint: a.endpoint,
|
|
225
|
+
expectedState: a.expected_state,
|
|
226
|
+
body: a.body,
|
|
227
|
+
}]),
|
|
228
|
+
),
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
out.set(r.resource, entry);
|
|
232
|
+
}
|
|
233
|
+
return out.size > 0 ? out : undefined;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
type ResourceConfigEntry = {
|
|
237
|
+
readbackDiff?: ReadbackDiffConfig;
|
|
238
|
+
idempotency?: IdempotencyConfig;
|
|
239
|
+
pagination?: PaginationConfig;
|
|
240
|
+
lifecycle?: LifecycleConfig;
|
|
241
|
+
seedBody?: SeedBodyConfig;
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
async function derivePathVarsFromApi(apiName: string | undefined, dbPath: string | undefined): Promise<Record<string, string>> {
|
|
245
|
+
if (!apiName) return {};
|
|
246
|
+
const col = resolveApiCollection(apiName, dbPath);
|
|
247
|
+
if ("error" in col || !col.baseDir) return {};
|
|
248
|
+
try {
|
|
249
|
+
const env = await loadEnvironment(undefined, col.baseDir);
|
|
250
|
+
const out: Record<string, string> = {};
|
|
251
|
+
for (const [k, v] of Object.entries(env)) {
|
|
252
|
+
if (typeof v !== "string" || v.length === 0) continue;
|
|
253
|
+
if (k === "base_url" || k === "auth_token" || k === "api_key") continue;
|
|
254
|
+
const trimmed = v.trim().toLowerCase();
|
|
255
|
+
// Mirror prepare-fixtures' placeholder filter — a "string"/"example"
|
|
256
|
+
// value would routinely 404 and undo the whole reactivity point.
|
|
257
|
+
if (trimmed === "" || trimmed === "string" || trimmed === "example") continue;
|
|
258
|
+
if (trimmed.startsWith("todo") || trimmed.startsWith("<")) continue;
|
|
259
|
+
out[k] = v;
|
|
260
|
+
}
|
|
261
|
+
return out;
|
|
262
|
+
} catch {
|
|
263
|
+
return {};
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** ARV-324: load `.fixture-gaps.yaml` (written by a prior prepare-fixtures/
|
|
268
|
+
* discover run) so findings on a known-empty/inaccessible operation get
|
|
269
|
+
* `fix_fixture` instead of `report_backend_bug`. Undefined (not an empty
|
|
270
|
+
* Set) when there's no API context or no gaps file, so the classifier
|
|
271
|
+
* can tell "nothing to check" apart from "checked, no gaps". */
|
|
272
|
+
async function deriveFixtureGapsFromApi(apiName: string | undefined, dbPath: string | undefined): Promise<Set<string> | undefined> {
|
|
273
|
+
if (!apiName) return undefined;
|
|
274
|
+
const col = resolveApiCollection(apiName, dbPath);
|
|
275
|
+
if ("error" in col || !col.baseDir) return undefined;
|
|
276
|
+
const gaps = await readFixtureGaps(col.baseDir);
|
|
277
|
+
return gaps.length > 0 ? gapIndex(gaps) : undefined;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async function deriveAuthHeadersFromApi(apiName: string | undefined, dbPath: string | undefined): Promise<Record<string, string>> {
|
|
281
|
+
if (!apiName) return {};
|
|
282
|
+
const col = resolveApiCollection(apiName, dbPath);
|
|
283
|
+
if ("error" in col || !col.baseDir) return {};
|
|
284
|
+
try {
|
|
285
|
+
const env = await loadEnvironment(undefined, col.baseDir);
|
|
286
|
+
const out: Record<string, string> = {};
|
|
287
|
+
if (typeof env.auth_token === "string" && env.auth_token.length > 0) {
|
|
288
|
+
out["Authorization"] = `Bearer ${env.auth_token}`;
|
|
289
|
+
}
|
|
290
|
+
if (typeof env.api_key === "string" && env.api_key.length > 0) {
|
|
291
|
+
out["X-API-Key"] = env.api_key;
|
|
292
|
+
}
|
|
293
|
+
return out;
|
|
294
|
+
} catch {
|
|
295
|
+
return {};
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* ARV-26: render the per-(check, reason) skip tally as a single line so
|
|
301
|
+
* "0 findings" doesn't read as "all green" when half the probes never got
|
|
302
|
+
* a checkable response (e.g. no auth → 4xx → no schema on that branch).
|
|
303
|
+
*
|
|
304
|
+
* Top-3 reasons are inlined; if more exist, append "; +N more". Returns
|
|
305
|
+
* empty string when nothing was skipped.
|
|
306
|
+
*/
|
|
307
|
+
function formatSkippedOutcomes(skipped: Record<string, number> | undefined): string {
|
|
308
|
+
if (!skipped) return "";
|
|
309
|
+
const entries = Object.entries(skipped).sort((a, b) => b[1] - a[1]);
|
|
310
|
+
if (entries.length === 0) return "";
|
|
311
|
+
const total = entries.reduce((acc, [, n]) => acc + n, 0);
|
|
312
|
+
const top = entries.slice(0, 3).map(([k, n]) => `${k} ×${n}`);
|
|
313
|
+
const tail = entries.length > 3 ? `; +${entries.length - 3} more` : "";
|
|
314
|
+
return `(${total} check outcome(s) skipped: ${top.join("; ")}${tail})`;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* ARV-118: minimal markdown render of a `checks run` result. Mirrors the
|
|
319
|
+
* console summary line + a grouped findings list. Kept deliberately small —
|
|
320
|
+
* SARIF / JSON envelope remain the canonical machine-readable artifacts.
|
|
321
|
+
*/
|
|
322
|
+
function renderMarkdownReport(
|
|
323
|
+
data: RunChecksResult["data"],
|
|
324
|
+
warnings: string[],
|
|
325
|
+
): string {
|
|
326
|
+
const s = data.summary;
|
|
327
|
+
const lines: string[] = [];
|
|
328
|
+
lines.push(`# zond checks report`);
|
|
329
|
+
lines.push("");
|
|
330
|
+
lines.push(
|
|
331
|
+
`**${s.findings} finding(s)** across ${s.cases} case(s) on ${s.operations} operation(s) — ${s.checks_run} check(s) active`,
|
|
332
|
+
);
|
|
333
|
+
// ARV-251: per-category roll-up. Small teams use this to triage —
|
|
334
|
+
// "0 security, 12 reliability" is a clear starting point compared to
|
|
335
|
+
// a flat severity pile.
|
|
336
|
+
if (s.findings > 0) {
|
|
337
|
+
const c = s.by_category;
|
|
338
|
+
lines.push("");
|
|
339
|
+
lines.push(
|
|
340
|
+
`🛡 security: ${c.security} · ⚙ reliability: ${c.reliability} · 📜 contract: ${c.contract} · · hygiene: ${c.hygiene}`,
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
const skipLine = formatSkippedOutcomes(s.skipped_outcomes);
|
|
344
|
+
if (skipLine) {
|
|
345
|
+
lines.push("");
|
|
346
|
+
lines.push(skipLine);
|
|
347
|
+
}
|
|
348
|
+
if (warnings.length > 0) {
|
|
349
|
+
lines.push("");
|
|
350
|
+
lines.push(`## Warnings`);
|
|
351
|
+
for (const w of warnings) lines.push(`- ${w}`);
|
|
352
|
+
}
|
|
353
|
+
if (data.findings.length > 0) {
|
|
354
|
+
lines.push("");
|
|
355
|
+
lines.push(`## Findings`);
|
|
356
|
+
for (const f of data.findings) {
|
|
357
|
+
const cat = f.category ? ` _${f.category}_` : "";
|
|
358
|
+
lines.push(
|
|
359
|
+
`- **[${f.severity}]**${cat} \`${f.check}\` ${f.operation.method} ${f.operation.path} — ${f.message}`,
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return lines.join("\n") + "\n";
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function splitList(values: string[] | undefined): string[] | undefined {
|
|
367
|
+
if (!values || values.length === 0) return undefined;
|
|
368
|
+
return values.flatMap((v) => v.split(",")).map((s) => s.trim()).filter(Boolean);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// ARV-211 (R13/F15): expand the `stateful` keyword in --check / --exclude-check
|
|
372
|
+
// into the full set of stateful check ids registered in core/checks/stateful.ts.
|
|
373
|
+
// This lets users (and the zond-checks SKILL.md DEPTH-PASS step) write
|
|
374
|
+
// zond checks run --check stateful
|
|
375
|
+
// instead of hand-listing cross_call_references, idempotency_replay, … —
|
|
376
|
+
// matching the prior `--phase stateful` UX promise without overloading the
|
|
377
|
+
// case-generation `--phase` flag.
|
|
378
|
+
// ARV-325: `ignored_auth` / `open_cors_on_sensitive` live in the stateful
|
|
379
|
+
// *registry* (they need the stateful harness to run), but semantically they
|
|
380
|
+
// are auth/security checks. Users reading `--check stateful` expect
|
|
381
|
+
// state-machine probes, not a full security pass — on Stripe the pair added
|
|
382
|
+
// ~520 extra cases and turned a sub-minute run into ~10 minutes. Keep them
|
|
383
|
+
// runnable by explicit id; just don't smuggle them in through the alias.
|
|
384
|
+
const STATEFUL_ALIAS_EXCLUDED: ReadonlySet<string> = new Set([
|
|
385
|
+
"ignored_auth",
|
|
386
|
+
"open_cors_on_sensitive",
|
|
387
|
+
]);
|
|
388
|
+
|
|
389
|
+
export function expandStatefulAlias(ids: string[] | undefined): string[] | undefined {
|
|
390
|
+
if (!ids) return ids;
|
|
391
|
+
const statefulIds = listStatefulChecks()
|
|
392
|
+
.map((c) => c.id)
|
|
393
|
+
.filter((id) => !STATEFUL_ALIAS_EXCLUDED.has(id));
|
|
394
|
+
const out: string[] = [];
|
|
395
|
+
for (const id of ids) {
|
|
396
|
+
if (id === "stateful") out.push(...statefulIds);
|
|
397
|
+
else out.push(id);
|
|
398
|
+
}
|
|
399
|
+
return out;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function resolveBaseUrl(
|
|
403
|
+
apiName: string | undefined,
|
|
404
|
+
baseUrlFlag: string | undefined,
|
|
405
|
+
dbPath: string | undefined,
|
|
406
|
+
): Promise<{ baseUrl: string } | { error: string }> {
|
|
407
|
+
if (typeof baseUrlFlag === "string" && baseUrlFlag.length > 0) {
|
|
408
|
+
return { baseUrl: baseUrlFlag };
|
|
409
|
+
}
|
|
410
|
+
// ARV-53: caller already resolved --api through cli/util/api-context.ts;
|
|
411
|
+
// we only widen here when nothing reached us (allows --base-url to be the
|
|
412
|
+
// sole input). Keep readCurrentApi() inline-free.
|
|
413
|
+
const effectiveApi = apiName;
|
|
414
|
+
if (!effectiveApi) {
|
|
415
|
+
return { error: "Need --base-url <url> (or --api <name> with base_url in apis/<name>/.env.yaml)" };
|
|
416
|
+
}
|
|
417
|
+
const col = resolveApiCollection(effectiveApi, dbPath);
|
|
418
|
+
if ("error" in col) return col;
|
|
419
|
+
if (!col.baseDir) {
|
|
420
|
+
return { error: `API '${effectiveApi}' has no base_dir registered — pass --base-url <url>` };
|
|
421
|
+
}
|
|
422
|
+
const env = await loadEnvironment(undefined, col.baseDir);
|
|
423
|
+
const v = env.base_url;
|
|
424
|
+
if (typeof v !== "string" || v.length === 0) {
|
|
425
|
+
return { error: `base_url not set in ${col.baseDir}/.env.yaml — pass --base-url <url>` };
|
|
426
|
+
}
|
|
427
|
+
return { baseUrl: v };
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
async function checksRunAction(_args: unknown, cmd: Command): Promise<void> {
|
|
431
|
+
const opts = cmd.opts<ChecksRunOptions>();
|
|
432
|
+
const json = opts.json === true || globalJson(cmd);
|
|
433
|
+
|
|
434
|
+
// ARV-118 (m-19): single source of truth for `--report` / `--output` /
|
|
435
|
+
// `--json` resolution. resolveOutput enforces mutual exclusion of
|
|
436
|
+
// `--json` and `--report`, validates the format name against the spec
|
|
437
|
+
// (ARV-97 — no silent acceptance), and computes the channel + path.
|
|
438
|
+
// The legacy `--ndjson` boolean and the inline alias rewrite are gone —
|
|
439
|
+
// `--report ndjson` is now a first-class format key.
|
|
440
|
+
let resolved: ResolvedOutput;
|
|
441
|
+
try {
|
|
442
|
+
resolved = resolveOutput(CHECKS_OUTPUT_SPEC, {
|
|
443
|
+
report: opts.report,
|
|
444
|
+
output: opts.output,
|
|
445
|
+
json,
|
|
446
|
+
});
|
|
447
|
+
} catch (err) {
|
|
448
|
+
if (err instanceof OutputSpecError) {
|
|
449
|
+
if (json) printJson(jsonError("checks run", [err.message]));
|
|
450
|
+
else printError(err.message);
|
|
451
|
+
process.exit(2);
|
|
452
|
+
}
|
|
453
|
+
throw err;
|
|
454
|
+
}
|
|
455
|
+
const ndjson = resolved.format === "ndjson";
|
|
456
|
+
|
|
457
|
+
// ARV-53: one resolver for --api across all of `checks run`'s sub-lookups
|
|
458
|
+
// (spec, base_url, auth-header derivation).
|
|
459
|
+
const apiName = getApi(cmd, opts as unknown as Record<string, unknown>);
|
|
460
|
+
const specRes = resolveSpecArg(opts.spec, apiName, opts.db);
|
|
461
|
+
if ("error" in specRes) {
|
|
462
|
+
if (json) printJson(jsonError("checks run", [specRes.error]));
|
|
463
|
+
else printError(specRes.error);
|
|
464
|
+
process.exit(2);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
const baseRes = await resolveBaseUrl(apiName, opts.baseUrl, opts.db);
|
|
468
|
+
if ("error" in baseRes) {
|
|
469
|
+
if (json) printJson(jsonError("checks run", [baseRes.error]));
|
|
470
|
+
else printError(baseRes.error);
|
|
471
|
+
process.exit(2);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// ARV-3: lift auth headers from --auth-header (wins) and/or the
|
|
475
|
+
// resolved --api's .env.yaml (auth_token / api_key conventions).
|
|
476
|
+
const fromEnv = await deriveAuthHeadersFromApi(apiName, opts.db);
|
|
477
|
+
const fromFlags = parseAuthHeaders(opts.authHeader);
|
|
478
|
+
const authHeaders = { ...fromEnv, ...fromFlags };
|
|
479
|
+
|
|
480
|
+
// ARV-141: feed filled fixtures into path-params so the run reacts to
|
|
481
|
+
// fixture-pack growth (otherwise findings/skips are pixel-identical across
|
|
482
|
+
// rounds and CI can't distinguish "spec stable" from "checks ignored deltas").
|
|
483
|
+
const pathVars = await derivePathVarsFromApi(apiName, opts.db);
|
|
484
|
+
const resourceConfigs = await deriveResourceConfigsFromApi(apiName, opts.db);
|
|
485
|
+
const fixtureGaps = await deriveFixtureGapsFromApi(apiName, opts.db);
|
|
486
|
+
|
|
487
|
+
const phaseRaw = typeof opts.phase === "string" ? opts.phase : "examples";
|
|
488
|
+
if (phaseRaw !== "examples" && phaseRaw !== "coverage" && phaseRaw !== "all") {
|
|
489
|
+
// ARV-211: redirect users typing --phase stateful (a common skill drift)
|
|
490
|
+
// to the canonical alias `--check stateful`.
|
|
491
|
+
const hint = phaseRaw === "stateful"
|
|
492
|
+
? " — stateful checks are a separate family; run them with `--check stateful` (or list individual ids)"
|
|
493
|
+
: "";
|
|
494
|
+
const msg = `Unknown --phase: "${phaseRaw}". Available: examples, coverage, all${hint}`;
|
|
495
|
+
if (json) printJson(jsonError("checks run", [msg]));
|
|
496
|
+
else printError(msg);
|
|
497
|
+
process.exit(2);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const modeRaw = typeof opts.mode === "string" ? opts.mode : "all";
|
|
501
|
+
if (modeRaw !== "positive" && modeRaw !== "negative" && modeRaw !== "all") {
|
|
502
|
+
const msg = `Unknown --mode: "${modeRaw}". Available: positive, negative, all`;
|
|
503
|
+
if (json) printJson(jsonError("checks run", [msg]));
|
|
504
|
+
else printError(msg);
|
|
505
|
+
process.exit(2);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// ARV-9: parse the unified --include/--exclude filter specs. Bad
|
|
509
|
+
// specs surface as a friendly multi-line error (not a stack trace) and
|
|
510
|
+
// exit 2 — the same code as other CLI-input failures here.
|
|
511
|
+
const compiled = compileOperationFilter({ includes: opts.include, excludes: opts.exclude });
|
|
512
|
+
if (compiled.errors.length > 0) {
|
|
513
|
+
if (json) printJson(jsonError("checks run", compiled.errors));
|
|
514
|
+
else for (const e of compiled.errors) printError(e);
|
|
515
|
+
process.exit(2);
|
|
516
|
+
}
|
|
517
|
+
const operationFilter = (opts.include?.length || opts.exclude?.length) ? compiled.filter : undefined;
|
|
518
|
+
|
|
519
|
+
// ARV-8: --workers <n|auto>. Errors here are CLI-input failures (exit
|
|
520
|
+
// 2) — we don't want a stack trace for a typo.
|
|
521
|
+
let workers: number;
|
|
522
|
+
try {
|
|
523
|
+
workers = parseWorkers(opts.workers);
|
|
524
|
+
} catch (err) {
|
|
525
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
526
|
+
if (json) printJson(jsonError("checks run", [msg]));
|
|
527
|
+
else printError(msg);
|
|
528
|
+
process.exit(2);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// ARV-8: --rate-limit <rps|auto>. `auto` = adaptive (reacts to
|
|
532
|
+
// RateLimit-* response headers); numeric = fixed RPS budget.
|
|
533
|
+
let rateLimiter: RateLimiter | undefined;
|
|
534
|
+
if (typeof opts.rateLimit === "string" && opts.rateLimit.length > 0) {
|
|
535
|
+
const v = opts.rateLimit.trim().toLowerCase();
|
|
536
|
+
if (v === "auto") {
|
|
537
|
+
rateLimiter = createAdaptiveRateLimiter();
|
|
538
|
+
} else {
|
|
539
|
+
const rps = Number.parseFloat(v);
|
|
540
|
+
if (!Number.isFinite(rps) || rps <= 0) {
|
|
541
|
+
const msg = `Invalid --rate-limit value: "${opts.rateLimit}" (expected positive number or "auto")`;
|
|
542
|
+
if (json) printJson(jsonError("checks run", [msg]));
|
|
543
|
+
else printError(msg);
|
|
544
|
+
process.exit(2);
|
|
545
|
+
}
|
|
546
|
+
rateLimiter = createRateLimiter(rps);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// ARV-97 (F2 / m-19): when `ndjson` lands in the file channel, open a
|
|
551
|
+
// write fd up front and pipe events into it; otherwise emit on stdout.
|
|
552
|
+
// Re-running with the same --output truncates (mirrors SARIF) so a
|
|
553
|
+
// stale artifact can't leak across runs.
|
|
554
|
+
const ndjsonOutputPath: string | undefined = ndjson && resolved.channel === "file" ? resolved.path : undefined;
|
|
555
|
+
let ndjsonFd: number | undefined;
|
|
556
|
+
let ndjsonEventCount = 0;
|
|
557
|
+
let ndjsonLastFullyWritten = true;
|
|
558
|
+
if (ndjsonOutputPath) {
|
|
559
|
+
ndjsonFd = openSync(ndjsonOutputPath, "w");
|
|
560
|
+
}
|
|
561
|
+
// ARV-343: accumulate a running summary from the stream so a SIGTERM'd
|
|
562
|
+
// run still emits a terminal `summary` line (operations/cases counted so
|
|
563
|
+
// far) instead of dying before the runner's emit stage — triage's
|
|
564
|
+
// `sweepWindows`/status-dist read `.summary.operations` and had to
|
|
565
|
+
// reconstruct it from raw NDJSON when the window was killed mid-flight.
|
|
566
|
+
const partialOps = new Set<string>();
|
|
567
|
+
const partialChecks = new Set<string>();
|
|
568
|
+
let partialCases = 0;
|
|
569
|
+
let partialFindings = 0;
|
|
570
|
+
const partialBySeverity = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
|
|
571
|
+
const accumulate = (ev: import("../../core/reporter/ndjson.ts").NdjsonEvent) => {
|
|
572
|
+
if (ev.type === "check_result") {
|
|
573
|
+
partialCases += 1;
|
|
574
|
+
partialChecks.add(ev.check);
|
|
575
|
+
partialOps.add(`${ev.operation.method} ${ev.operation.path}`);
|
|
576
|
+
} else if (ev.type === "check_start") {
|
|
577
|
+
partialOps.add(`${ev.operation.method} ${ev.operation.path}`);
|
|
578
|
+
} else if (ev.type === "finding") {
|
|
579
|
+
partialFindings += 1;
|
|
580
|
+
const sev = ev.finding.severity as keyof typeof partialBySeverity;
|
|
581
|
+
if (sev in partialBySeverity) partialBySeverity[sev] += 1;
|
|
582
|
+
}
|
|
583
|
+
};
|
|
584
|
+
const ndjsonOnEvent = ndjson
|
|
585
|
+
? (ndjsonFd !== undefined
|
|
586
|
+
? (ev: import("../../core/reporter/ndjson.ts").NdjsonEvent) => {
|
|
587
|
+
accumulate(ev);
|
|
588
|
+
ndjsonLastFullyWritten = false;
|
|
589
|
+
ndjsonEventCount += 1;
|
|
590
|
+
writeSync(ndjsonFd!, `${JSON.stringify(ev)}\n`);
|
|
591
|
+
ndjsonLastFullyWritten = true;
|
|
592
|
+
}
|
|
593
|
+
// ARV-323: count stdout-channel events too — the SIGTERM handler
|
|
594
|
+
// reports `ndjsonEventCount`, and with a shell-redirected stdout
|
|
595
|
+
// stream it claimed "0 event(s) flushed" while the redirect target
|
|
596
|
+
// already held thousands of lines.
|
|
597
|
+
: (ev: import("../../core/reporter/ndjson.ts").NdjsonEvent) => {
|
|
598
|
+
accumulate(ev);
|
|
599
|
+
ndjsonEventCount += 1;
|
|
600
|
+
emitToStdout(ev);
|
|
601
|
+
})
|
|
602
|
+
: undefined;
|
|
603
|
+
|
|
604
|
+
// ARV-230: clean SIGTERM/SIGINT shutdown for NDJSON streams. Without
|
|
605
|
+
// this, killing a long `checks run --report ndjson` leaves a truncated
|
|
606
|
+
// last line in the file (or in the consumer's pipe) and downstream
|
|
607
|
+
// `jq -s` chokes until the user strips it with `sed '$d'`. The handler
|
|
608
|
+
// closes the fd (or drains stdout) and exits with the conventional
|
|
609
|
+
// 128+signo code instead of letting Node default-terminate mid-write.
|
|
610
|
+
let removeNdjsonSigHandlers: (() => void) | undefined;
|
|
611
|
+
if (ndjson) {
|
|
612
|
+
const shutdown = (signo: number) => {
|
|
613
|
+
try {
|
|
614
|
+
// ARV-343: flush a partial terminal summary from the running
|
|
615
|
+
// accumulators. Schema-exact (CheckRunSummarySchema) so downstream
|
|
616
|
+
// `jq '.summary.operations'` and status-dist stay analyzable;
|
|
617
|
+
// by_category/skipped are left empty (not derivable in the CLI) —
|
|
618
|
+
// the stderr interrupt note below flags that it's a partial count.
|
|
619
|
+
const partialSummary = {
|
|
620
|
+
type: "summary" as const,
|
|
621
|
+
ts: nowIso(),
|
|
622
|
+
summary: {
|
|
623
|
+
operations: partialOps.size,
|
|
624
|
+
cases: partialCases,
|
|
625
|
+
checks_run: partialChecks.size,
|
|
626
|
+
findings: partialFindings,
|
|
627
|
+
by_severity: { ...partialBySeverity },
|
|
628
|
+
by_category: { security: 0, reliability: 0, contract: 0, hygiene: 0 },
|
|
629
|
+
skipped_outcomes: {},
|
|
630
|
+
skipped_outcomes_grouped: [],
|
|
631
|
+
},
|
|
632
|
+
};
|
|
633
|
+
if (ndjsonFd !== undefined) {
|
|
634
|
+
if (!ndjsonLastFullyWritten) {
|
|
635
|
+
try { writeSync(ndjsonFd, "\n"); } catch { /* fd already gone */ }
|
|
636
|
+
}
|
|
637
|
+
try { writeSync(ndjsonFd, `${JSON.stringify(partialSummary)}\n`); } catch { /* fd already gone */ }
|
|
638
|
+
try { closeSync(ndjsonFd); } catch { /* already closed */ }
|
|
639
|
+
ndjsonFd = undefined;
|
|
640
|
+
} else {
|
|
641
|
+
try { emitToStdout(partialSummary); } catch { /* stdout gone */ }
|
|
642
|
+
}
|
|
643
|
+
} catch { /* swallow; we're tearing down */ }
|
|
644
|
+
// ARV-323: "emitted" not "flushed" — on the stdout channel the last
|
|
645
|
+
// few lines may still sit in the pipe buffer, so the count is a
|
|
646
|
+
// lower bound of what the consumer/file will contain, never an
|
|
647
|
+
// excuse to discard a partial-but-real stream.
|
|
648
|
+
try { process.stderr.write(`zond: NDJSON run interrupted (signal ${signo}); ${ndjsonEventCount} event(s) + a partial summary (${partialOps.size} ops, ${partialCases} cases so far) emitted before interrupt (partial stream is usable).\n`); } catch { /* ignore */ }
|
|
649
|
+
process.exit(128 + signo);
|
|
650
|
+
};
|
|
651
|
+
const onTerm = () => shutdown(15);
|
|
652
|
+
const onInt = () => shutdown(2);
|
|
653
|
+
process.on("SIGTERM", onTerm);
|
|
654
|
+
process.on("SIGINT", onInt);
|
|
655
|
+
removeNdjsonSigHandlers = () => {
|
|
656
|
+
process.off("SIGTERM", onTerm);
|
|
657
|
+
process.off("SIGINT", onInt);
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// ARV-265: accumulate every HTTP case `runChecks` dispatches so we can
|
|
662
|
+
// persist them into the run/results tables after the run completes. The
|
|
663
|
+
// adapter maps ChecksCaseEvent → AuditCaseRecord (1:1) and groups by
|
|
664
|
+
// synthetic suite path (`apis/<api>/checks/<phase>`) so `detectRunKind`
|
|
665
|
+
// would still classify these rows as `check`-kind on legacy queries.
|
|
666
|
+
const auditPersist = checksPersistEnabled();
|
|
667
|
+
const auditCases: AuditCaseRecord[] = [];
|
|
668
|
+
const auditSuiteRoot = `apis/${apiName ?? "_"}/checks`;
|
|
669
|
+
const onCase = auditPersist
|
|
670
|
+
? (ev: ChecksCaseEvent) => {
|
|
671
|
+
const phaseLabel = ev.phase === "response" ? "response" : ev.phase.replace("stateful_", "stateful/");
|
|
672
|
+
const suiteFile = `${auditSuiteRoot}/${phaseLabel}.yaml`;
|
|
673
|
+
const status = ev.verdict === "pass" ? "pass"
|
|
674
|
+
: ev.verdict === "fail" ? "fail"
|
|
675
|
+
: ev.verdict === "skip" ? "skip"
|
|
676
|
+
: "error";
|
|
677
|
+
auditCases.push({
|
|
678
|
+
suiteName: `checks/${phaseLabel}`,
|
|
679
|
+
suiteFile,
|
|
680
|
+
testName: `${ev.checkId}::${ev.operation.method.toUpperCase()} ${ev.operation.path}`,
|
|
681
|
+
status,
|
|
682
|
+
request: ev.request,
|
|
683
|
+
...(ev.response ? { response: ev.response } : {}),
|
|
684
|
+
durationMs: ev.durationMs,
|
|
685
|
+
...(ev.error ? { error: ev.error } : {}),
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
: undefined;
|
|
689
|
+
|
|
690
|
+
let budget: Budget | undefined;
|
|
691
|
+
if (opts.budget !== undefined) {
|
|
692
|
+
if (!isBudget(opts.budget)) {
|
|
693
|
+
printError(`--budget must be one of: ${BUDGETS.join(", ")}; got '${opts.budget}'`);
|
|
694
|
+
process.exit(1);
|
|
695
|
+
}
|
|
696
|
+
budget = opts.budget;
|
|
697
|
+
}
|
|
698
|
+
const includeList = expandStatefulAlias(splitList(opts.check));
|
|
699
|
+
const statefulIds = new Set(listStatefulChecks().map((c) => c.id));
|
|
700
|
+
const includesStateful = includeList?.some((id) => statefulIds.has(id)) === true;
|
|
701
|
+
const budgetResolved = resolveBudget(budget, opts.maxRequests, {
|
|
702
|
+
forceStatefulIfIncluded: includesStateful,
|
|
703
|
+
});
|
|
704
|
+
|
|
705
|
+
// ARV-328: throttled progress line on stderr during long runs (CI jobs
|
|
706
|
+
// and subagents with wall-clock budgets had zero visibility — the only
|
|
707
|
+
// artifact was the growing ndjson file). stderr never corrupts stdout
|
|
708
|
+
// reports; 10s throttle keeps short runs silent.
|
|
709
|
+
const PROGRESS_INTERVAL_MS = 10_000;
|
|
710
|
+
let lastProgressAt = Date.now();
|
|
711
|
+
const onProgress = (p: { done: number; total: number; cases: number }) => {
|
|
712
|
+
const now = Date.now();
|
|
713
|
+
if (now - lastProgressAt < PROGRESS_INTERVAL_MS) return;
|
|
714
|
+
lastProgressAt = now;
|
|
715
|
+
try {
|
|
716
|
+
process.stderr.write(`zond: progress — ${p.done}/${p.total} operations, ${p.cases} case(s) run\n`);
|
|
717
|
+
} catch { /* ignore */ }
|
|
718
|
+
};
|
|
719
|
+
|
|
720
|
+
try {
|
|
721
|
+
const result = await runChecks({
|
|
722
|
+
specPath: specRes.spec,
|
|
723
|
+
baseUrl: baseRes.baseUrl,
|
|
724
|
+
include: includeList,
|
|
725
|
+
exclude: expandStatefulAlias(splitList(opts.excludeCheck)),
|
|
726
|
+
timeoutMs: typeof opts.timeout === "number" ? opts.timeout : undefined,
|
|
727
|
+
authHeaders: Object.keys(authHeaders).length > 0 ? authHeaders : undefined,
|
|
728
|
+
pathVars: Object.keys(pathVars).length > 0 ? pathVars : undefined,
|
|
729
|
+
fixtureGaps,
|
|
730
|
+
resourceConfigs,
|
|
731
|
+
bootstrapCleanupFailed: opts.bootstrapCleanupFailed === true,
|
|
732
|
+
phase: phaseRaw as "examples" | "coverage" | "all",
|
|
733
|
+
allowX00: opts.allowX00 === true,
|
|
734
|
+
strict405: opts.strict405 === true,
|
|
735
|
+
strict401: opts.strict401 === true,
|
|
736
|
+
mode: modeRaw as "positive" | "negative" | "all",
|
|
737
|
+
operationFilter,
|
|
738
|
+
onEvent: ndjsonOnEvent,
|
|
739
|
+
onCase,
|
|
740
|
+
onProgress,
|
|
741
|
+
// ARV-8: bounded concurrency at op-level + optional rate-limiter
|
|
742
|
+
// gating. workers=1 (default) preserves the pre-ARV-8 sequential
|
|
743
|
+
// path inside runPool — same observable behaviour.
|
|
744
|
+
workers,
|
|
745
|
+
rateLimiter,
|
|
746
|
+
maxRequests: budgetResolved.maxRequests,
|
|
747
|
+
skipStateful: budgetResolved.skipStateful,
|
|
748
|
+
maxOps: opts.maxOps,
|
|
749
|
+
skipOps: opts.skipOps,
|
|
750
|
+
safe: !resolveLive(opts),
|
|
751
|
+
});
|
|
752
|
+
|
|
753
|
+
// ARV-265: persist the accumulated cases as a `run_kind='check'` run
|
|
754
|
+
// so `zond coverage --scope audit` can count them. Failures here
|
|
755
|
+
// degrade to a warning — the user's primary command already succeeded.
|
|
756
|
+
if (auditPersist && auditCases.length > 0) {
|
|
757
|
+
try {
|
|
758
|
+
getDb(opts.db);
|
|
759
|
+
const { findCollectionByNameOrId } = await import("../../db/queries.ts");
|
|
760
|
+
const collectionId = apiName ? findCollectionByNameOrId(apiName)?.id : undefined;
|
|
761
|
+
const session = readCurrentSession();
|
|
762
|
+
const runId = beginAuditRun({
|
|
763
|
+
runKind: "check",
|
|
764
|
+
...(collectionId != null ? { collectionId } : {}),
|
|
765
|
+
...(session?.id ? { sessionId: session.id } : {}),
|
|
766
|
+
tags: ["checks", `phase:${phaseRaw}`, `mode:${modeRaw}`],
|
|
767
|
+
});
|
|
768
|
+
finalizeAuditRun(runId, auditCases);
|
|
769
|
+
} catch (err) {
|
|
770
|
+
// Audit persistence is best-effort. Surface the failure on stderr
|
|
771
|
+
// so an agent can detect it (the missing audit-coverage downstream
|
|
772
|
+
// will already point them here) without breaking the run.
|
|
773
|
+
const msg = (err as Error).message;
|
|
774
|
+
process.stderr.write(`zond: audit persistence failed (${msg}). Re-run with ZOND_CHECKS_PERSIST=0 to silence.\n`);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
const warnings: string[] = [];
|
|
778
|
+
for (const id of result.selection.unknown) {
|
|
779
|
+
warnings.push(`Unknown check: "${id}" — ignored. Run \`zond checks list\` to see registered ids.`);
|
|
780
|
+
}
|
|
781
|
+
// ARV-5 / ARV-118: SARIF v2.1.0 sidecar — file channel always (default
|
|
782
|
+
// filename `zond-checks.sarif` is set by the OutputSpec). Written before
|
|
783
|
+
// any text output so a partial-write failure surfaces before the success
|
|
784
|
+
// line and the exit code.
|
|
785
|
+
if (resolved.format === "sarif") {
|
|
786
|
+
const out = resolved.path!;
|
|
787
|
+
const absSpec = resolvePath(specRes.spec);
|
|
788
|
+
const specContent = readFileSync(absSpec, "utf8");
|
|
789
|
+
// Make spec uri repo-relative when possible — GitHub Code Scanning
|
|
790
|
+
// links findings to a file in the repo, absolute paths break that.
|
|
791
|
+
const specUri = relativePath(process.cwd(), absSpec) || "spec.json";
|
|
792
|
+
const sarif = generateSarifReport({
|
|
793
|
+
findings: result.data.findings,
|
|
794
|
+
specContent,
|
|
795
|
+
specUri,
|
|
796
|
+
toolVersion: VERSION,
|
|
797
|
+
});
|
|
798
|
+
writeFileSync(out, JSON.stringify(sarif, null, 2));
|
|
799
|
+
console.error(`SARIF report written to ${out}`);
|
|
800
|
+
for (const w of warnings) console.error(w);
|
|
801
|
+
} else if (resolved.format === "markdown") {
|
|
802
|
+
const body = renderMarkdownReport(result.data, warnings);
|
|
803
|
+
if (resolved.channel === "file") {
|
|
804
|
+
writeFileSync(resolved.path!, body);
|
|
805
|
+
console.error(`Markdown report written to ${resolved.path}`);
|
|
806
|
+
} else {
|
|
807
|
+
process.stdout.write(body);
|
|
808
|
+
}
|
|
809
|
+
} else if (resolved.format === "json") {
|
|
810
|
+
printJson(jsonOk("checks run", result.data, warnings.length > 0 ? warnings : undefined));
|
|
811
|
+
} else if (ndjson) {
|
|
812
|
+
// ARV-10: stdout already carries the NDJSON stream (events were
|
|
813
|
+
// flushed inside runChecks via onEvent). Warnings ride on stderr
|
|
814
|
+
// so a `| jq` consumer never sees them; the human one-liner is
|
|
815
|
+
// also routed to stderr to keep stdout discipline (AC #5).
|
|
816
|
+
// ARV-97: when events were redirected to a file via --output, the
|
|
817
|
+
// stdout-discipline rationale doesn't apply, but routing the summary
|
|
818
|
+
// to stderr keeps the contract uniform across the two ndjson modes.
|
|
819
|
+
for (const w of warnings) console.error(w);
|
|
820
|
+
const s = result.data.summary;
|
|
821
|
+
console.error(
|
|
822
|
+
`${s.findings} finding(s) across ${s.cases} case(s) on ${s.operations} operation(s) — ${s.checks_run} check(s) active`,
|
|
823
|
+
);
|
|
824
|
+
const skipLine = formatSkippedOutcomes(s.skipped_outcomes);
|
|
825
|
+
if (skipLine) console.error(skipLine);
|
|
826
|
+
if (ndjsonOutputPath) {
|
|
827
|
+
// Mirror the SARIF branch's "written to" line. Use process.stderr
|
|
828
|
+
// directly (not console.error) so test harnesses that mock the
|
|
829
|
+
// streams without intercepting console pick this up.
|
|
830
|
+
process.stderr.write(`NDJSON report written to ${ndjsonOutputPath} (${ndjsonEventCount} events)\n`);
|
|
831
|
+
}
|
|
832
|
+
} else {
|
|
833
|
+
for (const w of warnings) console.error(w);
|
|
834
|
+
const s = result.data.summary;
|
|
835
|
+
printSuccess(
|
|
836
|
+
`${s.findings} finding(s) across ${s.cases} case(s) on ${s.operations} operation(s) — ${s.checks_run} check(s) active`,
|
|
837
|
+
);
|
|
838
|
+
// ARV-251: per-category roll-up. Surfaces "0 security, 12
|
|
839
|
+
// reliability" so a triager sees where the volume sits before
|
|
840
|
+
// scrolling the finding list.
|
|
841
|
+
if (s.findings > 0) {
|
|
842
|
+
const c = s.by_category;
|
|
843
|
+
console.log(
|
|
844
|
+
` 🛡 security: ${c.security} ⚙ reliability: ${c.reliability} 📜 contract: ${c.contract} · hygiene: ${c.hygiene}`,
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
const skipLine = formatSkippedOutcomes(s.skipped_outcomes);
|
|
848
|
+
if (skipLine) console.log(` ${skipLine}`);
|
|
849
|
+
// ARV-60: spec-level rollup. When the runner detected ≥80% of a
|
|
850
|
+
// check's applicable ops sharing one root cause, print one summary
|
|
851
|
+
// row with an actionable fix hint instead of N per-op rows that
|
|
852
|
+
// all say the same thing. `--verbose` always shows per-op detail
|
|
853
|
+
// (and JSON/SARIF carry the full unaggregated list regardless).
|
|
854
|
+
const rolledUpOps = new Set<string>();
|
|
855
|
+
if (!opts.verbose) {
|
|
856
|
+
for (const sf of result.data.spec_findings) {
|
|
857
|
+
if (sf.kind === "status_drift") {
|
|
858
|
+
for (const op of sf.affected_operations) {
|
|
859
|
+
rolledUpOps.add(`${sf.check}|${op.method} ${op.path}`);
|
|
860
|
+
}
|
|
861
|
+
console.log(
|
|
862
|
+
` [${sf.severity}] ${sf.check} — ${sf.reason} (${sf.count}/${sf.applicable} operations)`,
|
|
863
|
+
);
|
|
864
|
+
console.log(` → ${sf.fix_hint}`);
|
|
865
|
+
} else {
|
|
866
|
+
// missing_declaration / no_detector / other — no affected_operations
|
|
867
|
+
// to enumerate; surfaces as a single info row + fix hint.
|
|
868
|
+
const tag = sf.kind === "no_detector"
|
|
869
|
+
? `${sf.count === 0 ? "0 cases" : `${sf.count} cases`} / ${sf.applicable} applicable ops`
|
|
870
|
+
: `${sf.count}/${sf.applicable} cases`;
|
|
871
|
+
console.log(` [${sf.severity}] ${sf.check} — ${sf.reason} (${tag})`);
|
|
872
|
+
console.log(` → ${sf.fix_hint}`);
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
// ARV-283 AC#4: suppressed findings stay in the ndjson audit-trail
|
|
877
|
+
// but are hidden from the human summary unless `--show-suppressed`
|
|
878
|
+
// is passed. They never count toward CI gates regardless.
|
|
879
|
+
const activeFindings = result.data.findings.filter((f) => !f.suppressed_by);
|
|
880
|
+
const suppressedFindings = result.data.findings.filter((f) => f.suppressed_by);
|
|
881
|
+
|
|
882
|
+
// Per-op findings — skip those already covered by a status_drift
|
|
883
|
+
// rollup unless --verbose was passed.
|
|
884
|
+
if (opts.verbose) {
|
|
885
|
+
for (const f of activeFindings) {
|
|
886
|
+
console.log(` [${f.severity}] ${f.check} ${f.operation.method} ${f.operation.path} — ${f.message}`);
|
|
887
|
+
}
|
|
888
|
+
} else {
|
|
889
|
+
// ARV-18: dedup identical findings on the SAME op (multiple cases
|
|
890
|
+
// hitting the same gap) before printing. Spec-rollup above already
|
|
891
|
+
// handled the across-op clusters; this collapses the within-op
|
|
892
|
+
// duplicates so the human summary stays readable even when boundary
|
|
893
|
+
// mutations each trip the same status.
|
|
894
|
+
const seen = new Set<string>();
|
|
895
|
+
for (const f of activeFindings) {
|
|
896
|
+
const opKey = `${f.check}|${f.operation.method} ${f.operation.path}`;
|
|
897
|
+
if (rolledUpOps.has(opKey)) continue;
|
|
898
|
+
const dedupKey = `${f.severity}|${opKey}|${f.response_summary?.status ?? 0}|${f.message}`;
|
|
899
|
+
if (seen.has(dedupKey)) continue;
|
|
900
|
+
seen.add(dedupKey);
|
|
901
|
+
console.log(` [${f.severity}] ${f.check} ${f.operation.method} ${f.operation.path} — ${f.message}`);
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
if (opts.showSuppressed && suppressedFindings.length > 0) {
|
|
906
|
+
console.log(`\nSuppressed (${suppressedFindings.length}, not counted in summary):`);
|
|
907
|
+
for (const f of suppressedFindings) {
|
|
908
|
+
const sb = f.suppressed_by!;
|
|
909
|
+
console.log(` [${f.severity}] ${f.check} ${f.operation.method} ${f.operation.path} — ${f.message}`);
|
|
910
|
+
console.log(` ↳ suppressed by ${sb.source}#${sb.rule_index}: ${sb.reason}`);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
// Exit-code rule: 0 when no HIGH/CRITICAL findings, 1 otherwise. LOW/MEDIUM
|
|
915
|
+
// findings are reported but don't gate CI by default — agents that want
|
|
916
|
+
// strict gating can post-process the JSON envelope.
|
|
917
|
+
//
|
|
918
|
+
// ARV-308: --no-fail-on-findings / --advisory keeps exit 0 even with
|
|
919
|
+
// HIGH/CRITICAL findings, mirroring `zond run --no-fail-on-failures`, so
|
|
920
|
+
// an orchestrator can tell "found drift" (exit 0, findings in envelope)
|
|
921
|
+
// from "command failed" (exit 2). The stderr tail still names the count.
|
|
922
|
+
const advisory = opts.advisory === true || opts.failOnFindings === false;
|
|
923
|
+
// ARV-320: this used to skip on ndjson under the assumption that "stderr
|
|
924
|
+
// already carried the summary just above" — false for ndjson, which takes
|
|
925
|
+
// a separate branch that only ever writes "NDJSON report written to
|
|
926
|
+
// <path>" to stderr. Under `--report ndjson` + `set -e` in CI, that made
|
|
927
|
+
// exit 1 look unexplained (report-zond friction on the 2026-07-02 Stripe
|
|
928
|
+
// run: "this step will 'fail' silently with valid data sitting right
|
|
929
|
+
// there"). Always write the reason to stderr — it's stderr, not stdout,
|
|
930
|
+
// so ndjson's stdout-discipline (AC#5) is untouched.
|
|
931
|
+
if (result.high_or_critical > 0) {
|
|
932
|
+
const suffix = advisory
|
|
933
|
+
? " — advisory mode, exiting 0 (findings are in the envelope)"
|
|
934
|
+
: " — exiting with code 1 (pass --no-fail-on-findings / --advisory to suppress, e.g. for advisory runs)";
|
|
935
|
+
process.stderr.write(
|
|
936
|
+
`zond: ${result.high_or_critical} HIGH/CRITICAL finding(s)${suffix}.\n`,
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
if (ndjsonFd !== undefined) closeSync(ndjsonFd);
|
|
940
|
+
removeNdjsonSigHandlers?.();
|
|
941
|
+
process.exit(result.high_or_critical > 0 && !advisory ? 1 : 0);
|
|
942
|
+
} catch (err) {
|
|
943
|
+
if (ndjsonFd !== undefined) {
|
|
944
|
+
try { closeSync(ndjsonFd); } catch { /* fd may already be invalid */ }
|
|
945
|
+
}
|
|
946
|
+
removeNdjsonSigHandlers?.();
|
|
947
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
948
|
+
if (json) printJson(jsonError("checks run", [msg]));
|
|
949
|
+
else printError(msg);
|
|
950
|
+
process.exit(2);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
function defineList(parent: Command): void {
|
|
955
|
+
parent
|
|
956
|
+
.command("list")
|
|
957
|
+
.description("List all registered checks (id, severity, default expected, references)")
|
|
958
|
+
.action(checksListAction);
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
function defineRun(parent: Command): void {
|
|
962
|
+
parent
|
|
963
|
+
.command("run")
|
|
964
|
+
.description("Run active checks against a live API and emit findings")
|
|
965
|
+
.option("--api <name>", "Use the registered API's spec + .env.yaml")
|
|
966
|
+
.option("--spec <path>", "Explicit OpenAPI spec path (overrides --api)")
|
|
967
|
+
.option("--base-url <url>", "Base URL for requests (overrides --api env file)")
|
|
968
|
+
.option("--check <ids...>", "Only run these checks (comma-separated or repeated). 'stateful' expands to the state-machine set: use_after_free, ensure_resource_availability, cross_call_references, idempotency_replay, pagination_invariants, lifecycle_transitions, cursor_boundary_fuzzing — NOT ignored_auth/open_cors_on_sensitive (run those by explicit id)")
|
|
969
|
+
.option("--exclude-check <ids...>", "Skip these checks (comma-separated or repeated)")
|
|
970
|
+
.option("--timeout <ms>", "Per-request timeout in ms", (v) => Number.parseInt(v, 10))
|
|
971
|
+
.option("--db <path>", "SQLite path (for --api lookup)")
|
|
972
|
+
.option(
|
|
973
|
+
"--auth-header <header...>",
|
|
974
|
+
"ARV-3: feed real-auth headers into stateful security checks. Format: 'Name: value'. Repeat for multiple headers. Auto-derived from apis/<name>/.env.yaml (auth_token, api_key) when --api is set.",
|
|
975
|
+
)
|
|
976
|
+
.option(
|
|
977
|
+
"--bootstrap-cleanup-failed",
|
|
978
|
+
"ARV-3: signal that bootstrap-cleanup failed before this run. Stateful security checks (ignored_auth, use_after_free, ensure_resource_availability) skip with a warning to avoid false positives on stale data.",
|
|
979
|
+
)
|
|
980
|
+
.option(
|
|
981
|
+
"--report <format>",
|
|
982
|
+
"ARV-118: output format. Available: console (default — human summary), json (envelope; equivalent to --json), ndjson (stream events on stdout — check_start | check_result | finding | summary), sarif (SARIF v2.1.0 for GitHub Code Scanning), markdown (short summary).",
|
|
983
|
+
)
|
|
984
|
+
.option(
|
|
985
|
+
"--output <path>",
|
|
986
|
+
"ARV-118: write the report to this file. With --report sarif, defaults to zond-checks.sarif when omitted. With --report ndjson, redirects the event stream from stdout into the file (one JSON event per line).",
|
|
987
|
+
)
|
|
988
|
+
.option(
|
|
989
|
+
"--phase <phase>",
|
|
990
|
+
"ARV-6: which case-generation phase to run. examples (default — one positive + single-site negative mutation), coverage (deterministic boundary-value enumeration), all (both).",
|
|
991
|
+
"examples",
|
|
992
|
+
)
|
|
993
|
+
.option(
|
|
994
|
+
"--allow-x00",
|
|
995
|
+
"ARV-6: include the NUL byte (\\x00) in string boundaries during coverage phase. Off by default — some HTTP/JSON stacks panic on it.",
|
|
996
|
+
)
|
|
997
|
+
.option(
|
|
998
|
+
"--mode <mode>",
|
|
999
|
+
"ARV-7: positive (contract verification only — drops checks/cases that send malicious input), negative (only malicious-input probes), all (default — both).",
|
|
1000
|
+
"all",
|
|
1001
|
+
)
|
|
1002
|
+
.option(
|
|
1003
|
+
"--include <spec...>",
|
|
1004
|
+
"ARV-9: keep only operations matching <selector>:<value>. Selectors: path:<regex>, method:<csv>, tag:<csv>, operation-id:<regex>. Repeat the flag for OR semantics.",
|
|
1005
|
+
)
|
|
1006
|
+
.option(
|
|
1007
|
+
"--exclude <spec...>",
|
|
1008
|
+
"ARV-9: drop operations matching <selector>:<value>. Same grammar as --include. Excludes evaluated after includes.",
|
|
1009
|
+
)
|
|
1010
|
+
.option(
|
|
1011
|
+
"--workers <n>",
|
|
1012
|
+
"ARV-8: bounded concurrency at the operation level. <n> = positive integer (clamped 1..64) or `auto` (= min(cpus, 8)). Default 1 (sequential, byte-for-byte the pre-ARV-8 behaviour). Cases inside one operation always run sequentially regardless — only ops are parallelized.",
|
|
1013
|
+
)
|
|
1014
|
+
.option(
|
|
1015
|
+
"--rate-limit <rps>",
|
|
1016
|
+
"ARV-8: cap outbound RPS — positive number (fixed budget) or `auto` (adaptive — paces from RateLimit-* response headers, RFC 9568). Combined with --workers, the limiter gates *all* workers globally so N workers never exceed <rps>.",
|
|
1017
|
+
)
|
|
1018
|
+
.option(
|
|
1019
|
+
"--verbose",
|
|
1020
|
+
"ARV-18: emit one stdout row per finding instead of aggregating identical findings (same check + same response status). JSON / NDJSON / SARIF outputs always carry the unaggregated list; this flag only controls the human summary.",
|
|
1021
|
+
)
|
|
1022
|
+
.option(
|
|
1023
|
+
"--strict-405",
|
|
1024
|
+
"ARV-179: require exactly 405 for `unsupported_method` (mirrors schemathesis V4 default). Off by default — zond's pragmatic policy also accepts 401/403/404 as valid rejections of an undeclared method.",
|
|
1025
|
+
)
|
|
1026
|
+
.option(
|
|
1027
|
+
"--strict-401",
|
|
1028
|
+
"ARV-181: require exactly 401 for `ignored_auth` no-auth / bogus-auth probes (mirrors schemathesis V4). Off by default — zond's pragmatic policy accepts any 4xx as a valid auth-reject.",
|
|
1029
|
+
)
|
|
1030
|
+
.option(
|
|
1031
|
+
"--max-requests <n>",
|
|
1032
|
+
"ARV-227: hard cap on outbound HTTP requests for the whole run (per-response + stateful share the same budget). Once reached, remaining cases short-circuit with `max-requests-cap-reached` in summary.skipped_outcomes. Always wins over the --budget tier cap.",
|
|
1033
|
+
(v) => Number.parseInt(v, 10),
|
|
1034
|
+
)
|
|
1035
|
+
.option(
|
|
1036
|
+
"--max-ops <n>",
|
|
1037
|
+
"ARV-342: cap this run to N operations (deterministic op-window). Pair with --skip-ops to sweep a large spec in bounded, resumable slices that each finish in a short budget. A window whose summary.operations < N is the last slice.",
|
|
1038
|
+
(v) => Number.parseInt(v, 10),
|
|
1039
|
+
)
|
|
1040
|
+
.option(
|
|
1041
|
+
"--skip-ops <n>",
|
|
1042
|
+
"ARV-342: skip the first N operations (post-filter, deterministic order) before applying --max-ops. Resume token for windowed sweeps: skip 0, skip 50, skip 100, …",
|
|
1043
|
+
(v) => Number.parseInt(v, 10),
|
|
1044
|
+
)
|
|
1045
|
+
.option(
|
|
1046
|
+
"--budget <tier>",
|
|
1047
|
+
"ARV-292: adaptive cap and stateful gating tier. `quick` (cap 50, skip stateful) → ~60-sec gate. `standard` (cap 500, all checks). `full` (uncapped). Omitted ⇒ legacy uncapped behaviour. --max-requests always overrides the tier cap; `--check stateful` opts back into stateful even under `quick`.",
|
|
1048
|
+
)
|
|
1049
|
+
.option(
|
|
1050
|
+
"--show-suppressed",
|
|
1051
|
+
"Show findings suppressed by the deterministic broken-baseline guard in the text summary (with their suppressed_by trace). Suppressed findings stay in the ndjson/JSON audit-trail regardless of this flag and never count toward CI exit codes.",
|
|
1052
|
+
)
|
|
1053
|
+
.option(
|
|
1054
|
+
"--no-fail-on-findings",
|
|
1055
|
+
"ARV-308: keep exit code 0 even when HIGH/CRITICAL findings exist (advisory runs). Mirrors `zond run --no-fail-on-failures`. Lets an orchestrator distinguish 'found drift' (exit 0) from 'command failed' (exit 2). Default: exit 1 on any HIGH/CRITICAL finding.",
|
|
1056
|
+
)
|
|
1057
|
+
.option(
|
|
1058
|
+
"--advisory",
|
|
1059
|
+
"ARV-308: alias for --no-fail-on-findings.",
|
|
1060
|
+
)
|
|
1061
|
+
.option("--safe", SAFE_HELP)
|
|
1062
|
+
.option("--live", LIVE_HELP)
|
|
1063
|
+
.action(checksRunAction);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
export function registerChecks(program: Command): void {
|
|
1067
|
+
const cmd = program
|
|
1068
|
+
.command("checks")
|
|
1069
|
+
.description("Run schemathesis-style conformance/security checks against an API");
|
|
1070
|
+
defineList(cmd);
|
|
1071
|
+
defineRun(cmd);
|
|
1072
|
+
}
|