@kirrosh/zond 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/CHANGELOG.md +648 -0
  2. package/README.md +58 -6
  3. package/package.json +9 -6
  4. package/src/cli/argv.ts +122 -0
  5. package/src/cli/commands/add-api.ts +134 -0
  6. package/src/cli/commands/api/annotate/idempotency.ts +59 -0
  7. package/src/cli/commands/api/annotate/index.ts +525 -0
  8. package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
  9. package/src/cli/commands/api/annotate/overlay.ts +206 -0
  10. package/src/cli/commands/api/annotate/pagination.ts +60 -0
  11. package/src/cli/commands/api/annotate/prompts.ts +183 -0
  12. package/src/cli/commands/api/annotate/readback.ts +58 -0
  13. package/src/cli/commands/api/annotate/resources.ts +91 -0
  14. package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
  15. package/src/cli/commands/audit.ts +480 -0
  16. package/src/cli/commands/bootstrap.ts +710 -0
  17. package/src/cli/commands/catalog.ts +35 -0
  18. package/src/cli/commands/check.ts +348 -0
  19. package/src/cli/commands/checks.ts +756 -0
  20. package/src/cli/commands/ci-init.ts +43 -0
  21. package/src/cli/commands/clean.ts +212 -0
  22. package/src/cli/commands/cleanup.ts +262 -0
  23. package/src/cli/commands/completions.ts +16 -0
  24. package/src/cli/commands/coverage.ts +605 -132
  25. package/src/cli/commands/db.ts +178 -7
  26. package/src/cli/commands/describe.ts +37 -2
  27. package/src/cli/commands/discover.ts +1236 -0
  28. package/src/cli/commands/doctor.ts +607 -0
  29. package/src/cli/commands/fixtures.ts +402 -0
  30. package/src/cli/commands/generate.ts +420 -46
  31. package/src/cli/commands/init/bootstrap.ts +30 -1
  32. package/src/cli/commands/{init.ts → init/index.ts} +99 -5
  33. package/src/cli/commands/init/skills.ts +56 -3
  34. package/src/cli/commands/init/templates/agents.md +65 -61
  35. package/src/cli/commands/init/templates/skills/zond-checks.md +397 -0
  36. package/src/cli/commands/init/templates/skills/zond-triage.md +210 -0
  37. package/src/cli/commands/init/templates/skills/zond.md +592 -125
  38. package/src/cli/commands/init/templates/zond-config.yml +8 -9
  39. package/src/cli/commands/prepare-fixtures.ts +135 -0
  40. package/src/cli/commands/probe/mass-assignment.ts +503 -0
  41. package/src/cli/commands/probe/security.ts +454 -0
  42. package/src/cli/commands/probe/static.ts +255 -0
  43. package/src/cli/commands/probe/webhooks.ts +161 -0
  44. package/src/cli/commands/probe.ts +459 -0
  45. package/src/cli/commands/reference.ts +87 -0
  46. package/src/cli/commands/refresh-api.ts +169 -0
  47. package/src/cli/commands/remove-api.ts +150 -0
  48. package/src/cli/commands/report-bundle.ts +318 -0
  49. package/src/cli/commands/report.ts +241 -0
  50. package/src/cli/commands/request.ts +379 -4
  51. package/src/cli/commands/run.ts +842 -53
  52. package/src/cli/commands/session.ts +244 -0
  53. package/src/cli/commands/use.ts +18 -1
  54. package/src/cli/index.ts +20 -3
  55. package/src/cli/json-envelope.ts +112 -3
  56. package/src/cli/json-schemas.ts +263 -0
  57. package/src/cli/program.ts +198 -635
  58. package/src/cli/resolve.ts +105 -0
  59. package/src/cli/status-filter.ts +124 -0
  60. package/src/cli/util/api-context.ts +85 -0
  61. package/src/cli/version.ts +5 -0
  62. package/src/core/anti-fp/bootstrap.ts +34 -0
  63. package/src/core/anti-fp/index.ts +33 -0
  64. package/src/core/anti-fp/registry.ts +44 -0
  65. package/src/core/anti-fp/rules/baseline-echo.ts +74 -0
  66. package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +52 -0
  67. package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +38 -0
  68. package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +35 -0
  69. package/src/core/anti-fp/rules/schemathesis/index.ts +24 -0
  70. package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +53 -0
  71. package/src/core/anti-fp/rules/subscription-gated/index.ts +11 -0
  72. package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +75 -0
  73. package/src/core/anti-fp/types.ts +68 -0
  74. package/src/core/checks/checks/_crud-helpers.ts +133 -0
  75. package/src/core/checks/checks/_negative_mutator.ts +133 -0
  76. package/src/core/checks/checks/_readback-helpers.ts +133 -0
  77. package/src/core/checks/checks/content_type_conformance.ts +39 -0
  78. package/src/core/checks/checks/cross_call_references.ts +134 -0
  79. package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
  80. package/src/core/checks/checks/idempotency_replay.ts +246 -0
  81. package/src/core/checks/checks/ignored_auth.ts +211 -0
  82. package/src/core/checks/checks/index.ts +65 -0
  83. package/src/core/checks/checks/lifecycle_transitions.ts +273 -0
  84. package/src/core/checks/checks/missing_required_header.ts +40 -0
  85. package/src/core/checks/checks/negative_data_rejection.ts +45 -0
  86. package/src/core/checks/checks/not_a_server_error.ts +27 -0
  87. package/src/core/checks/checks/open_cors_on_sensitive.ts +131 -0
  88. package/src/core/checks/checks/pagination_invariants.ts +238 -0
  89. package/src/core/checks/checks/positive_data_acceptance.ts +36 -0
  90. package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
  91. package/src/core/checks/checks/response_headers_conformance.ts +74 -0
  92. package/src/core/checks/checks/response_schema_conformance.ts +30 -0
  93. package/src/core/checks/checks/status_code_conformance.ts +61 -0
  94. package/src/core/checks/checks/unsupported_method.ts +63 -0
  95. package/src/core/checks/checks/use_after_free.ts +78 -0
  96. package/src/core/checks/index.ts +30 -0
  97. package/src/core/checks/mode.ts +79 -0
  98. package/src/core/checks/recommended-action.ts +64 -0
  99. package/src/core/checks/registry.ts +78 -0
  100. package/src/core/checks/runner.ts +874 -0
  101. package/src/core/checks/sarif.ts +230 -0
  102. package/src/core/checks/stateful.ts +121 -0
  103. package/src/core/checks/types.ts +189 -0
  104. package/src/core/classifier/recommended-action.ts +222 -0
  105. package/src/core/context/current.ts +22 -6
  106. package/src/core/context/session.ts +78 -0
  107. package/src/core/coverage/loader.ts +185 -0
  108. package/src/core/coverage/reasons.ts +300 -0
  109. package/src/core/diagnostics/db-analysis.ts +151 -11
  110. package/src/core/diagnostics/failure-class.ts +120 -0
  111. package/src/core/diagnostics/failure-hints.ts +212 -9
  112. package/src/core/diagnostics/spec-pointer.ts +99 -0
  113. package/src/core/diagnostics/suggested-fixes.ts +156 -0
  114. package/src/core/exporter/case-study/index.ts +270 -0
  115. package/src/core/exporter/curl.ts +40 -0
  116. package/src/core/exporter/exporter.ts +48 -0
  117. package/src/core/exporter/html-report/escape.ts +24 -0
  118. package/src/core/exporter/html-report/index.ts +479 -0
  119. package/src/core/exporter/html-report/script.ts +100 -0
  120. package/src/core/exporter/html-report/styles.ts +408 -0
  121. package/src/core/generator/chunker.ts +42 -16
  122. package/src/core/generator/coverage-phase.ts +0 -0
  123. package/src/core/generator/create-body.ts +89 -0
  124. package/src/core/generator/data-factory.ts +445 -19
  125. package/src/core/generator/describe.ts +1 -1
  126. package/src/core/generator/fixtures-builder.ts +325 -0
  127. package/src/core/generator/index.ts +7 -5
  128. package/src/core/generator/openapi-reader.ts +37 -3
  129. package/src/core/generator/path-param-disambig.ts +114 -0
  130. package/src/core/generator/resources-builder.ts +648 -0
  131. package/src/core/generator/schema-utils.ts +11 -3
  132. package/src/core/generator/serializer.ts +103 -13
  133. package/src/core/generator/suite-generator.ts +419 -111
  134. package/src/core/generator/types.ts +8 -0
  135. package/src/core/identity/identity-file.ts +129 -0
  136. package/src/core/lint/affects.ts +28 -0
  137. package/src/core/lint/config.ts +96 -0
  138. package/src/core/lint/format.ts +42 -0
  139. package/src/core/lint/index.ts +94 -0
  140. package/src/core/lint/reporter.ts +128 -0
  141. package/src/core/lint/rules/consistency.ts +158 -0
  142. package/src/core/lint/rules/heuristics.ts +97 -0
  143. package/src/core/lint/rules/strictness.ts +109 -0
  144. package/src/core/lint/types.ts +96 -0
  145. package/src/core/lint/walker.ts +248 -0
  146. package/src/core/meta/meta-store.ts +6 -73
  147. package/src/core/output/README.md +91 -0
  148. package/src/core/output/index.ts +13 -0
  149. package/src/core/output/run.ts +126 -0
  150. package/src/core/output/types.ts +129 -0
  151. package/src/core/parser/env-interpolation.ts +104 -0
  152. package/src/core/parser/filter.ts +57 -0
  153. package/src/core/parser/schema.ts +129 -4
  154. package/src/core/parser/types.ts +19 -1
  155. package/src/core/parser/variables.ts +0 -0
  156. package/src/core/parser/yaml-parser.ts +58 -12
  157. package/src/core/probe/bootstrap.ts +34 -0
  158. package/src/core/probe/dry-run-envelope.ts +57 -0
  159. package/src/core/probe/mass-assignment-probe-class.ts +198 -0
  160. package/src/core/probe/mass-assignment-probe.ts +1122 -0
  161. package/src/core/probe/mass-assignment-template.ts +212 -0
  162. package/src/core/probe/method-probe.ts +43 -76
  163. package/src/core/probe/method-shared.ts +69 -0
  164. package/src/core/probe/negative-probe.ts +183 -149
  165. package/src/core/probe/orphan-tracker.ts +188 -0
  166. package/src/core/probe/path-discovery.ts +440 -0
  167. package/src/core/probe/probe-harness.ts +120 -0
  168. package/src/core/probe/registry.ts +89 -0
  169. package/src/core/probe/runner.ts +136 -0
  170. package/src/core/probe/security-probe-class.ts +201 -0
  171. package/src/core/probe/security-probe.ts +1453 -0
  172. package/src/core/probe/shared.ts +505 -0
  173. package/src/core/probe/static-probe-class.ts +125 -0
  174. package/src/core/probe/types.ts +165 -0
  175. package/src/core/probe/verdict-aggregator.ts +33 -0
  176. package/src/core/probe/webhooks-probe.ts +284 -0
  177. package/src/core/reporter/console.ts +41 -2
  178. package/src/core/reporter/index.ts +2 -3
  179. package/src/core/reporter/json.ts +11 -1
  180. package/src/core/reporter/junit.ts +27 -12
  181. package/src/core/reporter/ndjson.ts +37 -0
  182. package/src/core/reporter/types.ts +3 -0
  183. package/src/core/runner/assertions.ts +58 -1
  184. package/src/core/runner/async-pool.ts +108 -0
  185. package/src/core/runner/auth-path.ts +8 -0
  186. package/src/core/runner/ci-context.ts +72 -0
  187. package/src/core/runner/executor.ts +264 -20
  188. package/src/core/runner/form-encode.ts +51 -0
  189. package/src/core/runner/http-client.ts +75 -2
  190. package/src/core/runner/learn-drift.ts +293 -0
  191. package/src/core/runner/preflight-vars.ts +149 -0
  192. package/src/core/runner/progress-tracker.ts +73 -0
  193. package/src/core/runner/rate-limiter.ts +89 -17
  194. package/src/core/runner/run-kind.ts +39 -0
  195. package/src/core/runner/schema-validator.ts +312 -0
  196. package/src/core/runner/send-request.ts +153 -20
  197. package/src/core/runner/types.ts +38 -0
  198. package/src/core/secrets/registry.ts +164 -0
  199. package/src/core/secrets/secrets-file.ts +115 -0
  200. package/src/core/selectors/operation-filter.ts +144 -0
  201. package/src/core/setup-api.ts +415 -16
  202. package/src/core/severity/category.ts +94 -0
  203. package/src/core/severity/index.ts +121 -0
  204. package/src/core/spec/layers.ts +154 -0
  205. package/src/core/util/format-eta.ts +21 -0
  206. package/src/core/utils.ts +5 -1
  207. package/src/core/workspace/config.ts +129 -0
  208. package/src/core/workspace/manifest.ts +283 -0
  209. package/src/core/workspace/output-rotation.ts +62 -0
  210. package/src/core/workspace/triage-path.ts +87 -0
  211. package/src/db/lint-runs.ts +47 -0
  212. package/src/db/migrate.ts +126 -0
  213. package/src/db/migrations/0001_run_kind.sql +25 -0
  214. package/src/db/migrations/sql.d.ts +4 -0
  215. package/src/db/queries/collections.ts +133 -0
  216. package/src/db/queries/coverage.ts +9 -0
  217. package/src/db/queries/dashboard.ts +59 -0
  218. package/src/db/queries/results.ts +128 -0
  219. package/src/db/queries/runs.ts +235 -0
  220. package/src/db/queries/sessions.ts +42 -0
  221. package/src/db/queries/settings.ts +28 -0
  222. package/src/db/queries/types.ts +172 -0
  223. package/src/db/queries.ts +72 -802
  224. package/src/db/schema.ts +178 -50
  225. package/src/cli/commands/export.ts +0 -144
  226. package/src/cli/commands/guide.ts +0 -127
  227. package/src/cli/commands/init/templates/skills/scenarios.md +0 -97
  228. package/src/cli/commands/probe-methods.ts +0 -108
  229. package/src/cli/commands/probe-validation.ts +0 -124
  230. package/src/cli/commands/serve.ts +0 -114
  231. package/src/cli/commands/sync.ts +0 -268
  232. package/src/cli/commands/update.ts +0 -189
  233. package/src/cli/commands/validate.ts +0 -34
  234. package/src/core/diagnostics/render-md.ts +0 -112
  235. package/src/core/exporter/postman.ts +0 -963
  236. package/src/core/generator/guide-builder.ts +0 -253
  237. package/src/core/meta/types.ts +0 -19
  238. package/src/core/parser/index.ts +0 -21
  239. package/src/core/runner/execute-run.ts +0 -132
  240. package/src/core/runner/index.ts +0 -12
  241. package/src/core/sync/spec-differ.ts +0 -38
  242. package/src/web/data/collection-state.ts +0 -362
  243. package/src/web/routes/api.ts +0 -314
  244. package/src/web/routes/dashboard.ts +0 -350
  245. package/src/web/routes/runs.ts +0 -64
  246. package/src/web/schemas.ts +0 -121
  247. package/src/web/server.ts +0 -134
  248. package/src/web/static/htmx.min.cjs +0 -1
  249. package/src/web/static/style.css +0 -1148
  250. package/src/web/views/endpoints-tab.ts +0 -174
  251. package/src/web/views/explorer-tab.ts +0 -402
  252. package/src/web/views/health-strip.ts +0 -92
  253. package/src/web/views/layout.ts +0 -48
  254. package/src/web/views/results.ts +0 -210
  255. package/src/web/views/runs-tab.ts +0 -126
  256. package/src/web/views/suites-tab.ts +0 -181
@@ -0,0 +1,293 @@
1
+ import { readFile, writeFile, mkdir } from "node:fs/promises";
2
+ import { dirname } from "node:path";
3
+ import type { TestRunResult, AssertionResult, StepResult } from "./types.ts";
4
+
5
+ export interface DriftCase {
6
+ suite_name: string;
7
+ suite_file?: string;
8
+ step_name: string;
9
+ method?: string;
10
+ path?: string;
11
+ expected: number;
12
+ observed: number;
13
+ schema_validated: boolean;
14
+ }
15
+
16
+ /**
17
+ * Detect status-code drift cases: step would have passed if `expect.status`
18
+ * matched the observed status, every body/header assertion is green, and the
19
+ * response body matches the OpenAPI schema (no `kind: schema` failures).
20
+ *
21
+ * Skipped on purpose:
22
+ * - `error` steps (network/transport — not a drift signal)
23
+ * - `expect.status` arrays (`one of [...]`) — drift only triggers on a single
24
+ * expected value; arrays already encode tolerance
25
+ * - steps without `kind: schema` evidence — treated as drift_without_schema
26
+ * and surfaced separately to the caller via `schema_validated: false`
27
+ */
28
+ export interface DetectOptions {
29
+ /** True when the run was launched with a schema validator attached. The
30
+ * validator produces no assertions on success, so step.assertions alone
31
+ * can't distinguish "schema ok" from "no validator". */
32
+ schemaValidatorAttached: boolean;
33
+ }
34
+
35
+ export function detectStatusDrifts(results: TestRunResult[], opts: DetectOptions = { schemaValidatorAttached: false }): DriftCase[] {
36
+ const cases: DriftCase[] = [];
37
+ for (const r of results) {
38
+ for (const step of r.steps) {
39
+ const drift = classifyStep(step, opts);
40
+ if (!drift) continue;
41
+ cases.push({
42
+ suite_name: r.suite_name,
43
+ suite_file: r.suite_file,
44
+ step_name: step.name,
45
+ method: step.request?.method,
46
+ path: extractPathFromUrl(step.request?.url),
47
+ expected: drift.expected,
48
+ observed: drift.observed,
49
+ schema_validated: drift.schemaValidated,
50
+ });
51
+ }
52
+ }
53
+ return cases;
54
+ }
55
+
56
+ interface StepDrift {
57
+ expected: number;
58
+ observed: number;
59
+ schemaValidated: boolean;
60
+ }
61
+
62
+ function classifyStep(step: StepResult, opts: DetectOptions): StepDrift | null {
63
+ if (step.status !== "fail") return null;
64
+ if (!step.response) return null;
65
+
66
+ const statusFails = step.assertions.filter(
67
+ a => a.field === "status" && !a.passed && typeof a.rule === "string" && a.rule.startsWith("equals "),
68
+ );
69
+ if (statusFails.length !== 1) return null;
70
+
71
+ const otherFails = step.assertions.filter(a => !a.passed && a !== statusFails[0]);
72
+ if (otherFails.length > 0) return null;
73
+
74
+ const expected = parseExpected(statusFails[0]!);
75
+ if (expected === null) return null;
76
+
77
+ const observed = step.response.status;
78
+ if (expected === observed) return null;
79
+
80
+ // Schema validation evidence — when the validator was attached, assertions
81
+ // contain `kind: "schema"` entries on failure and nothing on success. So
82
+ // "validator attached AND no failing schema assertion" is the success case.
83
+ const schemaFails = step.assertions.filter(a => a.kind === "schema" && !a.passed);
84
+ if (schemaFails.length > 0) return null; // body diverges from spec — not a drift
85
+ const schemaValidated = opts.schemaValidatorAttached;
86
+
87
+ return { expected, observed, schemaValidated };
88
+ }
89
+
90
+ function parseExpected(a: AssertionResult): number | null {
91
+ if (typeof a.expected === "number") return a.expected;
92
+ if (typeof a.expected === "string") {
93
+ const n = Number(a.expected);
94
+ return Number.isFinite(n) ? n : null;
95
+ }
96
+ return null;
97
+ }
98
+
99
+ function extractPathFromUrl(url: string | undefined): string | undefined {
100
+ if (!url) return undefined;
101
+ try {
102
+ return new URL(url).pathname;
103
+ } catch {
104
+ return url;
105
+ }
106
+ }
107
+
108
+ export function formatDriftPlan(cases: DriftCase[]): string {
109
+ if (cases.length === 0) return "No status-code drift detected.\n";
110
+ const lines: string[] = [];
111
+ lines.push(`Drift detected (${cases.length} case${cases.length === 1 ? "" : "s"}):`);
112
+ for (const c of cases) {
113
+ const ep = `${c.method ?? "?"} ${c.path ?? "?"}`.padEnd(40);
114
+ const schema = c.schema_validated ? "body-schema=ok" : "body-schema=unverified";
115
+ lines.push(` ${ep} spec=${c.expected} observed=${c.observed} ${schema} → suggest: update test, or add to drifts`);
116
+ }
117
+ lines.push("");
118
+ lines.push("Run with --learn-apply --learn-target=test to rewrite expect.status in YAML");
119
+ lines.push("Run with --learn-apply --learn-target=drifts to record in apis/<name>/tolerated-drifts.yaml");
120
+ return lines.join("\n") + "\n";
121
+ }
122
+
123
+ export interface ApplyResult {
124
+ updated: number;
125
+ errors: { suite_file: string; step_name: string; reason: string }[];
126
+ }
127
+
128
+ /**
129
+ * Rewrite `expect.status: <expected>` → `<observed>` in each suite file.
130
+ *
131
+ * Implementation is line-based: locate the step block by `name:`, then scan
132
+ * forward until the next sibling step or dedent looking for the first
133
+ * `status:` line at a deeper indent. We don't reparse the YAML — preserves
134
+ * comments, key order, and trailing whitespace so the diff is minimal.
135
+ */
136
+ export async function applyDriftsToTests(cases: DriftCase[]): Promise<ApplyResult> {
137
+ const result: ApplyResult = { updated: 0, errors: [] };
138
+
139
+ // Group by file — read once, write once per file.
140
+ const byFile = new Map<string, DriftCase[]>();
141
+ for (const c of cases) {
142
+ if (!c.suite_file) {
143
+ result.errors.push({ suite_file: "<unknown>", step_name: c.step_name, reason: "suite_file missing" });
144
+ continue;
145
+ }
146
+ const list = byFile.get(c.suite_file) ?? [];
147
+ list.push(c);
148
+ byFile.set(c.suite_file, list);
149
+ }
150
+
151
+ for (const [file, fileCases] of byFile) {
152
+ let content: string;
153
+ try {
154
+ content = await readFile(file, "utf-8");
155
+ } catch (err) {
156
+ for (const c of fileCases) {
157
+ result.errors.push({ suite_file: file, step_name: c.step_name, reason: `read failed: ${(err as Error).message}` });
158
+ }
159
+ continue;
160
+ }
161
+
162
+ let lines = content.split("\n");
163
+ let touched = false;
164
+ for (const c of fileCases) {
165
+ const edited = rewriteExpectStatus(lines, c.step_name, c.expected, c.observed);
166
+ if (edited.ok) {
167
+ lines = edited.lines;
168
+ touched = true;
169
+ result.updated++;
170
+ } else {
171
+ result.errors.push({ suite_file: file, step_name: c.step_name, reason: edited.reason });
172
+ }
173
+ }
174
+
175
+ if (touched) {
176
+ try {
177
+ await writeFile(file, lines.join("\n"), "utf-8");
178
+ } catch (err) {
179
+ for (const c of fileCases) {
180
+ result.errors.push({ suite_file: file, step_name: c.step_name, reason: `write failed: ${(err as Error).message}` });
181
+ }
182
+ }
183
+ }
184
+ }
185
+
186
+ return result;
187
+ }
188
+
189
+ function rewriteExpectStatus(
190
+ lines: string[],
191
+ stepName: string,
192
+ expected: number,
193
+ observed: number,
194
+ ): { ok: true; lines: string[] } | { ok: false; reason: string } {
195
+ // Match `- name: foo` or `- name: "foo"` — capture indent of the dash so we
196
+ // know where the step block ends.
197
+ const nameRe = new RegExp(`^(\\s*)-\\s+name:\\s+["']?${escapeRegExp(stepName)}["']?\\s*$`);
198
+ let stepStart = -1;
199
+ let stepIndent = 0;
200
+ for (let i = 0; i < lines.length; i++) {
201
+ const m = lines[i]!.match(nameRe);
202
+ if (m) {
203
+ stepStart = i;
204
+ stepIndent = m[1]!.length;
205
+ break;
206
+ }
207
+ }
208
+ if (stepStart < 0) return { ok: false, reason: `step "${stepName}" not found in YAML` };
209
+
210
+ // Step block ends at next sibling (`- name:` at the same indent) or at the
211
+ // first line that dedents past the step's column.
212
+ let stepEnd = lines.length;
213
+ for (let i = stepStart + 1; i < lines.length; i++) {
214
+ const line = lines[i]!;
215
+ if (line.trim() === "") continue;
216
+ const indent = line.match(/^(\s*)/)![1]!.length;
217
+ if (indent <= stepIndent) {
218
+ stepEnd = i;
219
+ break;
220
+ }
221
+ }
222
+
223
+ const statusRe = new RegExp(`^(\\s*)status:\\s*${expected}\\s*(#.*)?$`);
224
+ for (let i = stepStart + 1; i < stepEnd; i++) {
225
+ const m = lines[i]!.match(statusRe);
226
+ if (m) {
227
+ const tail = m[2] ? ` ${m[2]}` : "";
228
+ lines[i] = `${m[1]}status: ${observed}${tail}`;
229
+ return { ok: true, lines };
230
+ }
231
+ }
232
+ return { ok: false, reason: `expect.status: ${expected} not found within step "${stepName}"` };
233
+ }
234
+
235
+ function escapeRegExp(s: string): string {
236
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
237
+ }
238
+
239
+ /**
240
+ * Append drift cases to `<apiDir>/tolerated-drifts.yaml`. The file is a flat
241
+ * `drifts:` list; we de-duplicate on (method, path, expected, observed).
242
+ *
243
+ * Format kept intentionally minimal — runner-side enforcement (skip the
244
+ * status assertion when a tolerated drift matches) is a follow-up task; this
245
+ * call just records the data so a human can review.
246
+ */
247
+ export async function appendToleratedDrifts(apiDir: string, cases: DriftCase[]): Promise<{ written: number; file: string }> {
248
+ const file = `${apiDir}/tolerated-drifts.yaml`;
249
+ let existing = "";
250
+ try {
251
+ existing = await readFile(file, "utf-8");
252
+ } catch {
253
+ // new file — keep empty
254
+ }
255
+
256
+ const seen = new Set<string>();
257
+ const driftRe = /^\s*-\s*method:\s*(\w+)\s*$\n\s*path:\s*(\S+)\s*$\n\s*expected:\s*(\d+)\s*$\n\s*observed:\s*(\d+)/gm;
258
+ let m: RegExpExecArray | null;
259
+ while ((m = driftRe.exec(existing)) !== null) {
260
+ seen.add(`${m[1]!.toUpperCase()} ${m[2]} ${m[3]}->${m[4]}`);
261
+ }
262
+
263
+ const fresh: DriftCase[] = [];
264
+ for (const c of cases) {
265
+ const key = `${(c.method ?? "?").toUpperCase()} ${c.path ?? "?"} ${c.expected}->${c.observed}`;
266
+ if (seen.has(key)) continue;
267
+ seen.add(key);
268
+ fresh.push(c);
269
+ }
270
+
271
+ if (fresh.length === 0) return { written: 0, file };
272
+
273
+ let body = existing;
274
+ if (!/^drifts:\s*$/m.test(body)) {
275
+ body = body.trim();
276
+ if (body.length > 0) body += "\n";
277
+ body += "drifts:\n";
278
+ } else if (!body.endsWith("\n")) {
279
+ body += "\n";
280
+ }
281
+
282
+ for (const c of fresh) {
283
+ body += ` - method: ${c.method ?? "?"}\n`;
284
+ body += ` path: ${c.path ?? "?"}\n`;
285
+ body += ` expected: ${c.expected}\n`;
286
+ body += ` observed: ${c.observed}\n`;
287
+ body += ` note: ""\n`;
288
+ }
289
+
290
+ await mkdir(dirname(file), { recursive: true });
291
+ await writeFile(file, body, "utf-8");
292
+ return { written: fresh.length, file };
293
+ }
@@ -0,0 +1,149 @@
1
+ import type { TestSuite, TestStep, AssertionRule } from "../parser/types.ts";
2
+ import { GENERATORS } from "../parser/variables.ts";
3
+
4
+ const VAR_PATTERN = /\{\{([^{}]+)\}\}/g;
5
+
6
+ function scanRefs(value: unknown, out: Set<string>): void {
7
+ if (typeof value === "string") {
8
+ for (const match of value.matchAll(VAR_PATTERN)) {
9
+ const key = match[1]!.trim();
10
+ if (!key.startsWith("$")) out.add(key);
11
+ }
12
+ } else if (Array.isArray(value)) {
13
+ for (const item of value) scanRefs(item, out);
14
+ } else if (typeof value === "object" && value !== null) {
15
+ for (const v of Object.values(value)) scanRefs(v, out);
16
+ }
17
+ }
18
+
19
+ function collectCapturesAndSets(step: TestStep, out: Set<string>): void {
20
+ if (step.set) {
21
+ for (const k of Object.keys(step.set)) out.add(k);
22
+ }
23
+ if (step.for_each?.var) out.add(step.for_each.var);
24
+ const scanRule = (rule: AssertionRule | undefined): void => {
25
+ if (!rule) return;
26
+ if (rule.capture) out.add(rule.capture);
27
+ if (rule.each) {
28
+ for (const r of Object.values(rule.each)) scanRule(r);
29
+ }
30
+ if (rule.contains_item) {
31
+ for (const r of Object.values(rule.contains_item)) scanRule(r);
32
+ }
33
+ };
34
+ if (step.expect?.body) {
35
+ for (const r of Object.values(step.expect.body)) scanRule(r);
36
+ }
37
+ if (step.expect?.headers) {
38
+ for (const v of Object.values(step.expect.headers)) {
39
+ if (typeof v === "object" && v !== null) scanRule(v as AssertionRule);
40
+ }
41
+ }
42
+ }
43
+
44
+ export interface MissingVarHit {
45
+ suite: string;
46
+ file?: string;
47
+ step?: string;
48
+ variable: string;
49
+ }
50
+
51
+ /**
52
+ * Pre-flight scan: find {{var}} references in suites that have no producer
53
+ * (env value, suite-level parameterize/set, prior-step capture). Excludes
54
+ * built-in $generators.
55
+ *
56
+ * Conservative: per-suite — accumulates all captures/sets across steps, so
57
+ * forward references inside the suite are tolerated (correctness requires
58
+ * runtime ordering checks anyway).
59
+ */
60
+ export function preflightCheckVars(
61
+ suites: TestSuite[],
62
+ env: Record<string, string>,
63
+ ): MissingVarHit[] {
64
+ const hits: MissingVarHit[] = [];
65
+ const generatorKeys = new Set(Object.keys(GENERATORS));
66
+
67
+ for (const suite of suites) {
68
+ const known = new Set<string>(Object.keys(env));
69
+ if (suite.parameterize) {
70
+ for (const k of Object.keys(suite.parameterize)) known.add(k);
71
+ }
72
+ for (const step of suite.tests) collectCapturesAndSets(step, known);
73
+
74
+ const scanStepRefs = (step: TestStep): Set<string> => {
75
+ const refs = new Set<string>();
76
+ scanRefs(step.path, refs);
77
+ scanRefs(step.headers, refs);
78
+ scanRefs(step.json, refs);
79
+ scanRefs(step.form, refs);
80
+ scanRefs(step.multipart, refs);
81
+ scanRefs(step.query, refs);
82
+ if (step.skip_if) scanRefs(step.skip_if, refs);
83
+ if (step.retry_until) scanRefs(step.retry_until.condition, refs);
84
+ if (step.set) scanRefs(step.set, refs);
85
+ if (step.for_each) scanRefs(step.for_each.in, refs);
86
+ return refs;
87
+ };
88
+
89
+ const suiteRefs = new Set<string>();
90
+ if (suite.base_url) scanRefs(suite.base_url, suiteRefs);
91
+ if (suite.headers) scanRefs(suite.headers, suiteRefs);
92
+ for (const v of suiteRefs) {
93
+ if (!known.has(v) && !generatorKeys.has(v)) {
94
+ hits.push({ suite: suite.name, file: suite.filePath, variable: v });
95
+ }
96
+ }
97
+
98
+ for (const step of suite.tests) {
99
+ const refs = scanStepRefs(step);
100
+ for (const v of refs) {
101
+ if (!known.has(v) && !generatorKeys.has(v)) {
102
+ hits.push({
103
+ suite: suite.name,
104
+ file: suite.filePath,
105
+ step: step.name,
106
+ variable: v,
107
+ });
108
+ }
109
+ }
110
+ }
111
+ }
112
+
113
+ return hits;
114
+ }
115
+
116
+ export function formatMissingVarLine(hit: MissingVarHit): string {
117
+ const where = hit.step ? `${hit.suite} → ${hit.step}` : hit.suite;
118
+ const file = hit.file ? ` (${hit.file})` : "";
119
+ return `Undefined variable {{${hit.variable}}} in ${where}${file}`;
120
+ }
121
+
122
+ /**
123
+ * TASK-248: collapse per-(suite,step,variable) hits into one summary line per
124
+ * unique variable. When the same {{var}} is referenced in 8–12 places (typical
125
+ * when running outside a workspace and `.env.yaml` is missing), per-hit emit
126
+ * spams stderr — the aggregated form keeps signal (the *names* of the missing
127
+ * vars) while dropping noise.
128
+ */
129
+ export function summarizeMissingVars(hits: MissingVarHit[]): string[] {
130
+ if (hits.length === 0) return [];
131
+ const byVar = new Map<string, { refs: number; suites: Set<string> }>();
132
+ for (const h of hits) {
133
+ let entry = byVar.get(h.variable);
134
+ if (!entry) {
135
+ entry = { refs: 0, suites: new Set() };
136
+ byVar.set(h.variable, entry);
137
+ }
138
+ entry.refs++;
139
+ entry.suites.add(h.suite);
140
+ }
141
+ const names = [...byVar.keys()].sort();
142
+ const totalRefs = hits.length;
143
+ const totalSuites = new Set(hits.map((h) => h.suite)).size;
144
+ const head = names.slice(0, 6).map((n) => `{{${n}}}`).join(", ");
145
+ const tail = names.length > 6 ? `, … and ${names.length - 6} more` : "";
146
+ return [
147
+ `Undefined variables: ${head}${tail} (${totalRefs} reference${totalRefs === 1 ? "" : "s"} across ${totalSuites} suite${totalSuites === 1 ? "" : "s"})`,
148
+ ];
149
+ }
@@ -0,0 +1,73 @@
1
+ /**
2
+ * ARV-249: lightweight progress tracker for `zond run`. A separate module
3
+ * so the formatter can be unit-tested without spinning up an executor.
4
+ *
5
+ * Architecture: run.ts owns the tracker + `setInterval`. Every completed
6
+ * step calls `tracker.recordStep(...)` via `RunSuiteOptions.onStepDone`,
7
+ * the interval ticks every PROGRESS_INTERVAL_MS and writes one stderr
8
+ * line, and run.ts clears the interval before printing the final report.
9
+ */
10
+ import { formatEta } from "../util/format-eta.ts";
11
+ import type { StepResult } from "./types.ts";
12
+
13
+ export const PROGRESS_INTERVAL_MS = 5000;
14
+ /** Wait this long before emitting the first progress line. Short runs
15
+ * that finish under the threshold stay silent. */
16
+ export const PROGRESS_QUIET_MS = 5000;
17
+
18
+ export interface ProgressSnapshot {
19
+ elapsedMs: number;
20
+ completedSteps: number;
21
+ totalSteps: number;
22
+ httpRequests: number;
23
+ effectiveRps: number;
24
+ etaSeconds: number;
25
+ }
26
+
27
+ export class ProgressTracker {
28
+ private completedSteps = 0;
29
+ private httpRequests = 0;
30
+ private readonly startedAt: number;
31
+
32
+ constructor(private readonly totalSteps: number, now: number = Date.now()) {
33
+ this.startedAt = now;
34
+ }
35
+
36
+ recordStep(step: StepResult): void {
37
+ this.completedSteps += 1;
38
+ if (step.status !== "skip" && step.response !== undefined) {
39
+ this.httpRequests += 1;
40
+ }
41
+ }
42
+
43
+ snapshot(now: number = Date.now()): ProgressSnapshot {
44
+ const elapsedMs = Math.max(0, now - this.startedAt);
45
+ const elapsedSec = elapsedMs / 1000;
46
+ const effectiveRps = elapsedSec > 0 ? this.httpRequests / elapsedSec : 0;
47
+ const remaining = Math.max(0, this.totalSteps - this.completedSteps);
48
+ const stepRate = elapsedSec > 0 ? this.completedSteps / elapsedSec : 0;
49
+ const etaSeconds = stepRate > 0 ? remaining / stepRate : Infinity;
50
+ return {
51
+ elapsedMs,
52
+ completedSteps: this.completedSteps,
53
+ totalSteps: this.totalSteps,
54
+ httpRequests: this.httpRequests,
55
+ effectiveRps,
56
+ etaSeconds,
57
+ };
58
+ }
59
+ }
60
+
61
+ /** Format a progress snapshot for stderr. Stable wording — agents may
62
+ * grep the line. */
63
+ export function formatProgressLine(snap: ProgressSnapshot): string {
64
+ const elapsed = formatEta(snap.elapsedMs / 1000);
65
+ const pct = snap.totalSteps > 0
66
+ ? Math.min(100, Math.floor((snap.completedSteps / snap.totalSteps) * 100))
67
+ : 0;
68
+ const rps = snap.effectiveRps >= 10
69
+ ? Math.round(snap.effectiveRps).toString()
70
+ : snap.effectiveRps.toFixed(1);
71
+ const eta = Number.isFinite(snap.etaSeconds) ? formatEta(snap.etaSeconds) : "?";
72
+ return `zond: [${elapsed}] ${snap.completedSteps}/${snap.totalSteps} steps (${pct}%), ${snap.httpRequests} req, ~${rps} req/s, ETA ${eta}`;
73
+ }
@@ -2,9 +2,11 @@ export interface RateLimiter {
2
2
  acquire(): Promise<void>;
3
3
  /**
4
4
  * Feed rate-limit metadata from the latest response back into the limiter.
5
- * When `remaining` falls at or below the threshold, the limiter postpones
6
- * the next acquire until the API's reset window expires. No-op when the
7
- * server reports plenty of headroom.
5
+ * Two effects: (1) when `remaining` falls at or below the threshold, the
6
+ * limiter postpones the next acquire until the API's reset window expires;
7
+ * (2) when the response carries a `RateLimit-Policy` (RFC 9568), the
8
+ * limiter learns the per-request spacing so subsequent parallel acquires
9
+ * are forced into single-file at burst=1.
8
10
  *
9
11
  * Optional so existing callers / mocks need not implement it.
10
12
  */
@@ -16,18 +18,37 @@ export interface RateLimitMeta {
16
18
  remaining?: number;
17
19
  /** Either seconds-until-reset (RFC draft) or a Unix epoch in seconds (GitHub style). */
18
20
  reset?: number;
19
- /** Window cap; used only for diagnostics. */
21
+ /** Window cap; used for diagnostics and spacing fallback. */
20
22
  limit?: number;
23
+ /**
24
+ * Minimum spacing between requests in ms — derived from `RateLimit-Policy:
25
+ * N;w=W` (RFC 9568). When set, the limiter raises its own interval to at
26
+ * least this value so a burst of parallel requests is paced one-by-one
27
+ * instead of overshooting the window.
28
+ */
29
+ intervalMs?: number;
21
30
  }
22
31
 
23
- /** When `remaining` is at or below this number we proactively pause until reset. */
24
- const THROTTLE_THRESHOLD = 5;
32
+ /**
33
+ * When `remaining` is at or below this number we proactively pause until reset.
34
+ * Conservative threshold — at 2, we still have buffer for one in-flight retry
35
+ * (if we paused at 5 we'd over-throttle on small windows like 5-req/1-sec
36
+ * policies, where every request would trigger a sleep).
37
+ */
38
+ const THROTTLE_THRESHOLD = 2;
39
+
40
+ /**
41
+ * Padding added to policy-derived spacing to absorb clock drift between the
42
+ * server's window and our local Date.now() (TASK-88). Without this, spacing
43
+ * exactly at `W/N` ms can still hit the window boundary on the wrong side.
44
+ */
45
+ const POLICY_SAFETY_MS = 50;
25
46
 
26
47
  /** Magnitudes above this are treated as Unix timestamps; below as relative
27
48
  * seconds. 10^9 seconds ≈ Sep 2001, so any real reset window is far below. */
28
49
  const UNIX_TS_BOUNDARY = 1_000_000_000;
29
50
 
30
- function applyMeta(prevNextAvailable: number, meta: RateLimitMeta, now: number): number {
51
+ function applyResetPause(prevNextAvailable: number, meta: RateLimitMeta, now: number): number {
31
52
  if (meta.remaining === undefined) return prevNextAvailable;
32
53
  if (meta.remaining > THROTTLE_THRESHOLD) return prevNextAvailable;
33
54
  if (meta.reset === undefined || !Number.isFinite(meta.reset)) return prevNextAvailable;
@@ -37,7 +58,7 @@ function applyMeta(prevNextAvailable: number, meta: RateLimitMeta, now: number):
37
58
 
38
59
  class IntervalRateLimiter implements RateLimiter {
39
60
  private nextAvailable = 0;
40
- private readonly intervalMs: number;
61
+ private intervalMs: number;
41
62
 
42
63
  constructor(reqPerSec: number) {
43
64
  if (!Number.isFinite(reqPerSec) || reqPerSec <= 0) {
@@ -57,21 +78,38 @@ class IntervalRateLimiter implements RateLimiter {
57
78
  }
58
79
 
59
80
  note(meta: RateLimitMeta, now: number = Date.now()): void {
60
- this.nextAvailable = applyMeta(this.nextAvailable, meta, now);
81
+ if (meta.intervalMs !== undefined && meta.intervalMs > this.intervalMs) {
82
+ // Already-reserved slots were spaced at the OLD interval; push
83
+ // nextAvailable forward by the delta so the new spacing kicks in
84
+ // immediately rather than only on the next-next request.
85
+ this.nextAvailable += meta.intervalMs - this.intervalMs;
86
+ this.intervalMs = meta.intervalMs;
87
+ }
88
+ this.nextAvailable = applyResetPause(this.nextAvailable, meta, now);
61
89
  }
62
90
  }
63
91
 
64
92
  class AdaptiveRateLimiter implements RateLimiter {
65
93
  private nextAvailable = 0;
94
+ /** Learned from RateLimit-Policy. 0 until a policy is seen — until then,
95
+ * parallel acquires are not spaced (matches the original adaptive
96
+ * behaviour). Once known, every acquire reserves a slot of `intervalMs`. */
97
+ private intervalMs = 0;
66
98
 
67
99
  async acquire(): Promise<void> {
68
100
  const now = Date.now();
69
- const wait = this.nextAvailable - now;
70
- if (wait > 0) await Bun.sleep(wait);
101
+ const slot = Math.max(now, this.nextAvailable);
102
+ const waitMs = slot - now;
103
+ this.nextAvailable = slot + this.intervalMs;
104
+ if (waitMs > 0) await Bun.sleep(waitMs);
71
105
  }
72
106
 
73
107
  note(meta: RateLimitMeta, now: number = Date.now()): void {
74
- this.nextAvailable = applyMeta(this.nextAvailable, meta, now);
108
+ if (meta.intervalMs !== undefined && meta.intervalMs > this.intervalMs) {
109
+ this.nextAvailable += meta.intervalMs - this.intervalMs;
110
+ this.intervalMs = meta.intervalMs;
111
+ }
112
+ this.nextAvailable = applyResetPause(this.nextAvailable, meta, now);
75
113
  }
76
114
  }
77
115
 
@@ -83,17 +121,24 @@ export function createRateLimiter(reqPerSec: number | undefined): RateLimiter |
83
121
 
84
122
  /**
85
123
  * Adaptive limiter for `--rate-limit auto`. Issues no proactive throttling on
86
- * its own, but reacts to ratelimit-* response headers via `note()` and pauses
87
- * the request stream until the API's reset window elapses when headroom drops.
124
+ * its own initially, but reacts to ratelimit-* response headers via `note()`:
125
+ * (a) pauses the request stream until the API's reset window elapses when
126
+ * remaining headroom drops; (b) once a `RateLimit-Policy` is seen, paces
127
+ * subsequent requests at the policy's `W/N` spacing — this prevents bursts
128
+ * from blowing through small windows (e.g. 5-req/1-sec policies).
88
129
  */
89
130
  export function createAdaptiveRateLimiter(): RateLimiter {
90
131
  return new AdaptiveRateLimiter();
91
132
  }
92
133
 
93
134
  /**
94
- * Read RFC draft-ietf-httpapi-ratelimit-headers (`ratelimit-*`) plus the
95
- * GitHub / Stripe style `x-ratelimit-*` aliases out of a response header bag.
96
- * All keys are matched case-insensitively. Unparseable values are dropped.
135
+ * Read RFC 9568 `ratelimit-*` headers (was draft-ietf-httpapi-ratelimit-headers)
136
+ * plus the GitHub / Stripe style `x-ratelimit-*` aliases out of a response
137
+ * header bag. All keys are matched case-insensitively. Unparseable values are
138
+ * dropped.
139
+ *
140
+ * `RateLimit-Policy: N;w=W` is parsed into a per-request `intervalMs` of
141
+ * `(W/N)*1000 + POLICY_SAFETY_MS` so the limiter can pace bursts.
97
142
  */
98
143
  export function parseRateLimitHeaders(headers: Record<string, string>): RateLimitMeta {
99
144
  const lower: Record<string, string> = {};
@@ -107,13 +152,40 @@ export function parseRateLimitHeaders(headers: Record<string, string>): RateLimi
107
152
  const n = Number.parseFloat(match[0]);
108
153
  return Number.isFinite(n) ? n : undefined;
109
154
  };
155
+ const policy = lower["ratelimit-policy"] ?? lower["x-ratelimit-policy"];
156
+ const intervalMs = derivePolicyIntervalMs(policy);
110
157
  return {
111
158
  limit: num(lower["ratelimit-limit"] ?? lower["x-ratelimit-limit"]),
112
159
  remaining: num(lower["ratelimit-remaining"] ?? lower["x-ratelimit-remaining"]),
113
160
  reset: num(lower["ratelimit-reset"] ?? lower["x-ratelimit-reset"]),
161
+ intervalMs,
114
162
  };
115
163
  }
116
164
 
165
+ /**
166
+ * Parse `RateLimit-Policy: 5;w=1` (or comma-separated multi-policy — we honour
167
+ * the *strictest* one). Returns the implied per-request interval in ms,
168
+ * including a small safety margin. Returns undefined when the header is
169
+ * malformed or missing.
170
+ */
171
+ function derivePolicyIntervalMs(policy: string | undefined): number | undefined {
172
+ if (!policy) return undefined;
173
+ let strictest: number | undefined;
174
+ for (const item of policy.split(",")) {
175
+ const parts = item.trim().split(";").map(s => s.trim()).filter(Boolean);
176
+ if (parts.length === 0) continue;
177
+ const limit = Number.parseFloat(parts[0]!);
178
+ if (!Number.isFinite(limit) || limit <= 0) continue;
179
+ const wPart = parts.find(p => p.startsWith("w="));
180
+ if (!wPart) continue;
181
+ const window = Number.parseFloat(wPart.slice(2));
182
+ if (!Number.isFinite(window) || window <= 0) continue;
183
+ const interval = (window / limit) * 1000 + POLICY_SAFETY_MS;
184
+ if (strictest === undefined || interval > strictest) strictest = interval;
185
+ }
186
+ return strictest;
187
+ }
188
+
117
189
  export function parseRetryAfter(header: string | null | undefined, now: number = Date.now()): number | undefined {
118
190
  if (!header) return undefined;
119
191
  const trimmed = header.trim();