@kirrosh/zond 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/CHANGELOG.md +758 -3
  2. package/README.md +78 -15
  3. package/package.json +17 -10
  4. package/src/cli/argv.ts +122 -0
  5. package/src/cli/commands/add-api.ts +134 -0
  6. package/src/cli/commands/api/annotate/idempotency.ts +59 -0
  7. package/src/cli/commands/api/annotate/index.ts +525 -0
  8. package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
  9. package/src/cli/commands/api/annotate/overlay.ts +206 -0
  10. package/src/cli/commands/api/annotate/pagination.ts +60 -0
  11. package/src/cli/commands/api/annotate/prompts.ts +183 -0
  12. package/src/cli/commands/api/annotate/readback.ts +58 -0
  13. package/src/cli/commands/api/annotate/resources.ts +91 -0
  14. package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
  15. package/src/cli/commands/audit.ts +480 -0
  16. package/src/cli/commands/bootstrap.ts +710 -0
  17. package/src/cli/commands/catalog.ts +35 -0
  18. package/src/cli/commands/check.ts +348 -0
  19. package/src/cli/commands/checks.ts +756 -0
  20. package/src/cli/commands/ci-init.ts +55 -6
  21. package/src/cli/commands/clean.ts +212 -0
  22. package/src/cli/commands/cleanup.ts +262 -0
  23. package/src/cli/commands/completions.ts +192 -0
  24. package/src/cli/commands/coverage.ts +605 -132
  25. package/src/cli/commands/db.ts +180 -8
  26. package/src/cli/commands/describe.ts +37 -2
  27. package/src/cli/commands/discover.ts +1236 -0
  28. package/src/cli/commands/doctor.ts +607 -0
  29. package/src/cli/commands/fixtures.ts +402 -0
  30. package/src/cli/commands/generate.ts +420 -47
  31. package/src/cli/commands/init/agents-md.ts +61 -0
  32. package/src/cli/commands/init/bootstrap.ts +108 -0
  33. package/src/cli/commands/init/index.ts +244 -0
  34. package/src/cli/commands/init/skills.ts +98 -0
  35. package/src/cli/commands/init/templates/agents.md +77 -0
  36. package/src/cli/commands/init/templates/markdown.d.ts +4 -0
  37. package/src/cli/commands/init/templates/skills/zond-checks.md +397 -0
  38. package/src/cli/commands/init/templates/skills/zond-triage.md +210 -0
  39. package/src/cli/commands/init/templates/skills/zond.md +651 -0
  40. package/src/cli/commands/init/templates/zond-config.yml +14 -0
  41. package/src/cli/commands/prepare-fixtures.ts +135 -0
  42. package/src/cli/commands/probe/mass-assignment.ts +503 -0
  43. package/src/cli/commands/probe/security.ts +454 -0
  44. package/src/cli/commands/probe/static.ts +255 -0
  45. package/src/cli/commands/probe/webhooks.ts +161 -0
  46. package/src/cli/commands/probe.ts +459 -0
  47. package/src/cli/commands/reference.ts +87 -0
  48. package/src/cli/commands/refresh-api.ts +169 -0
  49. package/src/cli/commands/remove-api.ts +150 -0
  50. package/src/cli/commands/report-bundle.ts +318 -0
  51. package/src/cli/commands/report.ts +241 -0
  52. package/src/cli/commands/request.ts +379 -4
  53. package/src/cli/commands/run.ts +911 -33
  54. package/src/cli/commands/session.ts +244 -0
  55. package/src/cli/commands/use.ts +74 -0
  56. package/src/cli/index.ts +36 -607
  57. package/src/cli/json-envelope.ts +112 -3
  58. package/src/cli/json-schemas.ts +263 -0
  59. package/src/cli/program.ts +218 -0
  60. package/src/cli/resolve.ts +105 -0
  61. package/src/cli/status-filter.ts +124 -0
  62. package/src/cli/util/api-context.ts +85 -0
  63. package/src/cli/version.ts +8 -0
  64. package/src/core/anti-fp/bootstrap.ts +34 -0
  65. package/src/core/anti-fp/index.ts +33 -0
  66. package/src/core/anti-fp/registry.ts +44 -0
  67. package/src/core/anti-fp/rules/baseline-echo.ts +74 -0
  68. package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +52 -0
  69. package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +38 -0
  70. package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +35 -0
  71. package/src/core/anti-fp/rules/schemathesis/index.ts +24 -0
  72. package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +53 -0
  73. package/src/core/anti-fp/rules/subscription-gated/index.ts +11 -0
  74. package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +75 -0
  75. package/src/core/anti-fp/types.ts +68 -0
  76. package/src/core/checks/checks/_crud-helpers.ts +133 -0
  77. package/src/core/checks/checks/_negative_mutator.ts +133 -0
  78. package/src/core/checks/checks/_readback-helpers.ts +133 -0
  79. package/src/core/checks/checks/content_type_conformance.ts +39 -0
  80. package/src/core/checks/checks/cross_call_references.ts +134 -0
  81. package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
  82. package/src/core/checks/checks/idempotency_replay.ts +246 -0
  83. package/src/core/checks/checks/ignored_auth.ts +211 -0
  84. package/src/core/checks/checks/index.ts +65 -0
  85. package/src/core/checks/checks/lifecycle_transitions.ts +273 -0
  86. package/src/core/checks/checks/missing_required_header.ts +40 -0
  87. package/src/core/checks/checks/negative_data_rejection.ts +45 -0
  88. package/src/core/checks/checks/not_a_server_error.ts +27 -0
  89. package/src/core/checks/checks/open_cors_on_sensitive.ts +131 -0
  90. package/src/core/checks/checks/pagination_invariants.ts +238 -0
  91. package/src/core/checks/checks/positive_data_acceptance.ts +36 -0
  92. package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
  93. package/src/core/checks/checks/response_headers_conformance.ts +74 -0
  94. package/src/core/checks/checks/response_schema_conformance.ts +30 -0
  95. package/src/core/checks/checks/status_code_conformance.ts +61 -0
  96. package/src/core/checks/checks/unsupported_method.ts +63 -0
  97. package/src/core/checks/checks/use_after_free.ts +78 -0
  98. package/src/core/checks/index.ts +30 -0
  99. package/src/core/checks/mode.ts +79 -0
  100. package/src/core/checks/recommended-action.ts +64 -0
  101. package/src/core/checks/registry.ts +78 -0
  102. package/src/core/checks/runner.ts +874 -0
  103. package/src/core/checks/sarif.ts +230 -0
  104. package/src/core/checks/stateful.ts +121 -0
  105. package/src/core/checks/types.ts +189 -0
  106. package/src/core/classifier/recommended-action.ts +222 -0
  107. package/src/core/context/current.ts +51 -0
  108. package/src/core/context/session.ts +78 -0
  109. package/src/core/coverage/loader.ts +185 -0
  110. package/src/core/coverage/reasons.ts +300 -0
  111. package/src/core/diagnostics/db-analysis.ts +161 -12
  112. package/src/core/diagnostics/failure-class.ts +120 -0
  113. package/src/core/diagnostics/failure-hints.ts +212 -9
  114. package/src/core/diagnostics/spec-pointer.ts +99 -0
  115. package/src/core/diagnostics/suggested-fixes.ts +156 -0
  116. package/src/core/exporter/case-study/index.ts +270 -0
  117. package/src/core/exporter/curl.ts +40 -0
  118. package/src/core/exporter/exporter.ts +48 -0
  119. package/src/core/exporter/html-report/escape.ts +24 -0
  120. package/src/core/exporter/html-report/index.ts +479 -0
  121. package/src/core/exporter/html-report/script.ts +100 -0
  122. package/src/core/exporter/html-report/styles.ts +408 -0
  123. package/src/core/generator/chunker.ts +53 -15
  124. package/src/core/generator/coverage-phase.ts +0 -0
  125. package/src/core/generator/create-body.ts +89 -0
  126. package/src/core/generator/data-factory.ts +490 -33
  127. package/src/core/generator/describe.ts +1 -1
  128. package/src/core/generator/fixtures-builder.ts +325 -0
  129. package/src/core/generator/index.ts +7 -5
  130. package/src/core/generator/openapi-reader.ts +55 -3
  131. package/src/core/generator/path-param-disambig.ts +114 -0
  132. package/src/core/generator/resources-builder.ts +648 -0
  133. package/src/core/generator/schema-utils.ts +11 -3
  134. package/src/core/generator/serializer.ts +114 -15
  135. package/src/core/generator/suite-generator.ts +484 -77
  136. package/src/core/generator/types.ts +8 -0
  137. package/src/core/identity/identity-file.ts +129 -0
  138. package/src/core/lint/affects.ts +28 -0
  139. package/src/core/lint/config.ts +96 -0
  140. package/src/core/lint/format.ts +42 -0
  141. package/src/core/lint/index.ts +94 -0
  142. package/src/core/lint/reporter.ts +128 -0
  143. package/src/core/lint/rules/consistency.ts +158 -0
  144. package/src/core/lint/rules/heuristics.ts +97 -0
  145. package/src/core/lint/rules/strictness.ts +109 -0
  146. package/src/core/lint/types.ts +96 -0
  147. package/src/core/lint/walker.ts +248 -0
  148. package/src/core/meta/meta-store.ts +6 -73
  149. package/src/core/output/README.md +91 -0
  150. package/src/core/output/index.ts +13 -0
  151. package/src/core/output/run.ts +126 -0
  152. package/src/core/output/types.ts +129 -0
  153. package/src/core/parser/env-interpolation.ts +104 -0
  154. package/src/core/parser/filter.ts +57 -0
  155. package/src/core/parser/schema.ts +132 -5
  156. package/src/core/parser/types.ts +29 -2
  157. package/src/core/parser/variables.ts +0 -0
  158. package/src/core/parser/yaml-parser.ts +108 -13
  159. package/src/core/probe/bootstrap.ts +34 -0
  160. package/src/core/probe/dry-run-envelope.ts +57 -0
  161. package/src/core/probe/mass-assignment-probe-class.ts +198 -0
  162. package/src/core/probe/mass-assignment-probe.ts +1122 -0
  163. package/src/core/probe/mass-assignment-template.ts +212 -0
  164. package/src/core/probe/method-probe.ts +164 -0
  165. package/src/core/probe/method-shared.ts +69 -0
  166. package/src/core/probe/negative-probe.ts +691 -0
  167. package/src/core/probe/orphan-tracker.ts +188 -0
  168. package/src/core/probe/path-discovery.ts +440 -0
  169. package/src/core/probe/probe-harness.ts +120 -0
  170. package/src/core/probe/registry.ts +89 -0
  171. package/src/core/probe/runner.ts +136 -0
  172. package/src/core/probe/security-probe-class.ts +201 -0
  173. package/src/core/probe/security-probe.ts +1453 -0
  174. package/src/core/probe/shared.ts +505 -0
  175. package/src/core/probe/static-probe-class.ts +125 -0
  176. package/src/core/probe/types.ts +165 -0
  177. package/src/core/probe/verdict-aggregator.ts +33 -0
  178. package/src/core/probe/webhooks-probe.ts +284 -0
  179. package/src/core/reporter/console.ts +69 -4
  180. package/src/core/reporter/index.ts +2 -3
  181. package/src/core/reporter/json.ts +15 -2
  182. package/src/core/reporter/junit.ts +27 -12
  183. package/src/core/reporter/ndjson.ts +37 -0
  184. package/src/core/reporter/types.ts +3 -0
  185. package/src/core/runner/assertions.ts +62 -2
  186. package/src/core/runner/async-pool.ts +108 -0
  187. package/src/core/runner/auth-path.ts +8 -0
  188. package/src/core/runner/ci-context.ts +72 -0
  189. package/src/core/runner/executor.ts +391 -52
  190. package/src/core/runner/form-encode.ts +51 -0
  191. package/src/core/runner/http-client.ts +115 -7
  192. package/src/core/runner/learn-drift.ts +293 -0
  193. package/src/core/runner/preflight-vars.ts +149 -0
  194. package/src/core/runner/progress-tracker.ts +73 -0
  195. package/src/core/runner/rate-limiter.ts +203 -0
  196. package/src/core/runner/run-kind.ts +39 -0
  197. package/src/core/runner/schema-validator.ts +312 -0
  198. package/src/core/runner/send-request.ts +153 -20
  199. package/src/core/runner/types.ts +38 -0
  200. package/src/core/secrets/registry.ts +164 -0
  201. package/src/core/secrets/secrets-file.ts +115 -0
  202. package/src/core/selectors/operation-filter.ts +144 -0
  203. package/src/core/setup-api.ts +419 -17
  204. package/src/core/severity/category.ts +94 -0
  205. package/src/core/severity/index.ts +121 -0
  206. package/src/core/spec/layers.ts +154 -0
  207. package/src/core/util/format-eta.ts +21 -0
  208. package/src/core/utils.ts +5 -1
  209. package/src/core/workspace/config.ts +129 -0
  210. package/src/core/workspace/manifest.ts +283 -0
  211. package/src/core/workspace/output-rotation.ts +62 -0
  212. package/src/core/workspace/root.ts +94 -0
  213. package/src/core/workspace/triage-path.ts +87 -0
  214. package/src/db/lint-runs.ts +47 -0
  215. package/src/db/migrate.ts +126 -0
  216. package/src/db/migrations/0001_run_kind.sql +25 -0
  217. package/src/db/migrations/sql.d.ts +4 -0
  218. package/src/db/queries/collections.ts +133 -0
  219. package/src/db/queries/coverage.ts +9 -0
  220. package/src/db/queries/dashboard.ts +59 -0
  221. package/src/db/queries/results.ts +128 -0
  222. package/src/db/queries/runs.ts +235 -0
  223. package/src/db/queries/sessions.ts +42 -0
  224. package/src/db/queries/settings.ts +28 -0
  225. package/src/db/queries/types.ts +172 -0
  226. package/src/db/queries.ts +72 -802
  227. package/src/db/schema.ts +179 -48
  228. package/src/cli/commands/export.ts +0 -144
  229. package/src/cli/commands/guide.ts +0 -127
  230. package/src/cli/commands/init.ts +0 -57
  231. package/src/cli/commands/serve.ts +0 -81
  232. package/src/cli/commands/sync.ts +0 -269
  233. package/src/cli/commands/update.ts +0 -189
  234. package/src/cli/commands/validate.ts +0 -34
  235. package/src/core/exporter/postman.ts +0 -963
  236. package/src/core/generator/guide-builder.ts +0 -253
  237. package/src/core/meta/types.ts +0 -21
  238. package/src/core/parser/index.ts +0 -21
  239. package/src/core/runner/execute-run.ts +0 -132
  240. package/src/core/runner/index.ts +0 -12
  241. package/src/core/sync/spec-differ.ts +0 -38
  242. package/src/web/data/collection-state.ts +0 -362
  243. package/src/web/routes/api.ts +0 -314
  244. package/src/web/routes/dashboard.ts +0 -350
  245. package/src/web/routes/runs.ts +0 -64
  246. package/src/web/schemas.ts +0 -121
  247. package/src/web/server.ts +0 -134
  248. package/src/web/static/htmx.min.cjs +0 -1
  249. package/src/web/static/style.css +0 -1148
  250. package/src/web/views/endpoints-tab.ts +0 -174
  251. package/src/web/views/explorer-tab.ts +0 -402
  252. package/src/web/views/health-strip.ts +0 -92
  253. package/src/web/views/layout.ts +0 -48
  254. package/src/web/views/results.ts +0 -210
  255. package/src/web/views/runs-tab.ts +0 -126
  256. package/src/web/views/suites-tab.ts +0 -181
@@ -1,11 +1,14 @@
1
1
  import { resolve, dirname, basename } from "node:path";
2
- import type { TestSuite, TestStep, Environment } from "../parser/types.ts";
2
+ import type { TestSuite, TestStep, Environment, SourceMetadata, AssertionRule } from "../parser/types.ts";
3
3
  import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
4
- import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
4
+ import type { TestRunResult, StepResult, HttpRequest, AssertionResult } from "./types.ts";
5
5
  import { executeRequest, type FetchOptions } from "./http-client.ts";
6
- import { checkAssertions, extractCaptures } from "./assertions.ts";
6
+ import type { RateLimiter } from "./rate-limiter.ts";
7
+ import { checkAssertions, extractCaptures, findMissedCaptures } from "./assertions.ts";
7
8
  import { evaluateExpr } from "./expr-eval.ts";
8
9
  import { applyTransform } from "./transforms.ts";
10
+ import type { SchemaValidator } from "./schema-validator.ts";
11
+ import { classifyFailure } from "../diagnostics/failure-class.ts";
9
12
 
10
13
  function buildUrl(baseUrl: string | undefined, path: string, query?: Record<string, string>): string {
11
14
  let url = baseUrl ? `${baseUrl.replace(/\/+$/, "")}${path}` : path;
@@ -16,8 +19,96 @@ function buildUrl(baseUrl: string | undefined, path: string, query?: Record<stri
16
19
  return url;
17
20
  }
18
21
 
19
- function makeSkippedResult(stepName: string, reason: string): StepResult {
22
+ /** Shallow-merge suite-level и step-level provenance. Step перекрывает suite. */
23
+ function mergeProvenance(
24
+ suiteSrc?: SourceMetadata,
25
+ stepSrc?: SourceMetadata,
26
+ ): SourceMetadata | null {
27
+ if (!suiteSrc && !stepSrc) return null;
28
+ return { ...(suiteSrc ?? {}), ...(stepSrc ?? {}) };
29
+ }
30
+
31
+ /** ARV-157: build the top-level `schema_validation` summary for a step that
32
+ * was run with `--validate-schema`. Mirrors the shape `zond request
33
+ * --validate-schema` already produces (see src/cli/commands/request.ts);
34
+ * consumers can `jq '.steps[] | .schema_validation'` instead of digging
35
+ * into `assertions[] | select(.kind=="schema")`.
36
+ *
37
+ * Returns undefined when the validator wasn't attached or the response had
38
+ * no parseable JSON body — same precondition as `assertions.push(...)` at
39
+ * the call site, so the summary is present iff schema actually ran. */
40
+ function buildSchemaValidationSummary(
41
+ validator: SchemaValidator,
42
+ method: string,
43
+ path: string,
44
+ status: number,
45
+ schemaAssertions: AssertionResult[],
46
+ ): StepResult["schema_validation"] {
47
+ const ins = validator.inspect(method, path, status);
48
+ if (!ins.matchedEndpoint) {
49
+ return { result: "no-endpoint", matched_endpoint: null, matched_response_status: null, error_count: 0 };
50
+ }
51
+ if (!ins.hasJsonSchema) {
52
+ return {
53
+ result: "no-schema",
54
+ matched_endpoint: ins.matchedEndpoint,
55
+ matched_response_status: ins.matchedResponseStatus,
56
+ error_count: 0,
57
+ };
58
+ }
59
+ const failed = schemaAssertions.filter((a) => !a.passed).length;
20
60
  return {
61
+ result: failed === 0 ? "PASS" : "FAIL",
62
+ matched_endpoint: ins.matchedEndpoint,
63
+ matched_response_status: ins.matchedResponseStatus,
64
+ error_count: failed,
65
+ };
66
+ }
67
+
68
+ /** TASK-256: turn each missed-capture (path didn't resolve in response)
69
+ * into an auxiliary failed assertion. The step then fails loudly with
70
+ * "capture <var>: path '<path>' not found in body" instead of producing
71
+ * silent `captures: {}` that the user only notices when the next step in a
72
+ * CRUD chain skips with `Depends on missing capture`. */
73
+ function buildMissedCaptureAssertions(
74
+ misses: ReturnType<typeof findMissedCaptures>,
75
+ ): AssertionResult[] {
76
+ return misses.map((m) => ({
77
+ field: `capture ${m.var}`,
78
+ rule: m.source === "body" ? "body-path-exists" : "header-exists",
79
+ passed: false,
80
+ actual: undefined,
81
+ expected: `${m.source} '${m.path}' present`,
82
+ kind: "auxiliary",
83
+ }));
84
+ }
85
+
86
+ function collectChainCaptures(tests: TestStep[]): Set<string> {
87
+ const out = new Set<string>();
88
+ const visit = (rules: Record<string, AssertionRule> | undefined) => {
89
+ if (!rules) return;
90
+ for (const r of Object.values(rules)) {
91
+ if (r.capture) out.add(r.capture);
92
+ if (r.each) visit(r.each);
93
+ if (r.contains_item) visit(r.contains_item);
94
+ }
95
+ };
96
+ for (const step of tests) visit(step.expect?.body);
97
+ return out;
98
+ }
99
+
100
+ function emptyVarSkipReason(varName: string, chainCaptures: Set<string>): string {
101
+ return chainCaptures.has(varName)
102
+ ? `chain capture {{${varName}}} unbound (upstream step did not run or did not capture it)`
103
+ : `required fixture {{${varName}}} is empty`;
104
+ }
105
+
106
+ function makeSkippedResult(
107
+ stepName: string,
108
+ reason: string,
109
+ opts?: { cascade?: { missingCapture: string } },
110
+ ): StepResult {
111
+ const result: StepResult = {
21
112
  name: stepName,
22
113
  status: "skip",
23
114
  duration_ms: 0,
@@ -26,21 +117,145 @@ function makeSkippedResult(stepName: string, reason: string): StepResult {
26
117
  captures: {},
27
118
  error: reason,
28
119
  };
120
+ if (opts?.cascade) {
121
+ result.failure_class = "cascade";
122
+ result.failure_class_reason = `Upstream capture not produced: ${opts.cascade.missingCapture}`;
123
+ }
124
+ return result;
29
125
  }
30
126
 
31
- export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun = false): Promise<TestRunResult> {
127
+ /** Interpolate {{var}} placeholders inside a test/step name. Falls back to
128
+ * the raw name string if substitution returns a non-string value. */
129
+ function interpolateName(name: string, vars: Record<string, unknown>): string {
130
+ try {
131
+ const out = substituteString(name, vars);
132
+ return typeof out === "string" ? out : String(out);
133
+ } catch {
134
+ return name;
135
+ }
136
+ }
137
+
138
+ /**
139
+ * Expand a `parameterize: { key: [val, ...] }` map into the cross-product of
140
+ * iteration variable bindings. No `parameterize` (or an empty map) yields a
141
+ * single empty iteration so the existing single-pass behaviour is preserved.
142
+ *
143
+ * Exported for tests.
144
+ */
145
+ export function expandParameterize(params?: Record<string, unknown[]>): Record<string, unknown>[] {
146
+ if (!params) return [{}];
147
+ const keys = Object.keys(params).filter(k => Array.isArray(params[k]) && (params[k] as unknown[]).length > 0);
148
+ if (keys.length === 0) return [{}];
149
+ let combos: Record<string, unknown>[] = [{}];
150
+ for (const k of keys) {
151
+ const values = params[k] as unknown[];
152
+ const next: Record<string, unknown>[] = [];
153
+ for (const combo of combos) {
154
+ for (const v of values) {
155
+ next.push({ ...combo, [k]: v });
156
+ }
157
+ }
158
+ combos = next;
159
+ }
160
+ return combos;
161
+ }
162
+
163
+ export interface RunSuiteOptions {
164
+ rateLimiter?: RateLimiter;
165
+ /** Optional OpenAPI response-schema validator. When provided, every step's
166
+ * parsed JSON body is validated against the matching schema; failures are
167
+ * appended to the step's `assertions`. */
168
+ schemaValidator?: SchemaValidator;
169
+ /** TASK-144: per-step network-retry budget used by http-client for
170
+ * ECONNRESET / EPIPE / `socket hang up` / `fetch failed` / abort cases.
171
+ * Set by `zond run --retry-on-network <N>`. HTTP statuses are not retried
172
+ * by this path. */
173
+ networkRetries?: number;
174
+ /** ARV-249: shared HTTP-request budget across all parallel suites. When
175
+ * `used >= limit`, remaining steps short-circuit to `skip` with reason
176
+ * `max-requests-cap-reached`. Each `retry_until` attempt counts as one
177
+ * request; dry-run and set-only steps do not consume the budget. */
178
+ requestBudget?: RequestBudget;
179
+ /** ARV-249: invoked after every step completes (pass/fail/skip/error) so
180
+ * the CLI can render a periodic progress line without each suite
181
+ * knowing how many siblings are running in parallel. */
182
+ onStepDone?: (step: StepResult) => void;
183
+ }
184
+
185
+ /** ARV-249: shared `--max-requests` budget. Mutated in-place by every
186
+ * parallel `runSuite` call — single-threaded JS makes the
187
+ * check-then-increment race-free. `limit === Infinity` means "uncapped"
188
+ * (the default). */
189
+ export interface RequestBudget {
190
+ limit: number;
191
+ used: number;
192
+ }
193
+
194
+ /** Try to reserve one HTTP slot from the shared budget. Returns true if
195
+ * the caller may proceed, false if the cap has been reached. */
196
+ export function reserveRequest(budget: RequestBudget | undefined): boolean {
197
+ if (!budget) return true;
198
+ if (budget.used >= budget.limit) return false;
199
+ budget.used += 1;
200
+ return true;
201
+ }
202
+
203
+ export const MAX_REQUESTS_SKIP_REASON = "max-requests-cap-reached";
204
+
205
+ export async function runSuite(
206
+ suite: TestSuite,
207
+ env: Environment = {},
208
+ dryRun = false,
209
+ options: RunSuiteOptions = {},
210
+ ): Promise<TestRunResult> {
32
211
  const startedAt = new Date().toISOString();
33
212
  const steps: StepResult[] = [];
34
- const variables: Record<string, unknown> = { ...env };
35
- const failedCaptures = new Set<string>();
213
+
214
+ /** Push a step result, attaching provenance + failure classification. */
215
+ const pushStep = (result: StepResult, currentStep?: TestStep): void => {
216
+ const merged = mergeProvenance(suite.source, currentStep?.source);
217
+ if (merged !== null) result.provenance = merged;
218
+ const classification = classifyFailure(result);
219
+ if (classification) {
220
+ result.failure_class = classification.failure_class;
221
+ result.failure_class_reason = classification.failure_class_reason;
222
+ }
223
+ steps.push(result);
224
+ if (options.onStepDone) options.onStepDone(result);
225
+ };
36
226
 
37
227
  const fetchOptions: Partial<FetchOptions> = {
38
228
  timeout: suite.config.timeout,
39
229
  retries: suite.config.retries,
40
230
  retry_delay: suite.config.retry_delay,
41
231
  follow_redirects: suite.config.follow_redirects,
232
+ rate_limiter: options.rateLimiter,
233
+ ...(options.networkRetries !== undefined ? { network_retries: options.networkRetries } : {}),
42
234
  };
43
235
 
236
+ // Names of every variable a step in this suite tries to capture from a
237
+ // response (expect.body.<field>.capture: <name>). When a later step
238
+ // references one of these and the value is empty — under --dry-run, or
239
+ // because the capturing step was skipped — the missing var is a chain
240
+ // capture, NOT a fixture in .env.yaml. Distinguishing them in the skip
241
+ // message stops users from chasing fixture seeding for vars that
242
+ // shouldn't live in .env.yaml at all.
243
+ const chainCaptures = collectChainCaptures(suite.tests);
244
+
245
+ // parameterize cross-product → N iterations of the suite body.
246
+ // Captures and tainted/missing sets are reset per iteration so that
247
+ // values from one binding never leak into the next.
248
+ const iterations = expandParameterize(suite.parameterize);
249
+
250
+ for (const iterVars of iterations) {
251
+ const variables: Record<string, unknown> = { ...env, ...iterVars };
252
+ // Captures whose source step's assertions partially failed, but the value
253
+ // itself was extracted. Cleanup/always steps may still consume them.
254
+ const taintedCaptures = new Set<string>();
255
+ // Captures that were never extracted (response missing the field). Even
256
+ // always-steps can't run if their referenced capture is missing.
257
+ const missingCaptures = new Set<string>();
258
+
44
259
  // Expand steps lazily (for_each needs current variables)
45
260
  let stepIndex = 0;
46
261
  const rawSteps = [...suite.tests];
@@ -85,48 +300,113 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
85
300
  const substituted = substituteDeep(rawDirective, variables);
86
301
  variables[key] = applyTransform(substituted);
87
302
  }
88
- steps.push({
89
- name: step.name,
303
+ pushStep({
304
+ name: interpolateName(step.name, variables),
90
305
  status: "pass",
91
306
  duration_ms: 0,
92
307
  request: { method: "", url: "", headers: {} },
93
308
  assertions: [],
94
309
  captures: {},
95
- });
310
+ }, step);
96
311
  continue;
97
312
  }
98
313
 
99
- // Skip check: if step references a failed capture variable, skip it
314
+ // Skip check: if step references a failed capture, skip — unless
315
+ // step is `always: true` AND the capture is just tainted (still extracted).
100
316
  const referencedVars = extractVariableReferences(step);
101
- const missingCapture = referencedVars.find((v) => failedCaptures.has(v));
102
- if (missingCapture) {
103
- steps.push(makeSkippedResult(step.name, `Depends on missing capture: ${missingCapture}`));
317
+ const missing = referencedVars.find((v) => missingCaptures.has(v));
318
+ if (missing) {
319
+ pushStep(
320
+ makeSkippedResult(
321
+ interpolateName(step.name, variables),
322
+ `Depends on missing capture: ${missing}`,
323
+ { cascade: { missingCapture: missing } },
324
+ ),
325
+ step,
326
+ );
104
327
  continue;
105
328
  }
329
+ if (!step.always) {
330
+ const tainted = referencedVars.find((v) => taintedCaptures.has(v));
331
+ if (tainted) {
332
+ pushStep(
333
+ makeSkippedResult(
334
+ interpolateName(step.name, variables),
335
+ `Depends on tainted capture: ${tainted} (use always: true on cleanup steps)`,
336
+ { cascade: { missingCapture: tainted } },
337
+ ),
338
+ step,
339
+ );
340
+ continue;
341
+ }
342
+ }
106
343
 
107
344
  // skip_if evaluation
108
345
  if (step.skip_if) {
109
346
  const exprAfterSubst = String(substituteString(step.skip_if, variables));
110
347
  if (evaluateExpr(exprAfterSubst)) {
111
- steps.push(makeSkippedResult(step.name, `Skipped: ${step.skip_if}`));
348
+ const varMatch = step.skip_if.match(/\{\{([^{}]+)\}\}/);
349
+ const skipMsg = varMatch
350
+ ? emptyVarSkipReason(varMatch[1]!.trim(), chainCaptures)
351
+ : step.skip_if;
352
+ pushStep(makeSkippedResult(interpolateName(step.name, variables), skipMsg), step);
112
353
  continue;
113
354
  }
114
355
  }
115
356
 
116
- // Process set: on HTTP steps — evaluate generators once before building request
117
- if (step.set) {
118
- for (const [key, rawDirective] of Object.entries(step.set)) {
119
- const substituted = substituteDeep(rawDirective, variables);
120
- variables[key] = applyTransform(substituted);
357
+ // Process set: on HTTP steps — evaluate generators once before building request.
358
+ // Substitution can throw on unknown {{$generator}} — fail this step, not the suite.
359
+ let resolved: TestStep;
360
+ let resolvedBaseUrl: string | undefined;
361
+ let resolvedSuiteHeaders: Record<string, string> | undefined;
362
+ try {
363
+ if (step.set) {
364
+ for (const [key, rawDirective] of Object.entries(step.set)) {
365
+ const substituted = substituteDeep(rawDirective, variables);
366
+ variables[key] = applyTransform(substituted);
367
+ }
368
+ }
369
+ resolved = substituteStep(step, variables);
370
+ resolvedBaseUrl = suite.base_url ? substituteString(suite.base_url, variables) as string : undefined;
371
+ resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
372
+ } catch (err) {
373
+ const errorMsg = err instanceof Error ? err.message : String(err);
374
+ pushStep({
375
+ name: interpolateName(step.name, variables),
376
+ status: "error",
377
+ duration_ms: 0,
378
+ request: { method: step.method, url: step.path, headers: {} },
379
+ assertions: [],
380
+ captures: {},
381
+ error: errorMsg,
382
+ }, step);
383
+ // Substitution never produced a request → capture truly missing.
384
+ if (step.expect.body) {
385
+ for (const rule of Object.values(step.expect.body)) {
386
+ if (rule.capture) missingCaptures.add(rule.capture);
387
+ }
388
+ }
389
+ continue;
390
+ }
391
+ // Skip if any path-variable in the template resolved to empty — an empty
392
+ // path segment produces URLs like /repos//commits/ which always 404/500.
393
+ // The explicit skip_if guard only covers the first param (TASK-237);
394
+ // this catches all others.
395
+ {
396
+ let emptyVar: string | null = null;
397
+ for (const m of step.path.matchAll(/\{\{([^{}]+)\}\}/g)) {
398
+ const varName = m[1]!.trim();
399
+ const val = variables[varName];
400
+ if (val === "" || val === null || val === undefined) { emptyVar = varName; break; }
401
+ }
402
+ if (emptyVar) {
403
+ pushStep(makeSkippedResult(
404
+ interpolateName(step.name, variables),
405
+ emptyVarSkipReason(emptyVar, chainCaptures),
406
+ ), step);
407
+ continue;
121
408
  }
122
409
  }
123
-
124
- // Substitute variables
125
- const resolved = substituteStep(step, variables);
126
-
127
- // Build request — substitute base_url and suite headers with current variables
128
- const resolvedBaseUrl = suite.base_url ? substituteString(suite.base_url, variables) as string : undefined;
129
- const resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
130
410
  const url = buildUrl(resolvedBaseUrl, resolved.path, resolved.query);
131
411
  const headers: Record<string, string> = { ...resolvedSuiteHeaders, ...resolved.headers };
132
412
  let body: string | undefined;
@@ -162,18 +442,18 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
162
442
 
163
443
  // Validate absolute URL before attempting fetch
164
444
  if (!url.startsWith("http://") && !url.startsWith("https://")) {
165
- steps.push({
166
- name: step.name,
445
+ pushStep({
446
+ name: interpolateName(step.name, variables),
167
447
  status: "error",
168
448
  duration_ms: 0,
169
449
  request,
170
450
  assertions: [],
171
451
  captures: {},
172
452
  error: `base_url is not configured — URL resolved to a relative path: "${url}". Set base_url in .env.yaml`,
173
- });
453
+ }, step);
174
454
  if (step.expect.body) {
175
455
  for (const rule of Object.values(step.expect.body)) {
176
- if (rule.capture) failedCaptures.add(rule.capture);
456
+ if (rule.capture) missingCaptures.add(rule.capture);
177
457
  }
178
458
  }
179
459
  continue;
@@ -183,15 +463,15 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
183
463
  const bodyPreview = formData
184
464
  ? ` [multipart: ${[...formData.keys()].length} field(s)]`
185
465
  : body ? ` ${body.slice(0, 200)}` : "";
186
- steps.push({
187
- name: step.name,
466
+ pushStep({
467
+ name: interpolateName(step.name, variables),
188
468
  status: "pass",
189
469
  duration_ms: 0,
190
470
  request,
191
471
  assertions: [],
192
472
  captures: {},
193
473
  error: `[DRY RUN] ${resolved.method} ${url}${bodyPreview}`,
194
- });
474
+ }, step);
195
475
  continue;
196
476
  }
197
477
 
@@ -200,20 +480,45 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
200
480
  const rt = step.retry_until;
201
481
  let lastStepResult: StepResult | undefined;
202
482
  for (let attempt = 0; attempt < rt.max_attempts; attempt++) {
483
+ if (!reserveRequest(options.requestBudget)) {
484
+ lastStepResult = makeSkippedResult(
485
+ interpolateName(step.name, variables),
486
+ MAX_REQUESTS_SKIP_REASON,
487
+ );
488
+ break;
489
+ }
203
490
  try {
204
491
  const response = await executeRequest(request, fetchOptions);
205
492
  const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
493
+ const missedCaps = findMissedCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
206
494
  const assertions = checkAssertions(resolved.expect, response);
495
+ assertions.push(...buildMissedCaptureAssertions(missedCaps));
496
+ let schemaValidationSummary: StepResult["schema_validation"] | undefined;
497
+ if (options.schemaValidator && response.body_parsed !== undefined) {
498
+ const schemaAssertions = options.schemaValidator.validate(resolved.method, resolved.path, response.status, response.body_parsed);
499
+ assertions.push(...schemaAssertions);
500
+ schemaValidationSummary = buildSchemaValidationSummary(
501
+ options.schemaValidator,
502
+ resolved.method,
503
+ resolved.path,
504
+ response.status,
505
+ schemaAssertions,
506
+ );
507
+ }
207
508
  const allPassed = assertions.every((a) => a.passed);
208
509
 
209
510
  lastStepResult = {
210
- name: step.name,
511
+ name: interpolateName(step.name, variables),
211
512
  status: allPassed ? "pass" : "fail",
212
513
  duration_ms: response.duration_ms,
213
514
  request,
214
515
  response,
215
516
  assertions,
216
517
  captures,
518
+ ...(response.network_retry_count && response.network_retry_count > 0
519
+ ? { network_retry: response.network_retry_count }
520
+ : {}),
521
+ ...(schemaValidationSummary ? { schema_validation: schemaValidationSummary } : {}),
217
522
  };
218
523
 
219
524
  // Evaluate condition with response context
@@ -234,7 +539,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
234
539
  }
235
540
  } catch (err) {
236
541
  lastStepResult = {
237
- name: step.name,
542
+ name: interpolateName(step.name, variables),
238
543
  status: "error",
239
544
  duration_ms: 0,
240
545
  request,
@@ -244,7 +549,15 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
244
549
  };
245
550
  }
246
551
  }
247
- if (lastStepResult) steps.push(lastStepResult);
552
+ if (lastStepResult) pushStep(lastStepResult, step);
553
+ continue;
554
+ }
555
+
556
+ if (!reserveRequest(options.requestBudget)) {
557
+ pushStep(makeSkippedResult(
558
+ interpolateName(step.name, variables),
559
+ MAX_REQUESTS_SKIP_REASON,
560
+ ), step);
248
561
  continue;
249
562
  }
250
563
 
@@ -255,57 +568,78 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
255
568
  const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
256
569
  Object.assign(variables, captures);
257
570
 
258
- // Track expected captures that weren't obtained
571
+ // Track expected captures that weren't obtained — these are missing.
259
572
  if (resolved.expect.body) {
260
573
  for (const rule of Object.values(resolved.expect.body)) {
261
574
  if (rule.capture && !(rule.capture in captures)) {
262
- failedCaptures.add(rule.capture);
575
+ missingCaptures.add(rule.capture);
263
576
  }
264
577
  }
265
578
  }
266
579
 
267
580
  // Run assertions
581
+ const missedCaps = findMissedCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
268
582
  const assertions = checkAssertions(resolved.expect, response);
583
+ assertions.push(...buildMissedCaptureAssertions(missedCaps));
584
+ let schemaValidationSummary: StepResult["schema_validation"] | undefined;
585
+ if (options.schemaValidator && response.body_parsed !== undefined) {
586
+ const schemaAssertions = options.schemaValidator.validate(resolved.method, resolved.path, response.status, response.body_parsed);
587
+ assertions.push(...schemaAssertions);
588
+ schemaValidationSummary = buildSchemaValidationSummary(
589
+ options.schemaValidator,
590
+ resolved.method,
591
+ resolved.path,
592
+ response.status,
593
+ schemaAssertions,
594
+ );
595
+ }
269
596
  const allPassed = assertions.every((a) => a.passed);
270
597
 
271
- steps.push({
272
- name: step.name,
598
+ pushStep({
599
+ name: interpolateName(step.name, variables),
273
600
  status: allPassed ? "pass" : "fail",
274
601
  duration_ms: response.duration_ms,
275
602
  request,
276
603
  response,
277
604
  assertions,
278
605
  captures,
279
- });
606
+ ...(response.network_retry_count && response.network_retry_count > 0
607
+ ? { network_retry: response.network_retry_count }
608
+ : {}),
609
+ ...(schemaValidationSummary ? { schema_validation: schemaValidationSummary } : {}),
610
+ }, step);
280
611
 
281
- // If step failed, mark its captures as unreliable
612
+ // If step failed, captures that did extract are tainted (value is real
613
+ // but came from a step whose other assertions failed). Always-steps may
614
+ // still consume them; non-always steps cascade-skip.
282
615
  if (!allPassed && resolved.expect.body) {
283
616
  for (const rule of Object.values(resolved.expect.body)) {
284
- if (rule.capture) {
285
- failedCaptures.add(rule.capture);
617
+ if (rule.capture && rule.capture in captures) {
618
+ taintedCaptures.add(rule.capture);
286
619
  }
287
620
  }
288
621
  }
289
622
  } catch (err) {
290
623
  const errorMsg = err instanceof Error ? err.message : String(err);
291
- steps.push({
292
- name: step.name,
624
+ pushStep({
625
+ name: interpolateName(step.name, variables),
293
626
  status: "error",
294
627
  duration_ms: 0,
295
628
  request,
296
629
  assertions: [],
297
630
  captures: {},
298
631
  error: errorMsg,
299
- });
632
+ }, step);
300
633
 
301
- // Mark any captures from this step as failed
634
+ // Network/runtime error no response capture truly missing.
302
635
  if (step.expect.body) {
303
636
  for (const rule of Object.values(step.expect.body)) {
304
- if (rule.capture) failedCaptures.add(rule.capture);
637
+ if (rule.capture) missingCaptures.add(rule.capture);
305
638
  }
306
639
  }
307
640
  }
308
641
  }
642
+ } // end of parameterize iteration loop
309
643
 
310
644
  const finishedAt = new Date().toISOString();
311
645
  return {
@@ -323,6 +657,11 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
323
657
  };
324
658
  }
325
659
 
326
- export async function runSuites(suites: TestSuite[], env: Environment = {}, dryRun = false): Promise<TestRunResult[]> {
327
- return Promise.all(suites.map((suite) => runSuite(suite, env, dryRun)));
660
+ export async function runSuites(
661
+ suites: TestSuite[],
662
+ env: Environment = {},
663
+ dryRun = false,
664
+ options: RunSuiteOptions = {},
665
+ ): Promise<TestRunResult[]> {
666
+ return Promise.all(suites.map((suite) => runSuite(suite, env, dryRun, options)));
328
667
  }
@@ -0,0 +1,51 @@
1
+ /**
2
+ * ARV-149 / ARV-150: encode a nested JS object as
3
+ * `application/x-www-form-urlencoded` using bracket notation — the
4
+ * canonical Stripe / Rails / PHP convention for nested fields:
5
+ *
6
+ * { address: { line1: "x", line2: "y" }, items: [{ id: 1 }, { id: 2 }] }
7
+ * → address[line1]=x&address[line2]=y&items[0][id]=1&items[1][id]=2
8
+ *
9
+ * Shared between `zond request --form`, the YAML runner's `form:` step,
10
+ * and the mass-assignment probe's form-bodied endpoints. The probe-side
11
+ * adoption (ARV-150) is what restores 265 SKIPPED Stripe endpoints.
12
+ */
13
+ function appendFormParam(params: URLSearchParams, key: string, value: unknown): void {
14
+ if (value === null || value === undefined) return;
15
+ if (Array.isArray(value)) {
16
+ for (let i = 0; i < value.length; i++) appendFormParam(params, `${key}[${i}]`, value[i]);
17
+ } else if (typeof value === "object") {
18
+ for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
19
+ appendFormParam(params, `${key}[${k}]`, v);
20
+ }
21
+ } else {
22
+ params.append(key, String(value));
23
+ }
24
+ }
25
+
26
+ export function encodeFormBody(body: Record<string, unknown>): string {
27
+ const params = new URLSearchParams();
28
+ for (const [k, v] of Object.entries(body)) appendFormParam(params, k, v);
29
+ return params.toString();
30
+ }
31
+
32
+ /** Flatten a nested JS object to a `Record<string, string>` using bracket
33
+ * notation, suitable for the YAML runner's `form:` step (which is typed
34
+ * as `Record<string, string>` and serialised via `URLSearchParams`). */
35
+ export function flattenToFormFields(body: unknown): Record<string, string> {
36
+ const out: Record<string, string> = {};
37
+ const walk = (value: unknown, key: string): void => {
38
+ if (value === null || value === undefined) return;
39
+ if (Array.isArray(value)) {
40
+ for (let i = 0; i < value.length; i++) walk(value[i], key ? `${key}[${i}]` : String(i));
41
+ } else if (typeof value === "object") {
42
+ for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
43
+ walk(v, key ? `${key}[${k}]` : k);
44
+ }
45
+ } else if (key) {
46
+ out[key] = String(value);
47
+ }
48
+ };
49
+ if (body && typeof body === "object") walk(body, "");
50
+ return out;
51
+ }