@kirrosh/zond 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/CHANGELOG.md +648 -0
  2. package/README.md +58 -6
  3. package/package.json +9 -6
  4. package/src/cli/argv.ts +122 -0
  5. package/src/cli/commands/add-api.ts +134 -0
  6. package/src/cli/commands/api/annotate/idempotency.ts +59 -0
  7. package/src/cli/commands/api/annotate/index.ts +525 -0
  8. package/src/cli/commands/api/annotate/lifecycle.ts +74 -0
  9. package/src/cli/commands/api/annotate/overlay.ts +206 -0
  10. package/src/cli/commands/api/annotate/pagination.ts +60 -0
  11. package/src/cli/commands/api/annotate/prompts.ts +183 -0
  12. package/src/cli/commands/api/annotate/readback.ts +58 -0
  13. package/src/cli/commands/api/annotate/resources.ts +91 -0
  14. package/src/cli/commands/api/annotate/seed-bodies.ts +61 -0
  15. package/src/cli/commands/audit.ts +480 -0
  16. package/src/cli/commands/bootstrap.ts +710 -0
  17. package/src/cli/commands/catalog.ts +35 -0
  18. package/src/cli/commands/check.ts +348 -0
  19. package/src/cli/commands/checks.ts +756 -0
  20. package/src/cli/commands/ci-init.ts +43 -0
  21. package/src/cli/commands/clean.ts +212 -0
  22. package/src/cli/commands/cleanup.ts +262 -0
  23. package/src/cli/commands/completions.ts +16 -0
  24. package/src/cli/commands/coverage.ts +605 -132
  25. package/src/cli/commands/db.ts +178 -7
  26. package/src/cli/commands/describe.ts +37 -2
  27. package/src/cli/commands/discover.ts +1236 -0
  28. package/src/cli/commands/doctor.ts +607 -0
  29. package/src/cli/commands/fixtures.ts +402 -0
  30. package/src/cli/commands/generate.ts +420 -46
  31. package/src/cli/commands/init/bootstrap.ts +30 -1
  32. package/src/cli/commands/{init.ts → init/index.ts} +99 -5
  33. package/src/cli/commands/init/skills.ts +56 -3
  34. package/src/cli/commands/init/templates/agents.md +65 -61
  35. package/src/cli/commands/init/templates/skills/zond-checks.md +397 -0
  36. package/src/cli/commands/init/templates/skills/zond-triage.md +210 -0
  37. package/src/cli/commands/init/templates/skills/zond.md +592 -125
  38. package/src/cli/commands/init/templates/zond-config.yml +8 -9
  39. package/src/cli/commands/prepare-fixtures.ts +135 -0
  40. package/src/cli/commands/probe/mass-assignment.ts +503 -0
  41. package/src/cli/commands/probe/security.ts +454 -0
  42. package/src/cli/commands/probe/static.ts +255 -0
  43. package/src/cli/commands/probe/webhooks.ts +161 -0
  44. package/src/cli/commands/probe.ts +459 -0
  45. package/src/cli/commands/reference.ts +87 -0
  46. package/src/cli/commands/refresh-api.ts +169 -0
  47. package/src/cli/commands/remove-api.ts +150 -0
  48. package/src/cli/commands/report-bundle.ts +318 -0
  49. package/src/cli/commands/report.ts +241 -0
  50. package/src/cli/commands/request.ts +379 -4
  51. package/src/cli/commands/run.ts +842 -53
  52. package/src/cli/commands/session.ts +244 -0
  53. package/src/cli/commands/use.ts +18 -1
  54. package/src/cli/index.ts +20 -3
  55. package/src/cli/json-envelope.ts +112 -3
  56. package/src/cli/json-schemas.ts +263 -0
  57. package/src/cli/program.ts +198 -635
  58. package/src/cli/resolve.ts +105 -0
  59. package/src/cli/status-filter.ts +124 -0
  60. package/src/cli/util/api-context.ts +85 -0
  61. package/src/cli/version.ts +5 -0
  62. package/src/core/anti-fp/bootstrap.ts +34 -0
  63. package/src/core/anti-fp/index.ts +33 -0
  64. package/src/core/anti-fp/registry.ts +44 -0
  65. package/src/core/anti-fp/rules/baseline-echo.ts +74 -0
  66. package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +52 -0
  67. package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +38 -0
  68. package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +35 -0
  69. package/src/core/anti-fp/rules/schemathesis/index.ts +24 -0
  70. package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +53 -0
  71. package/src/core/anti-fp/rules/subscription-gated/index.ts +11 -0
  72. package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +75 -0
  73. package/src/core/anti-fp/types.ts +68 -0
  74. package/src/core/checks/checks/_crud-helpers.ts +133 -0
  75. package/src/core/checks/checks/_negative_mutator.ts +133 -0
  76. package/src/core/checks/checks/_readback-helpers.ts +133 -0
  77. package/src/core/checks/checks/content_type_conformance.ts +39 -0
  78. package/src/core/checks/checks/cross_call_references.ts +134 -0
  79. package/src/core/checks/checks/ensure_resource_availability.ts +62 -0
  80. package/src/core/checks/checks/idempotency_replay.ts +246 -0
  81. package/src/core/checks/checks/ignored_auth.ts +211 -0
  82. package/src/core/checks/checks/index.ts +65 -0
  83. package/src/core/checks/checks/lifecycle_transitions.ts +273 -0
  84. package/src/core/checks/checks/missing_required_header.ts +40 -0
  85. package/src/core/checks/checks/negative_data_rejection.ts +45 -0
  86. package/src/core/checks/checks/not_a_server_error.ts +27 -0
  87. package/src/core/checks/checks/open_cors_on_sensitive.ts +131 -0
  88. package/src/core/checks/checks/pagination_invariants.ts +238 -0
  89. package/src/core/checks/checks/positive_data_acceptance.ts +36 -0
  90. package/src/core/checks/checks/rate_limit_headers_absent.ts +77 -0
  91. package/src/core/checks/checks/response_headers_conformance.ts +74 -0
  92. package/src/core/checks/checks/response_schema_conformance.ts +30 -0
  93. package/src/core/checks/checks/status_code_conformance.ts +61 -0
  94. package/src/core/checks/checks/unsupported_method.ts +63 -0
  95. package/src/core/checks/checks/use_after_free.ts +78 -0
  96. package/src/core/checks/index.ts +30 -0
  97. package/src/core/checks/mode.ts +79 -0
  98. package/src/core/checks/recommended-action.ts +64 -0
  99. package/src/core/checks/registry.ts +78 -0
  100. package/src/core/checks/runner.ts +874 -0
  101. package/src/core/checks/sarif.ts +230 -0
  102. package/src/core/checks/stateful.ts +121 -0
  103. package/src/core/checks/types.ts +189 -0
  104. package/src/core/classifier/recommended-action.ts +222 -0
  105. package/src/core/context/current.ts +22 -6
  106. package/src/core/context/session.ts +78 -0
  107. package/src/core/coverage/loader.ts +185 -0
  108. package/src/core/coverage/reasons.ts +300 -0
  109. package/src/core/diagnostics/db-analysis.ts +151 -11
  110. package/src/core/diagnostics/failure-class.ts +120 -0
  111. package/src/core/diagnostics/failure-hints.ts +212 -9
  112. package/src/core/diagnostics/spec-pointer.ts +99 -0
  113. package/src/core/diagnostics/suggested-fixes.ts +156 -0
  114. package/src/core/exporter/case-study/index.ts +270 -0
  115. package/src/core/exporter/curl.ts +40 -0
  116. package/src/core/exporter/exporter.ts +48 -0
  117. package/src/core/exporter/html-report/escape.ts +24 -0
  118. package/src/core/exporter/html-report/index.ts +479 -0
  119. package/src/core/exporter/html-report/script.ts +100 -0
  120. package/src/core/exporter/html-report/styles.ts +408 -0
  121. package/src/core/generator/chunker.ts +42 -16
  122. package/src/core/generator/coverage-phase.ts +0 -0
  123. package/src/core/generator/create-body.ts +89 -0
  124. package/src/core/generator/data-factory.ts +445 -19
  125. package/src/core/generator/describe.ts +1 -1
  126. package/src/core/generator/fixtures-builder.ts +325 -0
  127. package/src/core/generator/index.ts +7 -5
  128. package/src/core/generator/openapi-reader.ts +37 -3
  129. package/src/core/generator/path-param-disambig.ts +114 -0
  130. package/src/core/generator/resources-builder.ts +648 -0
  131. package/src/core/generator/schema-utils.ts +11 -3
  132. package/src/core/generator/serializer.ts +103 -13
  133. package/src/core/generator/suite-generator.ts +419 -111
  134. package/src/core/generator/types.ts +8 -0
  135. package/src/core/identity/identity-file.ts +129 -0
  136. package/src/core/lint/affects.ts +28 -0
  137. package/src/core/lint/config.ts +96 -0
  138. package/src/core/lint/format.ts +42 -0
  139. package/src/core/lint/index.ts +94 -0
  140. package/src/core/lint/reporter.ts +128 -0
  141. package/src/core/lint/rules/consistency.ts +158 -0
  142. package/src/core/lint/rules/heuristics.ts +97 -0
  143. package/src/core/lint/rules/strictness.ts +109 -0
  144. package/src/core/lint/types.ts +96 -0
  145. package/src/core/lint/walker.ts +248 -0
  146. package/src/core/meta/meta-store.ts +6 -73
  147. package/src/core/output/README.md +91 -0
  148. package/src/core/output/index.ts +13 -0
  149. package/src/core/output/run.ts +126 -0
  150. package/src/core/output/types.ts +129 -0
  151. package/src/core/parser/env-interpolation.ts +104 -0
  152. package/src/core/parser/filter.ts +57 -0
  153. package/src/core/parser/schema.ts +129 -4
  154. package/src/core/parser/types.ts +19 -1
  155. package/src/core/parser/variables.ts +0 -0
  156. package/src/core/parser/yaml-parser.ts +58 -12
  157. package/src/core/probe/bootstrap.ts +34 -0
  158. package/src/core/probe/dry-run-envelope.ts +57 -0
  159. package/src/core/probe/mass-assignment-probe-class.ts +198 -0
  160. package/src/core/probe/mass-assignment-probe.ts +1122 -0
  161. package/src/core/probe/mass-assignment-template.ts +212 -0
  162. package/src/core/probe/method-probe.ts +43 -76
  163. package/src/core/probe/method-shared.ts +69 -0
  164. package/src/core/probe/negative-probe.ts +183 -149
  165. package/src/core/probe/orphan-tracker.ts +188 -0
  166. package/src/core/probe/path-discovery.ts +440 -0
  167. package/src/core/probe/probe-harness.ts +120 -0
  168. package/src/core/probe/registry.ts +89 -0
  169. package/src/core/probe/runner.ts +136 -0
  170. package/src/core/probe/security-probe-class.ts +201 -0
  171. package/src/core/probe/security-probe.ts +1453 -0
  172. package/src/core/probe/shared.ts +505 -0
  173. package/src/core/probe/static-probe-class.ts +125 -0
  174. package/src/core/probe/types.ts +165 -0
  175. package/src/core/probe/verdict-aggregator.ts +33 -0
  176. package/src/core/probe/webhooks-probe.ts +284 -0
  177. package/src/core/reporter/console.ts +41 -2
  178. package/src/core/reporter/index.ts +2 -3
  179. package/src/core/reporter/json.ts +11 -1
  180. package/src/core/reporter/junit.ts +27 -12
  181. package/src/core/reporter/ndjson.ts +37 -0
  182. package/src/core/reporter/types.ts +3 -0
  183. package/src/core/runner/assertions.ts +58 -1
  184. package/src/core/runner/async-pool.ts +108 -0
  185. package/src/core/runner/auth-path.ts +8 -0
  186. package/src/core/runner/ci-context.ts +72 -0
  187. package/src/core/runner/executor.ts +264 -20
  188. package/src/core/runner/form-encode.ts +51 -0
  189. package/src/core/runner/http-client.ts +75 -2
  190. package/src/core/runner/learn-drift.ts +293 -0
  191. package/src/core/runner/preflight-vars.ts +149 -0
  192. package/src/core/runner/progress-tracker.ts +73 -0
  193. package/src/core/runner/rate-limiter.ts +89 -17
  194. package/src/core/runner/run-kind.ts +39 -0
  195. package/src/core/runner/schema-validator.ts +312 -0
  196. package/src/core/runner/send-request.ts +153 -20
  197. package/src/core/runner/types.ts +38 -0
  198. package/src/core/secrets/registry.ts +164 -0
  199. package/src/core/secrets/secrets-file.ts +115 -0
  200. package/src/core/selectors/operation-filter.ts +144 -0
  201. package/src/core/setup-api.ts +415 -16
  202. package/src/core/severity/category.ts +94 -0
  203. package/src/core/severity/index.ts +121 -0
  204. package/src/core/spec/layers.ts +154 -0
  205. package/src/core/util/format-eta.ts +21 -0
  206. package/src/core/utils.ts +5 -1
  207. package/src/core/workspace/config.ts +129 -0
  208. package/src/core/workspace/manifest.ts +283 -0
  209. package/src/core/workspace/output-rotation.ts +62 -0
  210. package/src/core/workspace/triage-path.ts +87 -0
  211. package/src/db/lint-runs.ts +47 -0
  212. package/src/db/migrate.ts +126 -0
  213. package/src/db/migrations/0001_run_kind.sql +25 -0
  214. package/src/db/migrations/sql.d.ts +4 -0
  215. package/src/db/queries/collections.ts +133 -0
  216. package/src/db/queries/coverage.ts +9 -0
  217. package/src/db/queries/dashboard.ts +59 -0
  218. package/src/db/queries/results.ts +128 -0
  219. package/src/db/queries/runs.ts +235 -0
  220. package/src/db/queries/sessions.ts +42 -0
  221. package/src/db/queries/settings.ts +28 -0
  222. package/src/db/queries/types.ts +172 -0
  223. package/src/db/queries.ts +72 -802
  224. package/src/db/schema.ts +178 -50
  225. package/src/cli/commands/export.ts +0 -144
  226. package/src/cli/commands/guide.ts +0 -127
  227. package/src/cli/commands/init/templates/skills/scenarios.md +0 -97
  228. package/src/cli/commands/probe-methods.ts +0 -108
  229. package/src/cli/commands/probe-validation.ts +0 -124
  230. package/src/cli/commands/serve.ts +0 -114
  231. package/src/cli/commands/sync.ts +0 -268
  232. package/src/cli/commands/update.ts +0 -189
  233. package/src/cli/commands/validate.ts +0 -34
  234. package/src/core/diagnostics/render-md.ts +0 -112
  235. package/src/core/exporter/postman.ts +0 -963
  236. package/src/core/generator/guide-builder.ts +0 -253
  237. package/src/core/meta/types.ts +0 -19
  238. package/src/core/parser/index.ts +0 -21
  239. package/src/core/runner/execute-run.ts +0 -132
  240. package/src/core/runner/index.ts +0 -12
  241. package/src/core/sync/spec-differ.ts +0 -38
  242. package/src/web/data/collection-state.ts +0 -362
  243. package/src/web/routes/api.ts +0 -314
  244. package/src/web/routes/dashboard.ts +0 -350
  245. package/src/web/routes/runs.ts +0 -64
  246. package/src/web/schemas.ts +0 -121
  247. package/src/web/server.ts +0 -134
  248. package/src/web/static/htmx.min.cjs +0 -1
  249. package/src/web/static/style.css +0 -1148
  250. package/src/web/views/endpoints-tab.ts +0 -174
  251. package/src/web/views/explorer-tab.ts +0 -402
  252. package/src/web/views/health-strip.ts +0 -92
  253. package/src/web/views/layout.ts +0 -48
  254. package/src/web/views/results.ts +0 -210
  255. package/src/web/views/runs-tab.ts +0 -126
  256. package/src/web/views/suites-tab.ts +0 -181
@@ -1,12 +1,14 @@
1
1
  import { resolve, dirname, basename } from "node:path";
2
- import type { TestSuite, TestStep, Environment } from "../parser/types.ts";
2
+ import type { TestSuite, TestStep, Environment, SourceMetadata, AssertionRule } from "../parser/types.ts";
3
3
  import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
4
- import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
4
+ import type { TestRunResult, StepResult, HttpRequest, AssertionResult } from "./types.ts";
5
5
  import { executeRequest, type FetchOptions } from "./http-client.ts";
6
6
  import type { RateLimiter } from "./rate-limiter.ts";
7
- import { checkAssertions, extractCaptures } from "./assertions.ts";
7
+ import { checkAssertions, extractCaptures, findMissedCaptures } from "./assertions.ts";
8
8
  import { evaluateExpr } from "./expr-eval.ts";
9
9
  import { applyTransform } from "./transforms.ts";
10
+ import type { SchemaValidator } from "./schema-validator.ts";
11
+ import { classifyFailure } from "../diagnostics/failure-class.ts";
10
12
 
11
13
  function buildUrl(baseUrl: string | undefined, path: string, query?: Record<string, string>): string {
12
14
  let url = baseUrl ? `${baseUrl.replace(/\/+$/, "")}${path}` : path;
@@ -17,8 +19,96 @@ function buildUrl(baseUrl: string | undefined, path: string, query?: Record<stri
17
19
  return url;
18
20
  }
19
21
 
20
- function makeSkippedResult(stepName: string, reason: string): StepResult {
22
+ /** Shallow-merge suite-level и step-level provenance. Step перекрывает suite. */
23
+ function mergeProvenance(
24
+ suiteSrc?: SourceMetadata,
25
+ stepSrc?: SourceMetadata,
26
+ ): SourceMetadata | null {
27
+ if (!suiteSrc && !stepSrc) return null;
28
+ return { ...(suiteSrc ?? {}), ...(stepSrc ?? {}) };
29
+ }
30
+
31
+ /** ARV-157: build the top-level `schema_validation` summary for a step that
32
+ * was run with `--validate-schema`. Mirrors the shape `zond request
33
+ * --validate-schema` already produces (see src/cli/commands/request.ts);
34
+ * consumers can `jq '.steps[] | .schema_validation'` instead of digging
35
+ * into `assertions[] | select(.kind=="schema")`.
36
+ *
37
+ * Returns undefined when the validator wasn't attached or the response had
38
+ * no parseable JSON body — same precondition as `assertions.push(...)` at
39
+ * the call site, so the summary is present iff schema actually ran. */
40
+ function buildSchemaValidationSummary(
41
+ validator: SchemaValidator,
42
+ method: string,
43
+ path: string,
44
+ status: number,
45
+ schemaAssertions: AssertionResult[],
46
+ ): StepResult["schema_validation"] {
47
+ const ins = validator.inspect(method, path, status);
48
+ if (!ins.matchedEndpoint) {
49
+ return { result: "no-endpoint", matched_endpoint: null, matched_response_status: null, error_count: 0 };
50
+ }
51
+ if (!ins.hasJsonSchema) {
52
+ return {
53
+ result: "no-schema",
54
+ matched_endpoint: ins.matchedEndpoint,
55
+ matched_response_status: ins.matchedResponseStatus,
56
+ error_count: 0,
57
+ };
58
+ }
59
+ const failed = schemaAssertions.filter((a) => !a.passed).length;
21
60
  return {
61
+ result: failed === 0 ? "PASS" : "FAIL",
62
+ matched_endpoint: ins.matchedEndpoint,
63
+ matched_response_status: ins.matchedResponseStatus,
64
+ error_count: failed,
65
+ };
66
+ }
67
+
68
+ /** TASK-256: turn each missed-capture (path didn't resolve in response)
69
+ * into an auxiliary failed assertion. The step then fails loudly with
70
+ * "capture <var>: path '<path>' not found in body" instead of producing
71
+ * silent `captures: {}` that the user only notices when the next step in a
72
+ * CRUD chain skips with `Depends on missing capture`. */
73
+ function buildMissedCaptureAssertions(
74
+ misses: ReturnType<typeof findMissedCaptures>,
75
+ ): AssertionResult[] {
76
+ return misses.map((m) => ({
77
+ field: `capture ${m.var}`,
78
+ rule: m.source === "body" ? "body-path-exists" : "header-exists",
79
+ passed: false,
80
+ actual: undefined,
81
+ expected: `${m.source} '${m.path}' present`,
82
+ kind: "auxiliary",
83
+ }));
84
+ }
85
+
86
+ function collectChainCaptures(tests: TestStep[]): Set<string> {
87
+ const out = new Set<string>();
88
+ const visit = (rules: Record<string, AssertionRule> | undefined) => {
89
+ if (!rules) return;
90
+ for (const r of Object.values(rules)) {
91
+ if (r.capture) out.add(r.capture);
92
+ if (r.each) visit(r.each);
93
+ if (r.contains_item) visit(r.contains_item);
94
+ }
95
+ };
96
+ for (const step of tests) visit(step.expect?.body);
97
+ return out;
98
+ }
99
+
100
+ function emptyVarSkipReason(varName: string, chainCaptures: Set<string>): string {
101
+ return chainCaptures.has(varName)
102
+ ? `chain capture {{${varName}}} unbound (upstream step did not run or did not capture it)`
103
+ : `required fixture {{${varName}}} is empty`;
104
+ }
105
+
106
+ function makeSkippedResult(
107
+ stepName: string,
108
+ reason: string,
109
+ opts?: { cascade?: { missingCapture: string } },
110
+ ): StepResult {
111
+ const result: StepResult = {
22
112
  name: stepName,
23
113
  status: "skip",
24
114
  duration_ms: 0,
@@ -27,6 +117,11 @@ function makeSkippedResult(stepName: string, reason: string): StepResult {
27
117
  captures: {},
28
118
  error: reason,
29
119
  };
120
+ if (opts?.cascade) {
121
+ result.failure_class = "cascade";
122
+ result.failure_class_reason = `Upstream capture not produced: ${opts.cascade.missingCapture}`;
123
+ }
124
+ return result;
30
125
  }
31
126
 
32
127
  /** Interpolate {{var}} placeholders inside a test/step name. Falls back to
@@ -67,8 +162,46 @@ export function expandParameterize(params?: Record<string, unknown[]>): Record<s
67
162
 
68
163
  export interface RunSuiteOptions {
69
164
  rateLimiter?: RateLimiter;
165
+ /** Optional OpenAPI response-schema validator. When provided, every step's
166
+ * parsed JSON body is validated against the matching schema; failures are
167
+ * appended to the step's `assertions`. */
168
+ schemaValidator?: SchemaValidator;
169
+ /** TASK-144: per-step network-retry budget used by http-client for
170
+ * ECONNRESET / EPIPE / `socket hang up` / `fetch failed` / abort cases.
171
+ * Set by `zond run --retry-on-network <N>`. HTTP statuses are not retried
172
+ * by this path. */
173
+ networkRetries?: number;
174
+ /** ARV-249: shared HTTP-request budget across all parallel suites. When
175
+ * `used >= limit`, remaining steps short-circuit to `skip` with reason
176
+ * `max-requests-cap-reached`. Each `retry_until` attempt counts as one
177
+ * request; dry-run and set-only steps do not consume the budget. */
178
+ requestBudget?: RequestBudget;
179
+ /** ARV-249: invoked after every step completes (pass/fail/skip/error) so
180
+ * the CLI can render a periodic progress line without each suite
181
+ * knowing how many siblings are running in parallel. */
182
+ onStepDone?: (step: StepResult) => void;
183
+ }
184
+
185
+ /** ARV-249: shared `--max-requests` budget. Mutated in-place by every
186
+ * parallel `runSuite` call — single-threaded JS makes the
187
+ * check-then-increment race-free. `limit === Infinity` means "uncapped"
188
+ * (the default). */
189
+ export interface RequestBudget {
190
+ limit: number;
191
+ used: number;
192
+ }
193
+
194
+ /** Try to reserve one HTTP slot from the shared budget. Returns true if
195
+ * the caller may proceed, false if the cap has been reached. */
196
+ export function reserveRequest(budget: RequestBudget | undefined): boolean {
197
+ if (!budget) return true;
198
+ if (budget.used >= budget.limit) return false;
199
+ budget.used += 1;
200
+ return true;
70
201
  }
71
202
 
203
+ export const MAX_REQUESTS_SKIP_REASON = "max-requests-cap-reached";
204
+
72
205
  export async function runSuite(
73
206
  suite: TestSuite,
74
207
  env: Environment = {},
@@ -78,14 +211,37 @@ export async function runSuite(
78
211
  const startedAt = new Date().toISOString();
79
212
  const steps: StepResult[] = [];
80
213
 
214
+ /** Push a step result, attaching provenance + failure classification. */
215
+ const pushStep = (result: StepResult, currentStep?: TestStep): void => {
216
+ const merged = mergeProvenance(suite.source, currentStep?.source);
217
+ if (merged !== null) result.provenance = merged;
218
+ const classification = classifyFailure(result);
219
+ if (classification) {
220
+ result.failure_class = classification.failure_class;
221
+ result.failure_class_reason = classification.failure_class_reason;
222
+ }
223
+ steps.push(result);
224
+ if (options.onStepDone) options.onStepDone(result);
225
+ };
226
+
81
227
  const fetchOptions: Partial<FetchOptions> = {
82
228
  timeout: suite.config.timeout,
83
229
  retries: suite.config.retries,
84
230
  retry_delay: suite.config.retry_delay,
85
231
  follow_redirects: suite.config.follow_redirects,
86
232
  rate_limiter: options.rateLimiter,
233
+ ...(options.networkRetries !== undefined ? { network_retries: options.networkRetries } : {}),
87
234
  };
88
235
 
236
+ // Names of every variable a step in this suite tries to capture from a
237
+ // response (expect.body.<field>.capture: <name>). When a later step
238
+ // references one of these and the value is empty — under --dry-run, or
239
+ // because the capturing step was skipped — the missing var is a chain
240
+ // capture, NOT a fixture in .env.yaml. Distinguishing them in the skip
241
+ // message stops users from chasing fixture seeding for vars that
242
+ // shouldn't live in .env.yaml at all.
243
+ const chainCaptures = collectChainCaptures(suite.tests);
244
+
89
245
  // parameterize cross-product → N iterations of the suite body.
90
246
  // Captures and tainted/missing sets are reset per iteration so that
91
247
  // values from one binding never leak into the next.
@@ -144,14 +300,14 @@ export async function runSuite(
144
300
  const substituted = substituteDeep(rawDirective, variables);
145
301
  variables[key] = applyTransform(substituted);
146
302
  }
147
- steps.push({
303
+ pushStep({
148
304
  name: interpolateName(step.name, variables),
149
305
  status: "pass",
150
306
  duration_ms: 0,
151
307
  request: { method: "", url: "", headers: {} },
152
308
  assertions: [],
153
309
  captures: {},
154
- });
310
+ }, step);
155
311
  continue;
156
312
  }
157
313
 
@@ -160,13 +316,27 @@ export async function runSuite(
160
316
  const referencedVars = extractVariableReferences(step);
161
317
  const missing = referencedVars.find((v) => missingCaptures.has(v));
162
318
  if (missing) {
163
- steps.push(makeSkippedResult(interpolateName(step.name, variables), `Depends on missing capture: ${missing}`));
319
+ pushStep(
320
+ makeSkippedResult(
321
+ interpolateName(step.name, variables),
322
+ `Depends on missing capture: ${missing}`,
323
+ { cascade: { missingCapture: missing } },
324
+ ),
325
+ step,
326
+ );
164
327
  continue;
165
328
  }
166
329
  if (!step.always) {
167
330
  const tainted = referencedVars.find((v) => taintedCaptures.has(v));
168
331
  if (tainted) {
169
- steps.push(makeSkippedResult(interpolateName(step.name, variables), `Depends on tainted capture: ${tainted} (use always: true on cleanup steps)`));
332
+ pushStep(
333
+ makeSkippedResult(
334
+ interpolateName(step.name, variables),
335
+ `Depends on tainted capture: ${tainted} (use always: true on cleanup steps)`,
336
+ { cascade: { missingCapture: tainted } },
337
+ ),
338
+ step,
339
+ );
170
340
  continue;
171
341
  }
172
342
  }
@@ -175,7 +345,11 @@ export async function runSuite(
175
345
  if (step.skip_if) {
176
346
  const exprAfterSubst = String(substituteString(step.skip_if, variables));
177
347
  if (evaluateExpr(exprAfterSubst)) {
178
- steps.push(makeSkippedResult(interpolateName(step.name, variables), `Skipped: ${step.skip_if}`));
348
+ const varMatch = step.skip_if.match(/\{\{([^{}]+)\}\}/);
349
+ const skipMsg = varMatch
350
+ ? emptyVarSkipReason(varMatch[1]!.trim(), chainCaptures)
351
+ : step.skip_if;
352
+ pushStep(makeSkippedResult(interpolateName(step.name, variables), skipMsg), step);
179
353
  continue;
180
354
  }
181
355
  }
@@ -197,7 +371,7 @@ export async function runSuite(
197
371
  resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
198
372
  } catch (err) {
199
373
  const errorMsg = err instanceof Error ? err.message : String(err);
200
- steps.push({
374
+ pushStep({
201
375
  name: interpolateName(step.name, variables),
202
376
  status: "error",
203
377
  duration_ms: 0,
@@ -205,7 +379,7 @@ export async function runSuite(
205
379
  assertions: [],
206
380
  captures: {},
207
381
  error: errorMsg,
208
- });
382
+ }, step);
209
383
  // Substitution never produced a request → capture truly missing.
210
384
  if (step.expect.body) {
211
385
  for (const rule of Object.values(step.expect.body)) {
@@ -214,6 +388,25 @@ export async function runSuite(
214
388
  }
215
389
  continue;
216
390
  }
391
+ // Skip if any path-variable in the template resolved to empty — an empty
392
+ // path segment produces URLs like /repos//commits/ which always 404/500.
393
+ // The explicit skip_if guard only covers the first param (TASK-237);
394
+ // this catches all others.
395
+ {
396
+ let emptyVar: string | null = null;
397
+ for (const m of step.path.matchAll(/\{\{([^{}]+)\}\}/g)) {
398
+ const varName = m[1]!.trim();
399
+ const val = variables[varName];
400
+ if (val === "" || val === null || val === undefined) { emptyVar = varName; break; }
401
+ }
402
+ if (emptyVar) {
403
+ pushStep(makeSkippedResult(
404
+ interpolateName(step.name, variables),
405
+ emptyVarSkipReason(emptyVar, chainCaptures),
406
+ ), step);
407
+ continue;
408
+ }
409
+ }
217
410
  const url = buildUrl(resolvedBaseUrl, resolved.path, resolved.query);
218
411
  const headers: Record<string, string> = { ...resolvedSuiteHeaders, ...resolved.headers };
219
412
  let body: string | undefined;
@@ -249,7 +442,7 @@ export async function runSuite(
249
442
 
250
443
  // Validate absolute URL before attempting fetch
251
444
  if (!url.startsWith("http://") && !url.startsWith("https://")) {
252
- steps.push({
445
+ pushStep({
253
446
  name: interpolateName(step.name, variables),
254
447
  status: "error",
255
448
  duration_ms: 0,
@@ -257,7 +450,7 @@ export async function runSuite(
257
450
  assertions: [],
258
451
  captures: {},
259
452
  error: `base_url is not configured — URL resolved to a relative path: "${url}". Set base_url in .env.yaml`,
260
- });
453
+ }, step);
261
454
  if (step.expect.body) {
262
455
  for (const rule of Object.values(step.expect.body)) {
263
456
  if (rule.capture) missingCaptures.add(rule.capture);
@@ -270,7 +463,7 @@ export async function runSuite(
270
463
  const bodyPreview = formData
271
464
  ? ` [multipart: ${[...formData.keys()].length} field(s)]`
272
465
  : body ? ` ${body.slice(0, 200)}` : "";
273
- steps.push({
466
+ pushStep({
274
467
  name: interpolateName(step.name, variables),
275
468
  status: "pass",
276
469
  duration_ms: 0,
@@ -278,7 +471,7 @@ export async function runSuite(
278
471
  assertions: [],
279
472
  captures: {},
280
473
  error: `[DRY RUN] ${resolved.method} ${url}${bodyPreview}`,
281
- });
474
+ }, step);
282
475
  continue;
283
476
  }
284
477
 
@@ -287,10 +480,31 @@ export async function runSuite(
287
480
  const rt = step.retry_until;
288
481
  let lastStepResult: StepResult | undefined;
289
482
  for (let attempt = 0; attempt < rt.max_attempts; attempt++) {
483
+ if (!reserveRequest(options.requestBudget)) {
484
+ lastStepResult = makeSkippedResult(
485
+ interpolateName(step.name, variables),
486
+ MAX_REQUESTS_SKIP_REASON,
487
+ );
488
+ break;
489
+ }
290
490
  try {
291
491
  const response = await executeRequest(request, fetchOptions);
292
492
  const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
493
+ const missedCaps = findMissedCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
293
494
  const assertions = checkAssertions(resolved.expect, response);
495
+ assertions.push(...buildMissedCaptureAssertions(missedCaps));
496
+ let schemaValidationSummary: StepResult["schema_validation"] | undefined;
497
+ if (options.schemaValidator && response.body_parsed !== undefined) {
498
+ const schemaAssertions = options.schemaValidator.validate(resolved.method, resolved.path, response.status, response.body_parsed);
499
+ assertions.push(...schemaAssertions);
500
+ schemaValidationSummary = buildSchemaValidationSummary(
501
+ options.schemaValidator,
502
+ resolved.method,
503
+ resolved.path,
504
+ response.status,
505
+ schemaAssertions,
506
+ );
507
+ }
294
508
  const allPassed = assertions.every((a) => a.passed);
295
509
 
296
510
  lastStepResult = {
@@ -301,6 +515,10 @@ export async function runSuite(
301
515
  response,
302
516
  assertions,
303
517
  captures,
518
+ ...(response.network_retry_count && response.network_retry_count > 0
519
+ ? { network_retry: response.network_retry_count }
520
+ : {}),
521
+ ...(schemaValidationSummary ? { schema_validation: schemaValidationSummary } : {}),
304
522
  };
305
523
 
306
524
  // Evaluate condition with response context
@@ -331,7 +549,15 @@ export async function runSuite(
331
549
  };
332
550
  }
333
551
  }
334
- if (lastStepResult) steps.push(lastStepResult);
552
+ if (lastStepResult) pushStep(lastStepResult, step);
553
+ continue;
554
+ }
555
+
556
+ if (!reserveRequest(options.requestBudget)) {
557
+ pushStep(makeSkippedResult(
558
+ interpolateName(step.name, variables),
559
+ MAX_REQUESTS_SKIP_REASON,
560
+ ), step);
335
561
  continue;
336
562
  }
337
563
 
@@ -352,10 +578,24 @@ export async function runSuite(
352
578
  }
353
579
 
354
580
  // Run assertions
581
+ const missedCaps = findMissedCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
355
582
  const assertions = checkAssertions(resolved.expect, response);
583
+ assertions.push(...buildMissedCaptureAssertions(missedCaps));
584
+ let schemaValidationSummary: StepResult["schema_validation"] | undefined;
585
+ if (options.schemaValidator && response.body_parsed !== undefined) {
586
+ const schemaAssertions = options.schemaValidator.validate(resolved.method, resolved.path, response.status, response.body_parsed);
587
+ assertions.push(...schemaAssertions);
588
+ schemaValidationSummary = buildSchemaValidationSummary(
589
+ options.schemaValidator,
590
+ resolved.method,
591
+ resolved.path,
592
+ response.status,
593
+ schemaAssertions,
594
+ );
595
+ }
356
596
  const allPassed = assertions.every((a) => a.passed);
357
597
 
358
- steps.push({
598
+ pushStep({
359
599
  name: interpolateName(step.name, variables),
360
600
  status: allPassed ? "pass" : "fail",
361
601
  duration_ms: response.duration_ms,
@@ -363,7 +603,11 @@ export async function runSuite(
363
603
  response,
364
604
  assertions,
365
605
  captures,
366
- });
606
+ ...(response.network_retry_count && response.network_retry_count > 0
607
+ ? { network_retry: response.network_retry_count }
608
+ : {}),
609
+ ...(schemaValidationSummary ? { schema_validation: schemaValidationSummary } : {}),
610
+ }, step);
367
611
 
368
612
  // If step failed, captures that did extract are tainted (value is real
369
613
  // but came from a step whose other assertions failed). Always-steps may
@@ -377,7 +621,7 @@ export async function runSuite(
377
621
  }
378
622
  } catch (err) {
379
623
  const errorMsg = err instanceof Error ? err.message : String(err);
380
- steps.push({
624
+ pushStep({
381
625
  name: interpolateName(step.name, variables),
382
626
  status: "error",
383
627
  duration_ms: 0,
@@ -385,7 +629,7 @@ export async function runSuite(
385
629
  assertions: [],
386
630
  captures: {},
387
631
  error: errorMsg,
388
- });
632
+ }, step);
389
633
 
390
634
  // Network/runtime error → no response → capture truly missing.
391
635
  if (step.expect.body) {
@@ -0,0 +1,51 @@
1
+ /**
2
+ * ARV-149 / ARV-150: encode a nested JS object as
3
+ * `application/x-www-form-urlencoded` using bracket notation — the
4
+ * canonical Stripe / Rails / PHP convention for nested fields:
5
+ *
6
+ * { address: { line1: "x", line2: "y" }, items: [{ id: 1 }, { id: 2 }] }
7
+ * → address[line1]=x&address[line2]=y&items[0][id]=1&items[1][id]=2
8
+ *
9
+ * Shared between `zond request --form`, the YAML runner's `form:` step,
10
+ * and the mass-assignment probe's form-bodied endpoints. The probe-side
11
+ * adoption (ARV-150) is what restores 265 SKIPPED Stripe endpoints.
12
+ */
13
+ function appendFormParam(params: URLSearchParams, key: string, value: unknown): void {
14
+ if (value === null || value === undefined) return;
15
+ if (Array.isArray(value)) {
16
+ for (let i = 0; i < value.length; i++) appendFormParam(params, `${key}[${i}]`, value[i]);
17
+ } else if (typeof value === "object") {
18
+ for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
19
+ appendFormParam(params, `${key}[${k}]`, v);
20
+ }
21
+ } else {
22
+ params.append(key, String(value));
23
+ }
24
+ }
25
+
26
+ export function encodeFormBody(body: Record<string, unknown>): string {
27
+ const params = new URLSearchParams();
28
+ for (const [k, v] of Object.entries(body)) appendFormParam(params, k, v);
29
+ return params.toString();
30
+ }
31
+
32
+ /** Flatten a nested JS object to a `Record<string, string>` using bracket
33
+ * notation, suitable for the YAML runner's `form:` step (which is typed
34
+ * as `Record<string, string>` and serialised via `URLSearchParams`). */
35
+ export function flattenToFormFields(body: unknown): Record<string, string> {
36
+ const out: Record<string, string> = {};
37
+ const walk = (value: unknown, key: string): void => {
38
+ if (value === null || value === undefined) return;
39
+ if (Array.isArray(value)) {
40
+ for (let i = 0; i < value.length; i++) walk(value[i], key ? `${key}[${i}]` : String(i));
41
+ } else if (typeof value === "object") {
42
+ for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
43
+ walk(v, key ? `${key}[${k}]` : k);
44
+ }
45
+ } else if (key) {
46
+ out[key] = String(value);
47
+ }
48
+ };
49
+ if (body && typeof body === "object") walk(body, "");
50
+ return out;
51
+ }
@@ -9,17 +9,69 @@ export interface FetchOptions {
9
9
  rate_limiter?: RateLimiter;
10
10
  rate_limit_retries: number;
11
11
  rate_limit_max_delay_ms: number;
12
+ /** TASK-144: number of network-level retries (ECONNRESET, EPIPE, socket hang
13
+ * up, fetch failed without HTTP response, timeout without response). HTTP
14
+ * status codes are NEVER retried by this path. Exponential backoff with
15
+ * jitter, base = `network_retry_base_ms`. Default 0 (CLI sets it to 1). */
16
+ network_retries: number;
17
+ network_retry_base_ms: number;
18
+ network_retry_max_delay_ms: number;
12
19
  }
13
20
 
14
- export const DEFAULT_FETCH_OPTIONS: FetchOptions = {
21
+ const DEFAULT_FETCH_OPTIONS: FetchOptions = {
15
22
  timeout: 30000,
16
23
  retries: 0,
17
24
  retry_delay: 1000,
18
25
  follow_redirects: true,
19
26
  rate_limit_retries: 5,
20
27
  rate_limit_max_delay_ms: 30000,
28
+ network_retries: 0,
29
+ network_retry_base_ms: 250,
30
+ network_retry_max_delay_ms: 8000,
21
31
  };
22
32
 
33
+ /**
34
+ * Recognise transient TCP/transport-level errors that warrant a retry. We
35
+ * deliberately do NOT include HTTP status codes — a 5xx is a real response
36
+ * the server chose to send, not a flaky socket. Patterns cover Node/Bun
37
+ * error codes (`ECONNRESET`, `EPIPE`, `ECONNREFUSED`, `ETIMEDOUT`,
38
+ * `EAI_AGAIN`), the WHATWG `fetch failed` wrapper Bun throws, classic
39
+ * `socket hang up`, and `AbortError` raised by our own timeout watchdog.
40
+ */
41
+ export function isTransientNetworkError(err: unknown): boolean {
42
+ if (!err) return false;
43
+ const e = err as { code?: string; cause?: unknown; name?: string; message?: string };
44
+ const code = e.code ?? (e.cause as { code?: string } | undefined)?.code;
45
+ if (code) {
46
+ if (
47
+ code === "ECONNRESET" ||
48
+ code === "EPIPE" ||
49
+ code === "ECONNREFUSED" ||
50
+ code === "ETIMEDOUT" ||
51
+ code === "EAI_AGAIN" ||
52
+ code === "ENOTFOUND" ||
53
+ code === "ENETUNREACH"
54
+ ) {
55
+ return true;
56
+ }
57
+ }
58
+ const msg = (e.message ?? String(err)).toLowerCase();
59
+ if (e.name === "AbortError" || msg.includes("aborted")) return true;
60
+ if (msg.includes("socket hang up")) return true;
61
+ if (msg.includes("fetch failed")) return true;
62
+ if (msg.includes("connection reset") || msg.includes("econnreset")) return true;
63
+ if (msg.includes("epipe")) return true;
64
+ if (msg.includes("network error")) return true;
65
+ return false;
66
+ }
67
+
68
+ /** Exponential backoff with full jitter (AWS-style): pick uniformly in
69
+ * [0, min(cap, base * 2^attempt)). Returns ms. */
70
+ export function networkBackoffMs(attempt: number, baseMs: number, capMs: number): number {
71
+ const exp = Math.min(capMs, baseMs * 2 ** attempt);
72
+ return Math.floor(Math.random() * exp);
73
+ }
74
+
23
75
  export async function executeRequest(
24
76
  request: HttpRequest,
25
77
  options?: Partial<FetchOptions>,
@@ -27,6 +79,7 @@ export async function executeRequest(
27
79
  const opts = { ...DEFAULT_FETCH_OPTIONS, ...options };
28
80
  let lastError: Error | undefined;
29
81
  let networkAttempt = 0;
82
+ let networkRetryCount = 0;
30
83
  let rate429Attempt = 0;
31
84
 
32
85
  while (true) {
@@ -100,9 +153,29 @@ export async function executeRequest(
100
153
  }
101
154
  }
102
155
 
103
- return { status: response.status, headers, body: bodyText, body_parsed, duration_ms };
156
+ return {
157
+ status: response.status,
158
+ headers,
159
+ body: bodyText,
160
+ body_parsed,
161
+ duration_ms,
162
+ network_retry_count: networkRetryCount,
163
+ };
104
164
  } catch (err) {
105
165
  lastError = err instanceof Error ? err : new Error(String(err));
166
+ const isNet = isTransientNetworkError(lastError);
167
+ // TASK-144 path: dedicated network-retry budget with exp+jitter backoff.
168
+ if (isNet && networkRetryCount < opts.network_retries) {
169
+ const wait = networkBackoffMs(
170
+ networkRetryCount,
171
+ opts.network_retry_base_ms,
172
+ opts.network_retry_max_delay_ms,
173
+ );
174
+ networkRetryCount++;
175
+ await Bun.sleep(wait);
176
+ continue;
177
+ }
178
+ // Legacy linear path (yaml suite.config.retries).
106
179
  if (networkAttempt < opts.retries) {
107
180
  networkAttempt++;
108
181
  await Bun.sleep(opts.retry_delay);