@kirrosh/zond 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +22 -21
  3. package/bin/zond.mjs +23 -0
  4. package/package.json +13 -16
  5. package/scripts/npm/postinstall.mjs +76 -0
  6. package/src/CLAUDE.md +0 -112
  7. package/src/bun-types.d.ts +0 -5
  8. package/src/cli/argv.ts +0 -122
  9. package/src/cli/commands/add-api.ts +0 -146
  10. package/src/cli/commands/api/annotate/idempotency.ts +0 -59
  11. package/src/cli/commands/api/annotate/index.ts +0 -880
  12. package/src/cli/commands/api/annotate/lifecycle.ts +0 -74
  13. package/src/cli/commands/api/annotate/overlay.ts +0 -206
  14. package/src/cli/commands/api/annotate/pagination.ts +0 -64
  15. package/src/cli/commands/api/annotate/prompts.ts +0 -220
  16. package/src/cli/commands/api/annotate/readback.ts +0 -58
  17. package/src/cli/commands/api/annotate/resources.ts +0 -91
  18. package/src/cli/commands/api/annotate/seed-bodies.ts +0 -61
  19. package/src/cli/commands/audit.ts +0 -786
  20. package/src/cli/commands/catalog.ts +0 -97
  21. package/src/cli/commands/check.ts +0 -361
  22. package/src/cli/commands/checks.ts +0 -1072
  23. package/src/cli/commands/ci-init.ts +0 -223
  24. package/src/cli/commands/clean.ts +0 -212
  25. package/src/cli/commands/cleanup.ts +0 -236
  26. package/src/cli/commands/completions.ts +0 -192
  27. package/src/cli/commands/coverage.ts +0 -872
  28. package/src/cli/commands/db.ts +0 -611
  29. package/src/cli/commands/describe.ts +0 -120
  30. package/src/cli/commands/discover.ts +0 -1356
  31. package/src/cli/commands/doctor.ts +0 -661
  32. package/src/cli/commands/fixtures.ts +0 -402
  33. package/src/cli/commands/generate.ts +0 -577
  34. package/src/cli/commands/init/agents-md.ts +0 -61
  35. package/src/cli/commands/init/bootstrap.ts +0 -111
  36. package/src/cli/commands/init/index.ts +0 -244
  37. package/src/cli/commands/init/skills.ts +0 -141
  38. package/src/cli/commands/init/templates/agents.md +0 -86
  39. package/src/cli/commands/init/templates/markdown.d.ts +0 -4
  40. package/src/cli/commands/init/templates/skills/warm-up-target.md +0 -122
  41. package/src/cli/commands/init/templates/skills/zond-checks.md +0 -621
  42. package/src/cli/commands/init/templates/skills/zond-seed.md +0 -114
  43. package/src/cli/commands/init/templates/skills/zond-triage.md +0 -272
  44. package/src/cli/commands/init/templates/skills/zond.md +0 -861
  45. package/src/cli/commands/init/templates/zond-config.yml +0 -14
  46. package/src/cli/commands/prepare-fixtures.ts +0 -97
  47. package/src/cli/commands/probe/_seed-bodies.ts +0 -52
  48. package/src/cli/commands/probe/mass-assignment.ts +0 -594
  49. package/src/cli/commands/probe/security.ts +0 -537
  50. package/src/cli/commands/probe/static.ts +0 -255
  51. package/src/cli/commands/probe/webhooks.ts +0 -163
  52. package/src/cli/commands/probe.ts +0 -535
  53. package/src/cli/commands/reference.ts +0 -87
  54. package/src/cli/commands/refresh-api.ts +0 -227
  55. package/src/cli/commands/remove-api.ts +0 -150
  56. package/src/cli/commands/report-bundle.ts +0 -310
  57. package/src/cli/commands/report.ts +0 -241
  58. package/src/cli/commands/request.ts +0 -548
  59. package/src/cli/commands/run.ts +0 -1142
  60. package/src/cli/commands/schema-from-runs.ts +0 -128
  61. package/src/cli/commands/secrets.ts +0 -133
  62. package/src/cli/commands/session.ts +0 -244
  63. package/src/cli/commands/use.ts +0 -74
  64. package/src/cli/index.ts +0 -55
  65. package/src/cli/json-envelope.ts +0 -108
  66. package/src/cli/json-schemas.ts +0 -314
  67. package/src/cli/output.ts +0 -40
  68. package/src/cli/program.ts +0 -219
  69. package/src/cli/resolve.ts +0 -105
  70. package/src/cli/runtime.ts +0 -7
  71. package/src/cli/safe-live.ts +0 -24
  72. package/src/cli/status-filter.ts +0 -114
  73. package/src/cli/util/api-context.ts +0 -85
  74. package/src/cli/version.ts +0 -8
  75. package/src/core/audit/persist.ts +0 -183
  76. package/src/core/checks/budget.ts +0 -59
  77. package/src/core/checks/checks/_crud-helpers.ts +0 -133
  78. package/src/core/checks/checks/_negative_mutator.ts +0 -133
  79. package/src/core/checks/checks/_readback-helpers.ts +0 -133
  80. package/src/core/checks/checks/content_type_conformance.ts +0 -39
  81. package/src/core/checks/checks/cross_call_references.ts +0 -147
  82. package/src/core/checks/checks/cursor_boundary_fuzzing.ts +0 -219
  83. package/src/core/checks/checks/ensure_resource_availability.ts +0 -62
  84. package/src/core/checks/checks/idempotency_replay.ts +0 -242
  85. package/src/core/checks/checks/ignored_auth.ts +0 -254
  86. package/src/core/checks/checks/index.ts +0 -68
  87. package/src/core/checks/checks/lifecycle_transitions.ts +0 -416
  88. package/src/core/checks/checks/missing_required_header.ts +0 -40
  89. package/src/core/checks/checks/negative_data_rejection.ts +0 -148
  90. package/src/core/checks/checks/not_a_server_error.ts +0 -35
  91. package/src/core/checks/checks/open_cors_on_sensitive.ts +0 -160
  92. package/src/core/checks/checks/pagination_invariants.ts +0 -419
  93. package/src/core/checks/checks/positive_data_acceptance.ts +0 -33
  94. package/src/core/checks/checks/rate_limit_headers_absent.ts +0 -77
  95. package/src/core/checks/checks/response_headers_conformance.ts +0 -74
  96. package/src/core/checks/checks/response_schema_conformance.ts +0 -30
  97. package/src/core/checks/checks/status_code_conformance.ts +0 -132
  98. package/src/core/checks/checks/unsupported_method.ts +0 -63
  99. package/src/core/checks/checks/use_after_free.ts +0 -78
  100. package/src/core/checks/index.ts +0 -30
  101. package/src/core/checks/mode.ts +0 -82
  102. package/src/core/checks/recommended-action.ts +0 -68
  103. package/src/core/checks/registry.ts +0 -78
  104. package/src/core/checks/runner.ts +0 -1461
  105. package/src/core/checks/sarif.ts +0 -230
  106. package/src/core/checks/spec-findings.ts +0 -308
  107. package/src/core/checks/stateful.ts +0 -121
  108. package/src/core/checks/types.ts +0 -305
  109. package/src/core/checks/zond-extensions.ts +0 -73
  110. package/src/core/classifier/recommended-action.ts +0 -251
  111. package/src/core/context/current.ts +0 -51
  112. package/src/core/context/session.ts +0 -78
  113. package/src/core/coverage/loader.ts +0 -216
  114. package/src/core/coverage/reasons.ts +0 -300
  115. package/src/core/diagnostics/db-analysis.ts +0 -666
  116. package/src/core/diagnostics/failure-class.ts +0 -140
  117. package/src/core/diagnostics/failure-hints.ts +0 -112
  118. package/src/core/diagnostics/spec-pointer.ts +0 -99
  119. package/src/core/diagnostics/suggested-fixes.ts +0 -155
  120. package/src/core/exporter/case-study/index.ts +0 -270
  121. package/src/core/exporter/curl.ts +0 -40
  122. package/src/core/exporter/exporter.ts +0 -48
  123. package/src/core/exporter/html-report/escape.ts +0 -24
  124. package/src/core/exporter/html-report/index.ts +0 -479
  125. package/src/core/exporter/html-report/script.ts +0 -100
  126. package/src/core/exporter/html-report/styles.ts +0 -408
  127. package/src/core/generator/catalog-builder.ts +0 -179
  128. package/src/core/generator/chunker.ts +0 -78
  129. package/src/core/generator/coverage-phase.ts +0 -0
  130. package/src/core/generator/coverage-scanner.ts +0 -87
  131. package/src/core/generator/data-factory.ts +0 -759
  132. package/src/core/generator/describe.ts +0 -302
  133. package/src/core/generator/endpoint-warnings.ts +0 -52
  134. package/src/core/generator/fixtures-builder.ts +0 -332
  135. package/src/core/generator/index.ts +0 -14
  136. package/src/core/generator/openapi-reader.ts +0 -297
  137. package/src/core/generator/path-param-disambig.ts +0 -140
  138. package/src/core/generator/resources-builder.ts +0 -898
  139. package/src/core/generator/schema-utils.ts +0 -112
  140. package/src/core/generator/serializer.ts +0 -343
  141. package/src/core/generator/suite-generator.ts +0 -1301
  142. package/src/core/generator/types.ts +0 -63
  143. package/src/core/identity/identity-file.ts +0 -0
  144. package/src/core/lint/affects.ts +0 -28
  145. package/src/core/lint/config.ts +0 -96
  146. package/src/core/lint/format.ts +0 -42
  147. package/src/core/lint/index.ts +0 -94
  148. package/src/core/lint/reporter.ts +0 -128
  149. package/src/core/lint/rules/consistency.ts +0 -158
  150. package/src/core/lint/rules/heuristics.ts +0 -97
  151. package/src/core/lint/rules/strictness.ts +0 -109
  152. package/src/core/lint/types.ts +0 -96
  153. package/src/core/lint/walker.ts +0 -248
  154. package/src/core/meta/meta-store.ts +0 -11
  155. package/src/core/output/README.md +0 -73
  156. package/src/core/output/index.ts +0 -13
  157. package/src/core/output/run.ts +0 -91
  158. package/src/core/output/types.ts +0 -122
  159. package/src/core/parser/dynamic-values.ts +0 -160
  160. package/src/core/parser/env-interpolation.ts +0 -104
  161. package/src/core/parser/filter.ts +0 -97
  162. package/src/core/parser/schema.ts +0 -359
  163. package/src/core/parser/types.ts +0 -117
  164. package/src/core/parser/variables.ts +0 -0
  165. package/src/core/parser/yaml-parser.ts +0 -181
  166. package/src/core/probe/bootstrap.ts +0 -34
  167. package/src/core/probe/dry-run-envelope.ts +0 -61
  168. package/src/core/probe/mass-assignment/classify.ts +0 -175
  169. package/src/core/probe/mass-assignment/cleanup.ts +0 -52
  170. package/src/core/probe/mass-assignment/digest.ts +0 -114
  171. package/src/core/probe/mass-assignment/orchestrator.ts +0 -459
  172. package/src/core/probe/mass-assignment/regression.ts +0 -141
  173. package/src/core/probe/mass-assignment/suspects.ts +0 -92
  174. package/src/core/probe/mass-assignment/types.ts +0 -135
  175. package/src/core/probe/mass-assignment-probe-class.ts +0 -198
  176. package/src/core/probe/mass-assignment-probe.ts +0 -27
  177. package/src/core/probe/mass-assignment-template.ts +0 -240
  178. package/src/core/probe/method-probe.ts +0 -164
  179. package/src/core/probe/method-shared.ts +0 -69
  180. package/src/core/probe/negative-probe.ts +0 -691
  181. package/src/core/probe/orphan-tracker.ts +0 -188
  182. package/src/core/probe/path-discovery.ts +0 -439
  183. package/src/core/probe/probe-harness.ts +0 -119
  184. package/src/core/probe/registry.ts +0 -89
  185. package/src/core/probe/runner.ts +0 -136
  186. package/src/core/probe/security/baseline.ts +0 -174
  187. package/src/core/probe/security/classify.ts +0 -341
  188. package/src/core/probe/security/cleanup.ts +0 -125
  189. package/src/core/probe/security/detectors.ts +0 -71
  190. package/src/core/probe/security/digest.ts +0 -104
  191. package/src/core/probe/security/orchestrator.ts +0 -398
  192. package/src/core/probe/security/regression.ts +0 -103
  193. package/src/core/probe/security/types.ts +0 -151
  194. package/src/core/probe/security-probe-class.ts +0 -207
  195. package/src/core/probe/security-probe.ts +0 -32
  196. package/src/core/probe/shared.ts +0 -531
  197. package/src/core/probe/static-probe-class.ts +0 -125
  198. package/src/core/probe/types.ts +0 -165
  199. package/src/core/probe/verdict-aggregator.ts +0 -33
  200. package/src/core/probe/webhooks-probe.ts +0 -282
  201. package/src/core/reporter/console.ts +0 -240
  202. package/src/core/reporter/index.ts +0 -22
  203. package/src/core/reporter/json.ts +0 -22
  204. package/src/core/reporter/junit.ts +0 -93
  205. package/src/core/reporter/ndjson.ts +0 -37
  206. package/src/core/reporter/types.ts +0 -15
  207. package/src/core/runner/assertions.ts +0 -383
  208. package/src/core/runner/async-pool.ts +0 -108
  209. package/src/core/runner/auth-path.ts +0 -8
  210. package/src/core/runner/ci-context.ts +0 -72
  211. package/src/core/runner/executor.ts +0 -652
  212. package/src/core/runner/expr-eval.ts +0 -41
  213. package/src/core/runner/form-encode.ts +0 -41
  214. package/src/core/runner/http-client.ts +0 -224
  215. package/src/core/runner/learn-drift.ts +0 -293
  216. package/src/core/runner/preflight-vars.ts +0 -153
  217. package/src/core/runner/progress-tracker.ts +0 -73
  218. package/src/core/runner/rate-limiter.ts +0 -185
  219. package/src/core/runner/run-kind.ts +0 -45
  220. package/src/core/runner/schema-validator.ts +0 -308
  221. package/src/core/runner/send-request.ts +0 -232
  222. package/src/core/runner/transforms.ts +0 -65
  223. package/src/core/runner/types.ts +0 -94
  224. package/src/core/secrets/registry.ts +0 -164
  225. package/src/core/secrets/secrets-file.ts +0 -115
  226. package/src/core/selectors/operation-filter.ts +0 -144
  227. package/src/core/setup-api.ts +0 -590
  228. package/src/core/severity/category.ts +0 -94
  229. package/src/core/severity/index.ts +0 -58
  230. package/src/core/spec/infer-schema.ts +0 -102
  231. package/src/core/spec/layers.ts +0 -154
  232. package/src/core/spec/merge-specs.ts +0 -156
  233. package/src/core/spec/schema-from-runs.ts +0 -117
  234. package/src/core/spec/schema-overlay.ts +0 -130
  235. package/src/core/util/ajv.ts +0 -13
  236. package/src/core/util/format-eta.ts +0 -21
  237. package/src/core/util/headers.ts +0 -9
  238. package/src/core/util/url.ts +0 -24
  239. package/src/core/utils.ts +0 -13
  240. package/src/core/workspace/config.ts +0 -129
  241. package/src/core/workspace/fixture-gap-report.ts +0 -84
  242. package/src/core/workspace/fixture-gaps.ts +0 -71
  243. package/src/core/workspace/manifest.ts +0 -283
  244. package/src/core/workspace/output-rotation.ts +0 -62
  245. package/src/core/workspace/root.ts +0 -96
  246. package/src/core/workspace/triage-path.ts +0 -87
  247. package/src/db/lint-runs.ts +0 -47
  248. package/src/db/migrate.ts +0 -128
  249. package/src/db/migrations/0001_run_kind.sql +0 -25
  250. package/src/db/migrations/0002_run_kind_request.sql +0 -59
  251. package/src/db/migrations/sql.d.ts +0 -4
  252. package/src/db/queries/collections.ts +0 -133
  253. package/src/db/queries/coverage.ts +0 -9
  254. package/src/db/queries/dashboard.ts +0 -59
  255. package/src/db/queries/results.ts +0 -216
  256. package/src/db/queries/runs.ts +0 -289
  257. package/src/db/queries/sessions.ts +0 -42
  258. package/src/db/queries/settings.ts +0 -28
  259. package/src/db/queries/types.ts +0 -172
  260. package/src/db/queries.ts +0 -75
  261. package/src/db/schema.ts +0 -298
@@ -1,666 +0,0 @@
1
- import { getDb } from "../../db/schema.ts";
2
- import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
3
- import { join } from "node:path";
4
- import { classifyFailure, recommendedActionForGenerated, isGeneratedTest, type RecommendedAction } from "./failure-hints.ts";
5
- import { buildSuggestedFixes, type SuggestedFix } from "./suggested-fixes.ts";
6
-
7
- function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
8
- if (!raw) return undefined;
9
- if (verbose || raw.length < 500) return raw;
10
- const lines = raw.split(/\r?\n/);
11
- const msgLines = [lines[0]!];
12
- let traceCount = 0;
13
- for (let i = 1; i < lines.length && traceCount < 3; i++) {
14
- const line = lines[i]!;
15
- if (/^\s+/.test(line) || /^\s*at\s/.test(line)) {
16
- msgLines.push(line);
17
- traceCount++;
18
- }
19
- }
20
- const remaining = lines.length - msgLines.length;
21
- if (remaining > 0) {
22
- msgLines.push(`...[truncated ${remaining} lines]`);
23
- }
24
- return msgLines.join("\n");
25
- }
26
-
27
- /**
28
- * ARV-305: build a short reason string for `by_recommended_action.examples`.
29
- *
30
- * Preference order:
31
- * 1. trimmed `error_message` (assertion- or network-level message)
32
- * 2. first failing assertion → `<field> <rule>: got <actual>`
33
- * 3. undefined (nothing left to say)
34
- *
35
- * Without (2) every assertion-only failure leaves examples[].reason as
36
- * undefined, so triage agents lose the one signal that lets them route
37
- * past method/path/status (regenerate_suite vs tighten_validation
38
- * collapse to identical-looking buckets).
39
- */
40
- function buildExampleReason(
41
- errorMessage: unknown,
42
- assertions: unknown,
43
- ): string | undefined {
44
- const trim = (s: string) => (s.length > 120 ? `${s.slice(0, 117)}...` : s);
45
- if (typeof errorMessage === "string" && errorMessage.length > 0) {
46
- return trim(errorMessage);
47
- }
48
- if (!Array.isArray(assertions)) return undefined;
49
- for (const a of assertions) {
50
- if (!a || typeof a !== "object") continue;
51
- const row = a as Record<string, unknown>;
52
- if (row.passed === false) {
53
- const field = typeof row.field === "string" ? row.field : "";
54
- const rule = typeof row.rule === "string" ? row.rule : "";
55
- const actual = "actual" in row ? row.actual : undefined;
56
- const actualStr = actual === undefined || actual === null
57
- ? ""
58
- : typeof actual === "string"
59
- ? actual
60
- : JSON.stringify(actual);
61
- const parts = [field, rule].filter(Boolean).join(" ");
62
- const text = actualStr ? `${parts}: got ${actualStr}` : parts;
63
- return text ? trim(text) : undefined;
64
- }
65
- }
66
- return undefined;
67
- }
68
-
69
- function parseBodySafe(raw: string | null | undefined): unknown {
70
- if (!raw) return undefined;
71
- const truncated = raw.length > 2000 ? raw.slice(0, 2000) + "\u2026[truncated]" : raw;
72
- try {
73
- return JSON.parse(raw);
74
- } catch {
75
- return truncated;
76
- }
77
- }
78
-
79
- const USEFUL_HEADERS = new Set([
80
- "content-type", "content-length", "location", "retry-after",
81
- "www-authenticate", "allow",
82
- ]);
83
- const USEFUL_PREFIXES = ["x-", "ratelimit"];
84
-
85
- /** ARV-103 (F8): true when at least one assertion on the failing step is
86
- * a schema-validation kind. `--validate-schema` annotates each violated
87
- * field with `kind: "schema"` (set in src/core/runner/schema-validator.ts).
88
- * The assertions column is stored as JSON in SQLite; parse defensively. */
89
- function hasSchemaAssertion(raw: string | unknown[] | null | undefined): boolean {
90
- if (raw === null || raw === undefined) return false;
91
- let arr: unknown[];
92
- if (Array.isArray(raw)) {
93
- arr = raw;
94
- } else if (typeof raw === "string") {
95
- try {
96
- const parsed = JSON.parse(raw);
97
- if (!Array.isArray(parsed)) return false;
98
- arr = parsed;
99
- } catch {
100
- return false;
101
- }
102
- } else {
103
- return false;
104
- }
105
- for (const a of arr) {
106
- if (a && typeof a === "object" && (a as { kind?: unknown }).kind === "schema") return true;
107
- }
108
- return false;
109
- }
110
-
111
- function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
112
- if (!raw) return undefined;
113
- try {
114
- const h = JSON.parse(raw) as Record<string, string>;
115
- const out: Record<string, string> = {};
116
- for (const [k, v] of Object.entries(h)) {
117
- const l = k.toLowerCase();
118
- if (USEFUL_HEADERS.has(l) || USEFUL_PREFIXES.some(p => l.startsWith(p))) {
119
- out[k] = v;
120
- }
121
- }
122
- return Object.keys(out).length > 0 ? out : undefined;
123
- } catch { return undefined; }
124
- }
125
-
126
- export interface RunDetail {
127
- run: {
128
- id: number;
129
- started_at: string;
130
- finished_at: string | null;
131
- total: number;
132
- passed: number;
133
- failed: number;
134
- skipped: number;
135
- trigger: string | null;
136
- environment: string | null;
137
- duration_ms: number | null;
138
- };
139
- results: Array<{
140
- suite_name: string;
141
- test_name: string;
142
- status: string;
143
- duration_ms: number | null;
144
- request_method: string | null;
145
- request_url: string | null;
146
- response_status: number | null;
147
- error_message?: string;
148
- assertions: unknown;
149
- }>;
150
- }
151
-
152
- export function getRunDetail(runId: number, verbose?: boolean, dbPath?: string): RunDetail {
153
- getDb(dbPath);
154
- const run = getRunById(runId);
155
- if (!run) throw new Error(`Run ${runId} not found`);
156
- const results = getResultsByRunId(runId);
157
- return {
158
- run: {
159
- id: run.id,
160
- started_at: run.started_at,
161
- finished_at: run.finished_at,
162
- total: run.total,
163
- passed: run.passed,
164
- failed: run.failed,
165
- skipped: run.skipped,
166
- trigger: run.trigger,
167
- environment: run.environment,
168
- duration_ms: run.duration_ms,
169
- },
170
- results: results.map(r => ({
171
- suite_name: r.suite_name,
172
- test_name: r.test_name,
173
- status: r.status,
174
- duration_ms: r.duration_ms,
175
- request_method: r.request_method,
176
- request_url: r.request_url,
177
- response_status: r.response_status,
178
- error_message: truncateErrorMessage(r.error_message, verbose),
179
- assertions: r.assertions,
180
- })),
181
- };
182
- }
183
-
184
- export interface FailureGroup {
185
- pattern: string;
186
- count: number;
187
- failure_type: string;
188
- recommended_action: RecommendedAction;
189
- examples: string[];
190
- response_status: number | null;
191
- }
192
-
193
- export interface CascadeSkipGroup {
194
- capture_var: string;
195
- count: number;
196
- examples: string[];
197
- }
198
-
199
- export interface DiagnoseResult {
200
- run: {
201
- id: number;
202
- started_at: string;
203
- environment: string | null;
204
- duration_ms: number | null;
205
- };
206
- summary: {
207
- total: number;
208
- passed: number;
209
- failed: number;
210
- api_errors: number;
211
- assertion_failures: number;
212
- network_errors: number;
213
- };
214
- cascade_skips?: CascadeSkipGroup[];
215
- /** TASK-29: actionable suggestions populated from 404 placeholder
216
- * detection + .env.yaml unfilled-key audit. Empty / undefined when
217
- * nothing actionable was found. */
218
- suggested_fixes?: SuggestedFix[];
219
- failures: Array<{
220
- suite_name: string;
221
- test_name: string;
222
- suite_file?: string;
223
- status: string;
224
- failure_type: string;
225
- recommended_action: RecommendedAction;
226
- error_message?: string;
227
- request_method: string | null;
228
- request_url: string | null;
229
- response_status: number | null;
230
- response_body?: unknown;
231
- response_headers?: Record<string, string>;
232
- assertions: unknown;
233
- duration_ms: number | null;
234
- /** ARV-159: when this entry is the representative of a collapsed group
235
- * (status|failure_type signature), the total size of that group. Lets
236
- * consumers reading `.data.failures[]` see "this signature stands for
237
- * N underlying tests" without cross-referencing `.grouped_failures[]`.
238
- * Omitted when no collapsing occurred (failures ≤ 5 or
239
- * --verbose). */
240
- group_count?: number;
241
- }>;
242
- grouped_failures?: FailureGroup[];
243
- /** ARV-101 (F6): top-level aggregation keyed by `recommended_action`
244
- * enum so triage agents (zond-triage skill) can route on the canonical
245
- * action without re-folding `failures[].recommended_action` through
246
- * `jq | group_by`. Built from the *full* failure set (not the compact
247
- * subset), so counts match `.summary.failed`. Each bucket carries
248
- * total count + a small examples list (`<suite>/<test>`). Empty when
249
- * there are no failures. */
250
- by_recommended_action?: Record<string, {
251
- count: number;
252
- /** ARV-228: each example carries the per-failure context the
253
- * zond-triage skill renders in its output template (`POST
254
- * /v1/projects → 500 (×3) — <reason>`). Previously a bare
255
- * `string[]` of `<suite>/<test>` ids — agents had to cross-join
256
- * with `failures[]` to recover method/path/status, which broke
257
- * triage scripts on large runs. Bounded to 5 entries per bucket
258
- * (same cap as the legacy string form). */
259
- examples: Array<{
260
- suite: string;
261
- test: string;
262
- method: string;
263
- path: string;
264
- status: number;
265
- reason?: string;
266
- }>;
267
- }>;
268
- }
269
-
270
- export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, maxExamples?: number): DiagnoseResult {
271
- getDb(dbPath);
272
- const diagRun = getRunById(runId);
273
- if (!diagRun) throw new Error(`Run ${runId} not found`);
274
-
275
- let envFilePath: string | undefined;
276
- if (diagRun.collection_id) {
277
- const collection = getCollectionById(diagRun.collection_id);
278
- if (collection?.base_dir) {
279
- envFilePath = join(collection.base_dir, ".env.yaml").replace(/\\/g, "/");
280
- }
281
- }
282
-
283
- const allResults = getResultsByRunId(runId);
284
- const failures = allResults
285
- .filter(r => r.status === "fail" || r.status === "error")
286
- .map(r => {
287
- const parsedBody = parseBodySafe(r.response_body);
288
- const failure_type = classifyFailure(r.status, r.response_status);
289
- // ARV-42: generator-emitted suites should not route to fix_test_logic —
290
- // editing the YAML gets clobbered on the next `zond audit`.
291
- const generated = isGeneratedTest(r.provenance, r.suite_file);
292
- // ARV-103 (F8): walk the assertions array to detect a schema-kind
293
- // failure (--validate-schema annotates each assertion with its kind).
294
- // When present, propagate the flag so the classifier routes to
295
- // report_backend_bug — schema violations are real contract bugs, not
296
- // test-logic mistakes.
297
- const schema_violation = hasSchemaAssertion(r.assertions);
298
- const rec_action = recommendedActionForGenerated(failure_type, r.response_status, generated, schema_violation);
299
- return {
300
- suite_name: r.suite_name,
301
- test_name: r.test_name,
302
- ...(r.suite_file ? { suite_file: r.suite_file } : {}),
303
- status: r.status,
304
- failure_type,
305
- recommended_action: rec_action,
306
- error_message: truncateErrorMessage(r.error_message, verbose),
307
- request_method: r.request_method,
308
- request_url: r.request_url,
309
- response_status: r.response_status,
310
- response_body: parsedBody,
311
- response_headers: filterHeaders(r.response_headers),
312
- assertions: r.assertions,
313
- duration_ms: r.duration_ms,
314
- };
315
- });
316
-
317
- let apiErrors = 0, assertionFailures = 0, networkErrors = 0;
318
- for (const f of failures) {
319
- if (f.failure_type === "api_error") apiErrors++;
320
- else if (f.failure_type === "assertion_failed") assertionFailures++;
321
- else if (f.failure_type === "network_error") networkErrors++;
322
- }
323
-
324
- // Cascade skips: skipped tests due to missing captures from failed create steps
325
- const CASCADE_RE = /^Depends on missing capture: (.+)$/;
326
- const groupMap = new Map<string, string[]>();
327
- for (const r of allResults) {
328
- if (r.status !== "skip") continue;
329
- const match = CASCADE_RE.exec(r.error_message ?? "");
330
- if (!match) continue;
331
- const captureVar = match[1]!;
332
- const existing = groupMap.get(captureVar) ?? [];
333
- existing.push(`${r.suite_name}/${r.test_name}`);
334
- groupMap.set(captureVar, existing);
335
- }
336
- const cascade_skips: CascadeSkipGroup[] | undefined = groupMap.size > 0
337
- ? [...groupMap.entries()].map(([capture_var, examples]) => ({
338
- capture_var,
339
- count: examples.length,
340
- examples: examples.slice(0, 3),
341
- }))
342
- : undefined;
343
-
344
- const { grouped_failures, compactFailures } = verbose
345
- ? { grouped_failures: undefined, compactFailures: failures }
346
- : groupFailures(failures, maxExamples);
347
-
348
- // TASK-29: surface placeholder path-params + unfilled .env.yaml keys.
349
- const suggestedFixes = buildSuggestedFixes({
350
- failures: failures.map(f => ({
351
- response_status: f.response_status,
352
- request_url: f.request_url,
353
- suite_name: f.suite_name,
354
- test_name: f.test_name,
355
- })),
356
- envFilePath,
357
- });
358
-
359
- // ARV-101 (F6): aggregate failures by recommended_action enum so triage
360
- // agents read .data.by_recommended_action.fix_env.count instead of
361
- // re-folding failures[].recommended_action through `jq | group_by`. Built
362
- // from the full failure set (not compactFailures) so counts match
363
- // .summary.failed. Bounded examples list (5) keeps payload small while
364
- // still pointing at concrete suites the agent can open.
365
- //
366
- // ARV-228: each example is now an object carrying method/path/status/
367
- // reason so the zond-triage skill can render its output template
368
- // ("POST /v1/projects → 500 (×3) — TypeError") without cross-joining
369
- // failures[]. Bound preserved at 5/bucket; ordering = insertion order
370
- // (matches failures[] traversal, deterministic per run).
371
- const by_recommended_action: Record<string, NonNullable<DiagnoseResult["by_recommended_action"]>[string]> = {};
372
- for (const f of failures) {
373
- const key = f.recommended_action;
374
- let bucket = by_recommended_action[key];
375
- if (!bucket) {
376
- bucket = { count: 0, examples: [] };
377
- by_recommended_action[key] = bucket;
378
- }
379
- bucket.count += 1;
380
- if (bucket.examples.length < 5) {
381
- // Trim reason to keep the bucket compact — full error_message lives
382
- // in failures[].error_message for agents that want it.
383
- //
384
- // ARV-305: when there is no top-level error_message (typical for
385
- // assertion failures — the row carries the failing rule in
386
- // .assertions, not a free-form message), build a reason out of
387
- // the first failing assertion so the example is not stripped
388
- // down to method/path/status. The fallback keeps the field
389
- // populated whenever any failure context exists.
390
- const reason = buildExampleReason(f.error_message, f.assertions);
391
- bucket.examples.push({
392
- suite: f.suite_name,
393
- test: f.test_name,
394
- // DB columns are nullable (early steps may not have a request
395
- // recorded yet — e.g. fixture-load failures). Coerce to "" / 0
396
- // rather than leaking null into the contract.
397
- method: f.request_method ?? "",
398
- path: f.request_url ?? "",
399
- status: f.response_status ?? 0,
400
- ...(reason ? { reason } : {}),
401
- });
402
- }
403
- }
404
-
405
- return {
406
- run: {
407
- id: diagRun.id,
408
- started_at: diagRun.started_at,
409
- environment: diagRun.environment,
410
- duration_ms: diagRun.duration_ms,
411
- },
412
- summary: {
413
- total: diagRun.total,
414
- passed: diagRun.passed,
415
- failed: diagRun.failed,
416
- api_errors: apiErrors,
417
- assertion_failures: assertionFailures,
418
- network_errors: networkErrors,
419
- },
420
- ...(cascade_skips ? { cascade_skips } : {}),
421
- ...(suggestedFixes.length > 0 ? { suggested_fixes: suggestedFixes } : {}),
422
- failures: compactFailures,
423
- ...(grouped_failures ? { grouped_failures } : {}),
424
- ...(failures.length > 0 ? { by_recommended_action } : {}),
425
- };
426
- }
427
-
428
- type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; response_status: number | null; group_count?: number };
429
-
430
- /** Group similar failures for compact output. Exported for testing. */
431
- export function groupFailures<T extends FailureItem>(failures: T[], maxExamples = 2): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
432
- if (failures.length <= 5) {
433
- return { compactFailures: failures };
434
- }
435
-
436
- const groupMap = new Map<string, { items: T[]; failure_type: string; response_status: number | null }>();
437
-
438
- for (const f of failures) {
439
- const key = `${f.response_status ?? "null"}|${f.failure_type}`;
440
- const existing = groupMap.get(key);
441
- if (existing) {
442
- existing.items.push(f);
443
- } else {
444
- groupMap.set(key, {
445
- items: [f],
446
- failure_type: f.failure_type,
447
- response_status: f.response_status,
448
- });
449
- }
450
- }
451
-
452
- const hasGroups = [...groupMap.values()].some(g => g.items.length > 2);
453
- if (!hasGroups) {
454
- return { compactFailures: failures };
455
- }
456
-
457
- const grouped_failures: FailureGroup[] = [];
458
- const compactFailures: T[] = [];
459
-
460
- for (const [, group] of groupMap) {
461
- const pattern = group.response_status
462
- ? `${group.response_status} ${group.failure_type}`
463
- : group.failure_type;
464
- // 5xx (api_error) are critical backend bugs — never collapse them.
465
- // Diagnose must surface every 5xx, otherwise users miss real
466
- // regressions hidden behind a single sample.
467
- const isApiError = group.failure_type === "api_error";
468
- const showAll = isApiError || maxExamples === 0;
469
- grouped_failures.push({
470
- pattern,
471
- count: group.items.length,
472
- failure_type: group.failure_type,
473
- recommended_action: group.items[0]!.recommended_action,
474
- examples: (showAll ? group.items : group.items.slice(0, maxExamples)).map(f => `${f.suite_name}/${f.test_name}`),
475
- response_status: group.response_status,
476
- });
477
- if (isApiError) {
478
- compactFailures.push(...group.items);
479
- } else {
480
- // ARV-159: tag the representative with the group size so
481
- // `.data.failures[]` carries the multiplier inline.
482
- const rep = { ...group.items[0]!, group_count: group.items.length };
483
- compactFailures.push(rep as T);
484
- }
485
- }
486
-
487
- return { grouped_failures, compactFailures };
488
- }
489
-
490
- export interface BodyFieldChange {
491
- field: string;
492
- change: "added" | "removed" | "type_changed";
493
- before?: string;
494
- after?: string;
495
- /** ARV-352: structural scope of the change, derived deterministically from
496
- * the path. `element` = the path crosses an array boundary (`[]`), i.e.
497
- * it's a field of a *collection item* — on list/log endpoints two samplings
498
- * return DIFFERENT objects, so element-level added/removed/type_changed is
499
- * schema-of-union variance across the sampled set, not a contract move.
500
- * `container` = no `[]` in the path — the response envelope/pagination
501
- * skeleton, where a change IS real drift. NOT a suppression heuristic
502
- * (ARV-337): nothing is dropped or down-ranked; the agent judges using the
503
- * scope tag + endpoint context. */
504
- scope: "container" | "element";
505
- }
506
-
507
- export interface BodyDiff {
508
- suite: string;
509
- test: string;
510
- changes: BodyFieldChange[];
511
- }
512
-
513
- export interface CompareResult {
514
- runA: { id: number; started_at: string };
515
- runB: { id: number; started_at: string };
516
- summary: {
517
- regressions: number;
518
- fixes: number;
519
- unchanged: number;
520
- newTests: number;
521
- removedTests: number;
522
- bodyChanges: number;
523
- /** ARV-352: of `bodyChanges`, how many touch the response envelope
524
- * (`container`) vs collection-item fields (`element`). Element-heavy diffs
525
- * on list/log endpoints are schema-of-union variance across a re-sampled
526
- * set, not contract drift — split so triage doesn't read them as regression. */
527
- bodyChangesContainer: number;
528
- bodyChangesElement: number;
529
- };
530
- regressions: Array<{ suite: string; test: string; before: string; after: string }>;
531
- fixes: Array<{ suite: string; test: string; before: string; after: string }>;
532
- /** ARV-339: field-level response-shape diff for tests present in both runs.
533
- * Status-diff answers "what broke"; this answers "how the contract moved". */
534
- body_changes: BodyDiff[];
535
- hasRegressions: boolean;
536
- }
537
-
538
- /** ARV-339: flatten a parsed JSON body into `path → union of leaf types`.
539
- * Array elements collapse under `[]` so item count/order don't add noise. */
540
- function bodyShape(value: unknown, path: string, out: Map<string, Set<string>>): void {
541
- if (Array.isArray(value)) {
542
- if (value.length === 0) addShape(out, path, "array");
543
- for (const item of value) bodyShape(item, `${path}[]`, out);
544
- } else if (value !== null && typeof value === "object") {
545
- const entries = Object.entries(value as Record<string, unknown>);
546
- if (entries.length === 0) addShape(out, path, "object");
547
- for (const [k, v] of entries) bodyShape(v, path ? `${path}.${k}` : k, out);
548
- } else {
549
- addShape(out, path, value === null ? "null" : typeof value);
550
- }
551
- }
552
-
553
- function addShape(out: Map<string, Set<string>>, path: string, type: string): void {
554
- const key = path || "$";
555
- const set = out.get(key) ?? new Set<string>();
556
- set.add(type);
557
- out.set(key, set);
558
- }
559
-
560
- const typeLabel = (s: Set<string>): string => [...s].sort().join("|");
561
-
562
- /** ARV-339: diff two stored response bodies at field level. Returns [] when
563
- * either side is missing / non-JSON — a shape diff of prose is meaningless. */
564
- export function diffBodyShapes(rawA: string | null, rawB: string | null): BodyFieldChange[] {
565
- if (!rawA || !rawB || rawA === rawB) return [];
566
- let a: unknown, b: unknown;
567
- try { a = JSON.parse(rawA); b = JSON.parse(rawB); } catch { return []; }
568
- if (typeof a !== "object" || a === null || typeof b !== "object" || b === null) return [];
569
- const shapeA = new Map<string, Set<string>>();
570
- const shapeB = new Map<string, Set<string>>();
571
- bodyShape(a, "", shapeA);
572
- bodyShape(b, "", shapeB);
573
- const scopeOf = (field: string): "container" | "element" =>
574
- field.includes("[]") ? "element" : "container";
575
- const changes: BodyFieldChange[] = [];
576
- for (const [field, typesB] of shapeB) {
577
- const typesA = shapeA.get(field);
578
- if (!typesA) changes.push({ field, change: "added", after: typeLabel(typesB), scope: scopeOf(field) });
579
- else if (typeLabel(typesA) !== typeLabel(typesB)) {
580
- changes.push({ field, change: "type_changed", before: typeLabel(typesA), after: typeLabel(typesB), scope: scopeOf(field) });
581
- }
582
- }
583
- for (const [field, typesA] of shapeA) {
584
- if (!shapeB.has(field)) changes.push({ field, change: "removed", before: typeLabel(typesA), scope: scopeOf(field) });
585
- }
586
- return changes.sort((x, y) => x.field.localeCompare(y.field));
587
- }
588
-
589
- export function compareRuns(idA: number, idB: number, dbPath?: string): CompareResult {
590
- getDb(dbPath);
591
- const runARecord = getRunById(idA);
592
- const runBRecord = getRunById(idB);
593
- if (!runARecord) throw new Error(`Run #${idA} not found`);
594
- if (!runBRecord) throw new Error(`Run #${idB} not found`);
595
-
596
- const resultsA = getResultsByRunId(idA);
597
- const resultsB = getResultsByRunId(idB);
598
-
599
- const mapA = new Map<string, string>();
600
- const mapB = new Map<string, string>();
601
- const bodyA = new Map<string, string | null>();
602
- const bodyB = new Map<string, string | null>();
603
- for (const r of resultsA) {
604
- mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
605
- bodyA.set(`${r.suite_name}::${r.test_name}`, r.response_body);
606
- }
607
- for (const r of resultsB) {
608
- mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
609
- bodyB.set(`${r.suite_name}::${r.test_name}`, r.response_body);
610
- }
611
-
612
- const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
613
- const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
614
- let unchanged = 0;
615
- let newTests = 0;
616
- let removedTests = 0;
617
-
618
- const body_changes: BodyDiff[] = [];
619
-
620
- for (const [key, statusB] of mapB) {
621
- const statusA = mapA.get(key);
622
- if (statusA === undefined) { newTests++; continue; }
623
- const [suite, test] = key.split("::") as [string, string];
624
- const changes = diffBodyShapes(bodyA.get(key) ?? null, bodyB.get(key) ?? null);
625
- if (changes.length > 0) body_changes.push({ suite, test, changes });
626
- const wasPass = statusA === "pass";
627
- const isPass = statusB === "pass";
628
- const wasFail = statusA === "fail" || statusA === "error";
629
- const isFail = statusB === "fail" || statusB === "error";
630
- if (wasPass && isFail) regressions.push({ suite, test, before: statusA, after: statusB });
631
- else if (wasFail && isPass) fixes.push({ suite, test, before: statusA, after: statusB });
632
- else unchanged++;
633
- }
634
- for (const key of mapA.keys()) {
635
- if (!mapB.has(key)) removedTests++;
636
- }
637
-
638
- return {
639
- runA: { id: idA, started_at: runARecord.started_at },
640
- runB: { id: idB, started_at: runBRecord.started_at },
641
- summary: {
642
- regressions: regressions.length,
643
- fixes: fixes.length,
644
- unchanged,
645
- newTests,
646
- removedTests,
647
- bodyChanges: body_changes.length,
648
- bodyChangesContainer: body_changes.filter(d => d.changes.some(c => c.scope === "container")).length,
649
- bodyChangesElement: body_changes.filter(d => d.changes.every(c => c.scope === "element")).length,
650
- },
651
- regressions,
652
- fixes,
653
- body_changes,
654
- hasRegressions: regressions.length > 0,
655
- };
656
- }
657
-
658
- export function getCollections(dbPath?: string) {
659
- getDb(dbPath);
660
- return listCollections();
661
- }
662
-
663
- export function getRuns(limit?: number, dbPath?: string) {
664
- getDb(dbPath);
665
- return listRuns(limit ?? 20);
666
- }