@kirrosh/zond 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +110 -3
  2. package/README.md +26 -15
  3. package/package.json +10 -6
  4. package/src/cli/commands/catalog.ts +62 -0
  5. package/src/cli/commands/ci-init.ts +12 -6
  6. package/src/cli/commands/completions.ts +176 -0
  7. package/src/cli/commands/db.ts +2 -1
  8. package/src/cli/commands/generate.ts +18 -2
  9. package/src/cli/commands/init/agents-md.ts +61 -0
  10. package/src/cli/commands/init/bootstrap.ts +79 -0
  11. package/src/cli/commands/init/skills.ts +45 -0
  12. package/src/cli/commands/init/templates/agents.md +73 -0
  13. package/src/cli/commands/init/templates/markdown.d.ts +4 -0
  14. package/src/cli/commands/init/templates/skills/scenarios.md +97 -0
  15. package/src/cli/commands/init/templates/skills/zond.md +184 -0
  16. package/src/cli/commands/init/templates/zond-config.yml +15 -0
  17. package/src/cli/commands/init.ts +124 -31
  18. package/src/cli/commands/probe-methods.ts +108 -0
  19. package/src/cli/commands/probe-validation.ts +124 -0
  20. package/src/cli/commands/run.ts +99 -10
  21. package/src/cli/commands/serve.ts +52 -19
  22. package/src/cli/commands/sync.ts +28 -1
  23. package/src/cli/commands/update.ts +1 -1
  24. package/src/cli/commands/use.ts +57 -0
  25. package/src/cli/index.ts +21 -591
  26. package/src/cli/program.ts +655 -0
  27. package/src/cli/version.ts +3 -0
  28. package/src/core/context/current.ts +35 -0
  29. package/src/core/diagnostics/db-analysis.ts +11 -2
  30. package/src/core/diagnostics/render-md.ts +112 -0
  31. package/src/core/generator/catalog-builder.ts +179 -0
  32. package/src/core/generator/chunker.ts +14 -2
  33. package/src/core/generator/data-factory.ts +50 -19
  34. package/src/core/generator/guide-builder.ts +1 -1
  35. package/src/core/generator/index.ts +2 -0
  36. package/src/core/generator/openapi-reader.ts +18 -0
  37. package/src/core/generator/serializer.ts +11 -2
  38. package/src/core/generator/suite-generator.ts +106 -7
  39. package/src/core/meta/types.ts +0 -2
  40. package/src/core/parser/schema.ts +3 -1
  41. package/src/core/parser/types.ts +10 -1
  42. package/src/core/parser/variables.ts +90 -2
  43. package/src/core/parser/yaml-parser.ts +50 -1
  44. package/src/core/probe/method-probe.ts +197 -0
  45. package/src/core/probe/negative-probe.ts +657 -0
  46. package/src/core/reporter/console.ts +29 -3
  47. package/src/core/reporter/index.ts +2 -2
  48. package/src/core/reporter/json.ts +5 -2
  49. package/src/core/runner/assertions.ts +4 -1
  50. package/src/core/runner/executor.ts +132 -37
  51. package/src/core/runner/http-client.ts +40 -5
  52. package/src/core/runner/rate-limiter.ts +131 -0
  53. package/src/core/setup-api.ts +4 -1
  54. package/src/core/workspace/root.ts +94 -0
  55. package/src/db/schema.ts +4 -1
  56. package/src/web/routes/api.ts +80 -0
  57. package/src/web/routes/dashboard.ts +15 -0
  58. package/src/web/static/style.css +290 -0
  59. package/src/web/views/explorer-tab.ts +402 -0
@@ -28,6 +28,30 @@ function convertPathWithSeeds(path: string, ep: EndpointInfo): string {
28
28
  });
29
29
  }
30
30
 
31
+ /**
32
+ * For negative-smoke suites: replace path params with guaranteed-non-existent values.
33
+ * Picks a value that's syntactically valid for the param's type/format but very
34
+ * unlikely to match a real resource (zero-UUID, very large int, sentinel string).
35
+ */
36
+ function getNonexistentSeed(schema: OpenAPIV3.SchemaObject | undefined): string {
37
+ if (!schema) return "nonexistent_id_zzzzzz";
38
+ if (schema.format === "uuid") return "00000000-0000-0000-0000-000000000000";
39
+ if (schema.type === "integer" || schema.type === "number") return "999999999";
40
+ return "nonexistent_id_zzzzzz";
41
+ }
42
+
43
+ function convertPathWithBadIds(path: string, ep: EndpointInfo): string {
44
+ return path.replace(/\{([^}]+)\}/g, (_, name: string) => {
45
+ const param = ep.parameters.find(p => p.name === name && p.in === "path");
46
+ const schema = param?.schema as OpenAPIV3.SchemaObject | undefined;
47
+ return getNonexistentSeed(schema);
48
+ });
49
+ }
50
+
51
+ function endpointHasPathParams(ep: EndpointInfo): boolean {
52
+ return ep.parameters.some(p => p.in === "path");
53
+ }
54
+
31
55
  function slugify(s: string): string {
32
56
  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
33
57
  }
@@ -360,9 +384,11 @@ export function generateCrudSuite(
360
384
  });
361
385
  }
362
386
 
387
+ // T44: cleanup must run even if earlier assertions failed (tainted captures)
363
388
  const step: RawStep = {
364
389
  name: group.delete.operationId ?? `Delete ${group.resource.replace(/s$/, "")}`,
365
390
  DELETE: itemPath,
391
+ always: true,
366
392
  expect: {
367
393
  status: getExpectedStatus(group.delete),
368
394
  },
@@ -372,11 +398,12 @@ export function generateCrudSuite(
372
398
  }
373
399
  tests.push(step);
374
400
 
375
- // 5. Verify deleted
401
+ // 5. Verify deleted — also always, so we confirm cleanup happened
376
402
  if (group.read) {
377
403
  tests.push({
378
404
  name: `Verify ${group.resource.replace(/s$/, "")} deleted`,
379
405
  GET: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
406
+ always: true,
380
407
  expect: {
381
408
  status: 404,
382
409
  },
@@ -384,9 +411,13 @@ export function generateCrudSuite(
384
411
  }
385
412
  }
386
413
 
414
+ // T28: classify by cleanup behavior. A suite that owns a DELETE leaves the API
415
+ // in its starting state (ephemeral); without DELETE it leaves residual data.
416
+ const cleanupTag = group.delete ? "ephemeral" : "persistent-write";
417
+
387
418
  const suite: RawSuite = {
388
419
  name: `${group.resource}-crud`,
389
- tags: ["crud"],
420
+ tags: ["crud", cleanupTag],
390
421
  fileStem: `crud-${slugify(group.resource)}`,
391
422
  base_url: "{{base_url}}",
392
423
  tests,
@@ -634,17 +665,20 @@ export function generateSuites(opts: {
634
665
  for (const [tag, tagEndpoints] of byTag) {
635
666
  const tagSlug = slugify(tag) || "api";
636
667
 
637
- // GET endpoints → smoke suite (use seed values for path params — no capture context)
668
+ // GET endpoints → split into paramless (regular smoke) and path-param (negative+positive smoke)
638
669
  const getEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() === "GET");
639
- if (getEndpoints.length > 0) {
640
- const tests = getEndpoints.map(ep => {
670
+ const paramlessGets = getEndpoints.filter(ep => !endpointHasPathParams(ep));
671
+ const pathParamGets = getEndpoints.filter(ep => endpointHasPathParams(ep));
672
+
673
+ // Regular smoke: paramless GETs (e.g. list endpoints, health checks)
674
+ if (paramlessGets.length > 0) {
675
+ const tests = paramlessGets.map(ep => {
641
676
  const step = generateStep(ep, securitySchemes);
642
- // Replace path param placeholders with seed values so the suite runs out of the box
643
677
  const seededPath = convertPathWithSeeds(ep.path, ep);
644
678
  (step as any)[ep.method.toUpperCase()] = seededPath;
645
679
  return step;
646
680
  });
647
- const headers = getSuiteHeaders(getEndpoints, securitySchemes);
681
+ const headers = getSuiteHeaders(paramlessGets, securitySchemes);
648
682
 
649
683
  const suite: RawSuite = {
650
684
  name: `${tagSlug}-smoke`,
@@ -666,6 +700,71 @@ export function generateSuites(opts: {
666
700
  suites.push(suite);
667
701
  }
668
702
 
703
+ // Negative smoke: path-param GETs with guaranteed-bad IDs, expect 400/404/422
704
+ if (pathParamGets.length > 0) {
705
+ const tests = pathParamGets.map(ep => {
706
+ const step = generateStep(ep, securitySchemes);
707
+ (step as any)[ep.method.toUpperCase()] = convertPathWithBadIds(ep.path, ep);
708
+ // Negative path: resource doesn't exist. Drop body assertions (response shape varies).
709
+ step.expect = { status: [400, 404, 422] };
710
+ return step;
711
+ });
712
+ const headers = getSuiteHeaders(pathParamGets, securitySchemes);
713
+
714
+ const suite: RawSuite = {
715
+ name: `${tagSlug}-smoke-negative`,
716
+ tags: ["smoke", "negative"],
717
+ fileStem: `smoke-${tagSlug}-negative`,
718
+ base_url: "{{base_url}}",
719
+ tests,
720
+ };
721
+
722
+ if (headers) {
723
+ suite.headers = headers;
724
+ for (const t of tests) {
725
+ if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
726
+ delete (t as any).headers;
727
+ }
728
+ }
729
+ }
730
+
731
+ suites.push(suite);
732
+ }
733
+
734
+ // Positive smoke: path-param GETs with {{var}} placeholders + skip_if for unset env
735
+ if (pathParamGets.length > 0) {
736
+ const tests = pathParamGets.map(ep => {
737
+ const step = generateStep(ep, securitySchemes);
738
+ // Path stays as {{param}} so user-provided env values flow in
739
+ // Pick the first path param for skip_if guard (the resource ID)
740
+ const firstPathParam = ep.parameters.find(p => p.in === "path");
741
+ if (firstPathParam) {
742
+ step.skip_if = `{{${firstPathParam.name}}} ==`;
743
+ }
744
+ return step;
745
+ });
746
+ const headers = getSuiteHeaders(pathParamGets, securitySchemes);
747
+
748
+ const suite: RawSuite = {
749
+ name: `${tagSlug}-smoke-positive`,
750
+ tags: ["smoke", "positive", "needs-id"],
751
+ fileStem: `smoke-${tagSlug}-positive`,
752
+ base_url: "{{base_url}}",
753
+ tests,
754
+ };
755
+
756
+ if (headers) {
757
+ suite.headers = headers;
758
+ for (const t of tests) {
759
+ if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
760
+ delete (t as any).headers;
761
+ }
762
+ }
763
+ }
764
+
765
+ suites.push(suite);
766
+ }
767
+
669
768
  // Non-GET endpoints: split reset/system endpoints out of smoke-unsafe
670
769
  const nonGetEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() !== "GET");
671
770
  const resetEndpoints = nonGetEndpoints.filter(ep => RESET_PATH_RE.test(ep.path));
@@ -12,8 +12,6 @@ export interface ZondMeta {
12
12
  zondVersion: string;
13
13
  /** ISO timestamp of last sync/generate */
14
14
  lastSyncedAt: string;
15
- /** Spec URL or file path used for last generation */
16
- specUrl: string;
17
15
  /** SHA-256 hex of spec content at time of last generation */
18
16
  specHash: string;
19
17
  /** Per-file metadata, keyed by filename (e.g. "smoke-users.yaml") */
@@ -92,7 +92,7 @@ const AssertionRuleSchemaInner: z.ZodType<AssertionRule> = z.preprocess(
92
92
  },
93
93
  z.object({
94
94
  capture: z.string().optional(),
95
- type: z.enum(["string", "integer", "number", "boolean", "array", "object"]).optional(),
95
+ type: z.enum(["string", "integer", "number", "boolean", "array", "object", "null"]).optional(),
96
96
  equals: z.unknown().optional(),
97
97
  not_equals: z.unknown().optional(),
98
98
  contains: z.string().optional(),
@@ -184,6 +184,7 @@ const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
184
184
  retry_until: RetryUntilSchema.optional(),
185
185
  for_each: ForEachSchema.optional(),
186
186
  set: z.record(z.string(), z.unknown()).optional(),
187
+ always: z.boolean().optional(),
187
188
  }),
188
189
  ) as z.ZodType<TestStep>;
189
190
 
@@ -220,6 +221,7 @@ const TestSuiteSchema = z.preprocess(
220
221
  tags: z.array(z.string()).optional(),
221
222
  base_url: z.string().optional(),
222
223
  headers: z.record(z.string(), z.string()).optional(),
224
+ parameterize: z.record(z.string(), z.array(z.unknown()).min(1)).optional(),
223
225
  config: SuiteConfigSchema,
224
226
  tests: z.array(TestStepSchema).min(1),
225
227
  }),
@@ -2,7 +2,7 @@ export type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
2
2
 
3
3
  export interface AssertionRule {
4
4
  capture?: string;
5
- type?: "string" | "integer" | "number" | "boolean" | "array" | "object";
5
+ type?: "string" | "integer" | "number" | "boolean" | "array" | "object" | "null";
6
6
  equals?: unknown;
7
7
  not_equals?: unknown;
8
8
  contains?: string;
@@ -63,6 +63,12 @@ export interface TestStep {
63
63
  retry_until?: RetryUntil;
64
64
  for_each?: ForEach;
65
65
  set?: Record<string, unknown>;
66
+ /**
67
+ * Run this step even when prior steps in the suite have failed assertions
68
+ * (so their captures are "tainted"). Designed for cleanup steps. Still
69
+ * skips if a referenced capture is genuinely missing from a response.
70
+ */
71
+ always?: boolean;
66
72
  }
67
73
 
68
74
  export interface SuiteConfig {
@@ -81,6 +87,9 @@ export interface TestSuite {
81
87
  tags?: string[];
82
88
  base_url?: string;
83
89
  headers?: Record<string, string>;
90
+ /** Cross-product parameterisation: each key contributes one variable
91
+ * binding per array entry. Suite body runs once per combination. */
92
+ parameterize?: Record<string, unknown[]>;
84
93
  config: SuiteConfig;
85
94
  tests: TestStep[];
86
95
  /** Absolute path to the source file, set by yaml-parser */
@@ -16,6 +16,23 @@ function randomChars(len: number): string {
16
16
  return result;
17
17
  }
18
18
 
19
+ function lowerChars(len: number): string {
20
+ let result = "";
21
+ for (let i = 0; i < len; i++) {
22
+ const idx = Math.floor(Math.random() * 36);
23
+ result += CHARS[idx]!.toLowerCase();
24
+ }
25
+ return result;
26
+ }
27
+
28
+ function randomOctet(): number {
29
+ return Math.floor(Math.random() * 254) + 1;
30
+ }
31
+
32
+ function randomDate(): string {
33
+ return new Date().toISOString().slice(0, 10);
34
+ }
35
+
19
36
  export const GENERATORS: Record<string, () => string | number> = {
20
37
  "$uuid": () => crypto.randomUUID(),
21
38
  "$timestamp": () => Math.floor(Date.now() / 1000),
@@ -24,10 +41,38 @@ export const GENERATORS: Record<string, () => string | number> = {
24
41
  "$randomEmail": () => `${randomChars(8).toLowerCase()}@test.com`,
25
42
  "$randomInt": () => Math.floor(Math.random() * 10000),
26
43
  "$randomString": () => randomChars(8),
44
+ "$randomUrl": () => `https://example-${lowerChars(8)}.com/path`,
45
+ "$randomFqdn": () => `test-${lowerChars(8)}.example.com`,
46
+ "$randomIpv4": () => `10.${randomOctet()}.${randomOctet()}.${randomOctet()}`,
47
+ "$randomDate": randomDate,
48
+ "$randomIsoDate": () => new Date().toISOString(),
27
49
  };
28
50
 
29
51
  const VAR_PATTERN = /\{\{(.+?)\}\}/g;
30
52
 
53
+ /**
54
+ * Suggest a known generator close to the misspelled name.
55
+ * Case-insensitive prefix match first, then case-insensitive exact match.
56
+ */
57
+ function suggestGenerator(name: string): string | undefined {
58
+ const lower = name.toLowerCase();
59
+ const known = Object.keys(GENERATORS);
60
+ // Case-insensitive exact (catches case-only typos like $randomfqdn → $randomFqdn)
61
+ const ciExact = known.find((k) => k.toLowerCase() === lower);
62
+ if (ciExact) return ciExact;
63
+ // Prefix match
64
+ return known.find((k) => k.toLowerCase().startsWith(lower.slice(0, 6)));
65
+ }
66
+
67
+ function unknownGeneratorError(key: string): Error {
68
+ const suggestion = suggestGenerator(key);
69
+ const hint = suggestion ? ` (did you mean ${suggestion}?)` : "";
70
+ const available = Object.keys(GENERATORS).join(", ");
71
+ return new Error(
72
+ `Unknown generator: {{${key}}}${hint}. Available: ${available}`,
73
+ );
74
+ }
75
+
31
76
  export function substituteString(template: string, vars: Record<string, unknown>): unknown {
32
77
  // If entire string is a single {{var}}, return raw value (number stays number)
33
78
  const singleMatch = template.match(/^\{\{([^{}]+)\}\}$/);
@@ -35,6 +80,7 @@ export function substituteString(template: string, vars: Record<string, unknown>
35
80
  const key = singleMatch[1]!;
36
81
  if (key in vars) return vars[key];
37
82
  if (key in GENERATORS) return GENERATORS[key]!();
83
+ if (key.startsWith("$")) throw unknownGeneratorError(key);
38
84
  return template;
39
85
  }
40
86
 
@@ -42,6 +88,7 @@ export function substituteString(template: string, vars: Record<string, unknown>
42
88
  return template.replace(new RegExp(VAR_PATTERN.source, "g"), (_, key: string) => {
43
89
  if (key in vars) return String(vars[key]);
44
90
  if (key in GENERATORS) return String(GENERATORS[key]!());
91
+ if (key.startsWith("$")) throw unknownGeneratorError(key);
45
92
  return `{{${key}}}`;
46
93
  });
47
94
  }
@@ -110,7 +157,7 @@ export function extractVariableReferences(step: TestStep): string[] {
110
157
  return [...refs];
111
158
  }
112
159
 
113
- async function loadEnvFile(filePath: string): Promise<Record<string, string> | null> {
160
+ export async function loadEnvFile(filePath: string): Promise<Record<string, string> | null> {
114
161
  try {
115
162
  const text = await Bun.file(filePath).text();
116
163
  const parsed = Bun.YAML.parse(text);
@@ -154,5 +201,46 @@ export async function loadEnvironment(envName?: string, searchDir: string = ".")
154
201
  const fileVars = await loadEnvFile(`${searchDir}/${fileName}`);
155
202
  const parentFileVars = await loadEnvFile(`${dirname(searchDir)}/${fileName}`);
156
203
 
157
- return { ...parentFileVars, ...fileVars };
204
+ const merged = { ...parentFileVars, ...fileVars };
205
+ // Strip reserved meta keys so they don't leak into variable substitution
206
+ for (const key of META_KEYS) {
207
+ delete merged[key];
208
+ }
209
+ return merged;
210
+ }
211
+
212
+ const META_KEYS = ["rateLimit"] as const;
213
+
214
+ export interface EnvMeta {
215
+ rateLimit?: number;
216
+ }
217
+
218
+ async function readEnvMetaFile(filePath: string): Promise<EnvMeta | null> {
219
+ try {
220
+ const text = await Bun.file(filePath).text();
221
+ const parsed = Bun.YAML.parse(text);
222
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
223
+ const obj = parsed as Record<string, unknown>;
224
+ const meta: EnvMeta = {};
225
+ if ("rateLimit" in obj) {
226
+ const v = obj.rateLimit;
227
+ if (typeof v === "number" && Number.isFinite(v) && v > 0) {
228
+ meta.rateLimit = v;
229
+ } else if (typeof v === "string") {
230
+ const n = Number.parseFloat(v);
231
+ if (Number.isFinite(n) && n > 0) meta.rateLimit = n;
232
+ }
233
+ }
234
+ return meta;
235
+ } catch (err) {
236
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") return null;
237
+ return null;
238
+ }
239
+ }
240
+
241
+ export async function loadEnvMeta(envName?: string, searchDir: string = "."): Promise<EnvMeta> {
242
+ const fileName = envName ? `.env.${envName}.yaml` : ".env.yaml";
243
+ const parent = await readEnvMetaFile(`${dirname(searchDir)}/${fileName}`);
244
+ const own = await readEnvMetaFile(`${searchDir}/${fileName}`);
245
+ return { ...(parent ?? {}), ...(own ?? {}) };
158
246
  }
@@ -1,8 +1,46 @@
1
1
  import { Glob } from "bun";
2
2
  import { resolve } from "node:path";
3
+ import YAML from "yaml";
3
4
  import { validateSuite } from "./schema.ts";
4
5
  import type { TestSuite } from "./types.ts";
5
6
 
7
+ /** Convert a 0-based byte offset into a 1-based (line, col) position. */
8
+ function offsetToLineCol(text: string, offset: number): { line: number; col: number } {
9
+ let line = 1;
10
+ let col = 1;
11
+ for (let i = 0; i < offset && i < text.length; i++) {
12
+ if (text.charCodeAt(i) === 0x0a) {
13
+ line++;
14
+ col = 1;
15
+ } else {
16
+ col++;
17
+ }
18
+ }
19
+ return { line, col };
20
+ }
21
+
22
+ /**
23
+ * Format a YAML parse error as `file:line:col: <reason>` plus a snippet with
24
+ * a column pointer. Bun.YAML's SyntaxError exposes JS stack coordinates, not
25
+ * YAML positions, so on parse failure we re-parse with eemeli/yaml (which
26
+ * provides accurate `linePos`) just for diagnostics.
27
+ *
28
+ * Exported for tests.
29
+ */
30
+ export function formatYamlParseError(filePath: string, text: string, primary: Error): Error {
31
+ const doc = YAML.parseDocument(text);
32
+ const e = doc.errors[0];
33
+ if (e?.linePos?.[0]) {
34
+ const { line, col } = e.linePos[0];
35
+ // eemeli's message reads "<reason> at line X, column Y:\n\n<snippet>".
36
+ // Strip the "at line ..." part since we surface line:col in the prefix.
37
+ const cleaned = e.message.replace(/\s+at line \d+, column \d+:/, ":");
38
+ return new Error(`Invalid YAML in ${filePath}:${line}:${col}: ${cleaned}`);
39
+ }
40
+ // eemeli accepted but Bun rejected — fall back to original message.
41
+ return new Error(`Invalid YAML in ${filePath}: ${primary.message}`);
42
+ }
43
+
6
44
  export async function parseFile(filePath: string): Promise<TestSuite> {
7
45
  let text: string;
8
46
  try {
@@ -11,11 +49,22 @@ export async function parseFile(filePath: string): Promise<TestSuite> {
11
49
  throw new Error(`Failed to read file ${filePath}: ${(err as Error).message}`);
12
50
  }
13
51
 
52
+ // Both Bun.YAML and eemeli/yaml accept NUL bytes silently, but they corrupt
53
+ // downstream consumers (sqlite TEXT, JSON, terminals). Surface explicitly.
54
+ const nulIdx = text.indexOf("\x00");
55
+ if (nulIdx >= 0) {
56
+ const { line, col } = offsetToLineCol(text, nulIdx);
57
+ throw new Error(
58
+ `Invalid YAML in ${filePath}:${line}:${col}: NUL byte (\\x00) in source — ` +
59
+ `if you need a NUL in a request body, use the {{$nullByte}} generator instead of inlining the byte`
60
+ );
61
+ }
62
+
14
63
  let raw: unknown;
15
64
  try {
16
65
  raw = Bun.YAML.parse(text);
17
66
  } catch (err) {
18
- throw new Error(`Invalid YAML in ${filePath}: ${(err as Error).message}`);
67
+ throw formatYamlParseError(filePath, text, err as Error);
19
68
  }
20
69
 
21
70
  try {
@@ -0,0 +1,197 @@
1
+ /**
2
+ * HTTP method completeness probe (T48).
3
+ *
4
+ * Goal: catch the class of bugs where an API responds to *unsupported* HTTP
5
+ * methods with anything other than 405 / 404. A 500 here means an unhandled
6
+ * exception in the routing layer; a 200/201 means a forgotten or shadowed
7
+ * route; both are bug candidates.
8
+ *
9
+ * For every path declared in the spec, we look at which of {GET, POST, PUT,
10
+ * PATCH, DELETE} are *not* declared and emit one probe step per missing
11
+ * method. Each probe expects status in [404, 405, 401, 403] — anything else
12
+ * (notably 5xx, 200, 201) is a regular test failure surfaced via the
13
+ * existing runner / reporter / `zond db diagnose` flow.
14
+ *
15
+ * The probes are deterministic — same spec → same suites — so the generated
16
+ * YAML can be committed as a regression test.
17
+ */
18
+ import type { OpenAPIV3 } from "openapi-types";
19
+ import type { EndpointInfo, SecuritySchemeInfo } from "../generator/types.ts";
20
+ import type { RawSuite, RawStep } from "../generator/serializer.ts";
21
+
22
+ // ──────────────────────────────────────────────
23
+ // Constants
24
+ // ──────────────────────────────────────────────
25
+
26
+ const ALL_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const;
27
+ type Method = (typeof ALL_METHODS)[number];
28
+
29
+ /** Statuses we accept on a *missing* method. 405 is canonical, 404 is a
30
+ * common fallback (path not registered for that method), 401/403 are
31
+ * acceptable when auth is checked before routing. Anything else — notably
32
+ * 5xx (unhandled), 200/201 (silent acceptance) — is a probe failure. */
33
+ const ACCEPTABLE_STATUSES = [401, 403, 404, 405];
34
+
35
+ // ──────────────────────────────────────────────
36
+ // Types
37
+ // ──────────────────────────────────────────────
38
+
39
+ export interface MethodProbeOptions {
40
+ endpoints: EndpointInfo[];
41
+ securitySchemes: SecuritySchemeInfo[];
42
+ }
43
+
44
+ export interface MethodProbeResult {
45
+ suites: RawSuite[];
46
+ /** Number of distinct paths probed. */
47
+ probedPaths: number;
48
+ /** Paths skipped because every method in {GET,POST,PUT,PATCH,DELETE} is declared. */
49
+ skippedPaths: number;
50
+ /** Total generated probe steps. */
51
+ totalProbes: number;
52
+ }
53
+
54
+ // ──────────────────────────────────────────────
55
+ // Helpers
56
+ // ──────────────────────────────────────────────
57
+
58
+ function convertPath(path: string): string {
59
+ return path.replace(/\{([^}]+)\}/g, "{{$1}}");
60
+ }
61
+
62
+ function slugify(s: string): string {
63
+ return s
64
+ .toLowerCase()
65
+ .replace(/[^a-z0-9]+/g, "-")
66
+ .replace(/^-|-$/g, "");
67
+ }
68
+
69
+ function pathStem(path: string): string {
70
+ const cleaned = path
71
+ .replace(/\{[^}]+\}/g, "by-id")
72
+ .replace(/^\//, "")
73
+ .replace(/\//g, "-");
74
+ return slugify(cleaned) || "root";
75
+ }
76
+
77
+ /** Replace path params with valid-shape placeholders so the request can
78
+ * reach the routing layer without being rejected purely on path syntax. */
79
+ function pathWithPlaceholders(
80
+ path: string,
81
+ parameters: OpenAPIV3.ParameterObject[],
82
+ ): string {
83
+ return path.replace(/\{([^}]+)\}/g, (_, name: string) => {
84
+ const param = parameters.find((p) => p.name === name && p.in === "path");
85
+ const schema = param?.schema as OpenAPIV3.SchemaObject | undefined;
86
+ if (schema?.format === "uuid") return "00000000-0000-0000-0000-000000000000";
87
+ if (schema?.type === "integer" || schema?.type === "number") return "999999999";
88
+ return "nonexistent-zzzzz";
89
+ });
90
+ }
91
+
92
+ function getAuthHeaders(
93
+ ep: EndpointInfo,
94
+ schemes: SecuritySchemeInfo[],
95
+ ): Record<string, string> | undefined {
96
+ if (ep.security.length === 0) return undefined;
97
+ for (const secName of ep.security) {
98
+ const scheme = schemes.find((s) => s.name === secName);
99
+ if (!scheme) continue;
100
+ if (scheme.type === "http") {
101
+ if (scheme.scheme === "bearer" || !scheme.scheme) {
102
+ return { Authorization: "Bearer {{auth_token}}" };
103
+ }
104
+ if (scheme.scheme === "basic") {
105
+ return { Authorization: "Basic {{auth_token}}" };
106
+ }
107
+ }
108
+ if (scheme.type === "apiKey" && scheme.in === "header" && scheme.apiKeyName) {
109
+ if (scheme.apiKeyName === "Authorization") {
110
+ return { Authorization: "Bearer {{auth_token}}" };
111
+ }
112
+ return { [scheme.apiKeyName]: "{{api_key}}" };
113
+ }
114
+ }
115
+ return undefined;
116
+ }
117
+
118
+ interface PathBucket {
119
+ path: string;
120
+ /** Methods declared on this path, normalized to upper-case. */
121
+ declared: Set<string>;
122
+ /** A representative endpoint we can borrow auth/path-param shape from. */
123
+ sample: EndpointInfo;
124
+ }
125
+
126
+ function bucketByPath(endpoints: EndpointInfo[]): PathBucket[] {
127
+ const map = new Map<string, PathBucket>();
128
+ for (const ep of endpoints) {
129
+ if (ep.deprecated) continue;
130
+ let bucket = map.get(ep.path);
131
+ if (!bucket) {
132
+ bucket = { path: ep.path, declared: new Set(), sample: ep };
133
+ map.set(ep.path, bucket);
134
+ }
135
+ bucket.declared.add(ep.method.toUpperCase());
136
+ }
137
+ return Array.from(map.values());
138
+ }
139
+
140
+ // ──────────────────────────────────────────────
141
+ // Public API
142
+ // ──────────────────────────────────────────────
143
+
144
+ export function generateMethodProbes(opts: MethodProbeOptions): MethodProbeResult {
145
+ const { endpoints, securitySchemes } = opts;
146
+ const methodSet: readonly Method[] = ALL_METHODS;
147
+
148
+ const buckets = bucketByPath(endpoints);
149
+ const suites: RawSuite[] = [];
150
+ let probedPaths = 0;
151
+ let skippedPaths = 0;
152
+ let totalProbes = 0;
153
+
154
+ for (const bucket of buckets) {
155
+ const missing = methodSet.filter((m) => !bucket.declared.has(m));
156
+ if (missing.length === 0) {
157
+ skippedPaths++;
158
+ continue;
159
+ }
160
+
161
+ const concretePath = pathWithPlaceholders(
162
+ bucket.path,
163
+ bucket.sample.parameters,
164
+ );
165
+ const headers = getAuthHeaders(bucket.sample, securitySchemes);
166
+
167
+ const steps: RawStep[] = missing.map((method) => {
168
+ const step: RawStep = {
169
+ name: `${method} ${bucket.path} — undeclared method must reject (no 5xx, no 2xx)`,
170
+ [method]: convertPath(concretePath),
171
+ expect: { status: ACCEPTABLE_STATUSES },
172
+ };
173
+ // Body-bearing methods on an undeclared route — send a minimal valid
174
+ // JSON object to provoke any body-parsing path while the router is
175
+ // still expected to reject the method first.
176
+ if (method === "POST" || method === "PUT" || method === "PATCH") {
177
+ (step as any).json = {};
178
+ }
179
+ return step;
180
+ });
181
+
182
+ probedPaths++;
183
+ totalProbes += steps.length;
184
+
185
+ const stem = pathStem(bucket.path);
186
+ suites.push({
187
+ name: `probe methods ${bucket.path}`,
188
+ tags: ["probe-methods", "negative-method", "no-5xx", "smoke"],
189
+ fileStem: `probe-methods-${stem}`,
190
+ base_url: "{{base_url}}",
191
+ ...(headers ? { headers } : {}),
192
+ tests: steps,
193
+ });
194
+ }
195
+
196
+ return { suites, probedPaths, skippedPaths, totalProbes };
197
+ }