@kirrosh/zond 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -3
- package/README.md +26 -15
- package/package.json +10 -6
- package/src/cli/commands/catalog.ts +62 -0
- package/src/cli/commands/ci-init.ts +12 -6
- package/src/cli/commands/completions.ts +176 -0
- package/src/cli/commands/db.ts +2 -1
- package/src/cli/commands/generate.ts +18 -2
- package/src/cli/commands/init/agents-md.ts +61 -0
- package/src/cli/commands/init/bootstrap.ts +79 -0
- package/src/cli/commands/init/skills.ts +45 -0
- package/src/cli/commands/init/templates/agents.md +73 -0
- package/src/cli/commands/init/templates/markdown.d.ts +4 -0
- package/src/cli/commands/init/templates/skills/scenarios.md +97 -0
- package/src/cli/commands/init/templates/skills/zond.md +184 -0
- package/src/cli/commands/init/templates/zond-config.yml +15 -0
- package/src/cli/commands/init.ts +124 -31
- package/src/cli/commands/probe-methods.ts +108 -0
- package/src/cli/commands/probe-validation.ts +124 -0
- package/src/cli/commands/run.ts +99 -10
- package/src/cli/commands/serve.ts +52 -19
- package/src/cli/commands/sync.ts +28 -1
- package/src/cli/commands/update.ts +1 -1
- package/src/cli/commands/use.ts +57 -0
- package/src/cli/index.ts +21 -591
- package/src/cli/program.ts +655 -0
- package/src/cli/version.ts +3 -0
- package/src/core/context/current.ts +35 -0
- package/src/core/diagnostics/db-analysis.ts +11 -2
- package/src/core/diagnostics/render-md.ts +112 -0
- package/src/core/generator/catalog-builder.ts +179 -0
- package/src/core/generator/chunker.ts +14 -2
- package/src/core/generator/data-factory.ts +50 -19
- package/src/core/generator/guide-builder.ts +1 -1
- package/src/core/generator/index.ts +2 -0
- package/src/core/generator/openapi-reader.ts +18 -0
- package/src/core/generator/serializer.ts +11 -2
- package/src/core/generator/suite-generator.ts +106 -7
- package/src/core/meta/types.ts +0 -2
- package/src/core/parser/schema.ts +3 -1
- package/src/core/parser/types.ts +10 -1
- package/src/core/parser/variables.ts +90 -2
- package/src/core/parser/yaml-parser.ts +50 -1
- package/src/core/probe/method-probe.ts +197 -0
- package/src/core/probe/negative-probe.ts +657 -0
- package/src/core/reporter/console.ts +29 -3
- package/src/core/reporter/index.ts +2 -2
- package/src/core/reporter/json.ts +5 -2
- package/src/core/runner/assertions.ts +4 -1
- package/src/core/runner/executor.ts +132 -37
- package/src/core/runner/http-client.ts +40 -5
- package/src/core/runner/rate-limiter.ts +131 -0
- package/src/core/setup-api.ts +4 -1
- package/src/core/workspace/root.ts +94 -0
- package/src/db/schema.ts +4 -1
- package/src/web/routes/api.ts +80 -0
- package/src/web/routes/dashboard.ts +15 -0
- package/src/web/static/style.css +290 -0
- package/src/web/views/explorer-tab.ts +402 -0
|
@@ -28,6 +28,30 @@ function convertPathWithSeeds(path: string, ep: EndpointInfo): string {
|
|
|
28
28
|
});
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* For negative-smoke suites: replace path params with guaranteed-non-existent values.
|
|
33
|
+
* Picks a value that's syntactically valid for the param's type/format but very
|
|
34
|
+
* unlikely to match a real resource (zero-UUID, very large int, sentinel string).
|
|
35
|
+
*/
|
|
36
|
+
function getNonexistentSeed(schema: OpenAPIV3.SchemaObject | undefined): string {
|
|
37
|
+
if (!schema) return "nonexistent_id_zzzzzz";
|
|
38
|
+
if (schema.format === "uuid") return "00000000-0000-0000-0000-000000000000";
|
|
39
|
+
if (schema.type === "integer" || schema.type === "number") return "999999999";
|
|
40
|
+
return "nonexistent_id_zzzzzz";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function convertPathWithBadIds(path: string, ep: EndpointInfo): string {
|
|
44
|
+
return path.replace(/\{([^}]+)\}/g, (_, name: string) => {
|
|
45
|
+
const param = ep.parameters.find(p => p.name === name && p.in === "path");
|
|
46
|
+
const schema = param?.schema as OpenAPIV3.SchemaObject | undefined;
|
|
47
|
+
return getNonexistentSeed(schema);
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function endpointHasPathParams(ep: EndpointInfo): boolean {
|
|
52
|
+
return ep.parameters.some(p => p.in === "path");
|
|
53
|
+
}
|
|
54
|
+
|
|
31
55
|
function slugify(s: string): string {
|
|
32
56
|
return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
33
57
|
}
|
|
@@ -360,9 +384,11 @@ export function generateCrudSuite(
|
|
|
360
384
|
});
|
|
361
385
|
}
|
|
362
386
|
|
|
387
|
+
// T44: cleanup must run even if earlier assertions failed (tainted captures)
|
|
363
388
|
const step: RawStep = {
|
|
364
389
|
name: group.delete.operationId ?? `Delete ${group.resource.replace(/s$/, "")}`,
|
|
365
390
|
DELETE: itemPath,
|
|
391
|
+
always: true,
|
|
366
392
|
expect: {
|
|
367
393
|
status: getExpectedStatus(group.delete),
|
|
368
394
|
},
|
|
@@ -372,11 +398,12 @@ export function generateCrudSuite(
|
|
|
372
398
|
}
|
|
373
399
|
tests.push(step);
|
|
374
400
|
|
|
375
|
-
// 5. Verify deleted
|
|
401
|
+
// 5. Verify deleted — also always, so we confirm cleanup happened
|
|
376
402
|
if (group.read) {
|
|
377
403
|
tests.push({
|
|
378
404
|
name: `Verify ${group.resource.replace(/s$/, "")} deleted`,
|
|
379
405
|
GET: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
|
|
406
|
+
always: true,
|
|
380
407
|
expect: {
|
|
381
408
|
status: 404,
|
|
382
409
|
},
|
|
@@ -384,9 +411,13 @@ export function generateCrudSuite(
|
|
|
384
411
|
}
|
|
385
412
|
}
|
|
386
413
|
|
|
414
|
+
// T28: classify by cleanup behavior. A suite that owns a DELETE leaves the API
|
|
415
|
+
// in its starting state (ephemeral); without DELETE it leaves residual data.
|
|
416
|
+
const cleanupTag = group.delete ? "ephemeral" : "persistent-write";
|
|
417
|
+
|
|
387
418
|
const suite: RawSuite = {
|
|
388
419
|
name: `${group.resource}-crud`,
|
|
389
|
-
tags: ["crud"],
|
|
420
|
+
tags: ["crud", cleanupTag],
|
|
390
421
|
fileStem: `crud-${slugify(group.resource)}`,
|
|
391
422
|
base_url: "{{base_url}}",
|
|
392
423
|
tests,
|
|
@@ -634,17 +665,20 @@ export function generateSuites(opts: {
|
|
|
634
665
|
for (const [tag, tagEndpoints] of byTag) {
|
|
635
666
|
const tagSlug = slugify(tag) || "api";
|
|
636
667
|
|
|
637
|
-
// GET endpoints →
|
|
668
|
+
// GET endpoints → split into paramless (regular smoke) and path-param (negative+positive smoke)
|
|
638
669
|
const getEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() === "GET");
|
|
639
|
-
|
|
640
|
-
|
|
670
|
+
const paramlessGets = getEndpoints.filter(ep => !endpointHasPathParams(ep));
|
|
671
|
+
const pathParamGets = getEndpoints.filter(ep => endpointHasPathParams(ep));
|
|
672
|
+
|
|
673
|
+
// Regular smoke: paramless GETs (e.g. list endpoints, health checks)
|
|
674
|
+
if (paramlessGets.length > 0) {
|
|
675
|
+
const tests = paramlessGets.map(ep => {
|
|
641
676
|
const step = generateStep(ep, securitySchemes);
|
|
642
|
-
// Replace path param placeholders with seed values so the suite runs out of the box
|
|
643
677
|
const seededPath = convertPathWithSeeds(ep.path, ep);
|
|
644
678
|
(step as any)[ep.method.toUpperCase()] = seededPath;
|
|
645
679
|
return step;
|
|
646
680
|
});
|
|
647
|
-
const headers = getSuiteHeaders(
|
|
681
|
+
const headers = getSuiteHeaders(paramlessGets, securitySchemes);
|
|
648
682
|
|
|
649
683
|
const suite: RawSuite = {
|
|
650
684
|
name: `${tagSlug}-smoke`,
|
|
@@ -666,6 +700,71 @@ export function generateSuites(opts: {
|
|
|
666
700
|
suites.push(suite);
|
|
667
701
|
}
|
|
668
702
|
|
|
703
|
+
// Negative smoke: path-param GETs with guaranteed-bad IDs, expect 400/404/422
|
|
704
|
+
if (pathParamGets.length > 0) {
|
|
705
|
+
const tests = pathParamGets.map(ep => {
|
|
706
|
+
const step = generateStep(ep, securitySchemes);
|
|
707
|
+
(step as any)[ep.method.toUpperCase()] = convertPathWithBadIds(ep.path, ep);
|
|
708
|
+
// Negative path: resource doesn't exist. Drop body assertions (response shape varies).
|
|
709
|
+
step.expect = { status: [400, 404, 422] };
|
|
710
|
+
return step;
|
|
711
|
+
});
|
|
712
|
+
const headers = getSuiteHeaders(pathParamGets, securitySchemes);
|
|
713
|
+
|
|
714
|
+
const suite: RawSuite = {
|
|
715
|
+
name: `${tagSlug}-smoke-negative`,
|
|
716
|
+
tags: ["smoke", "negative"],
|
|
717
|
+
fileStem: `smoke-${tagSlug}-negative`,
|
|
718
|
+
base_url: "{{base_url}}",
|
|
719
|
+
tests,
|
|
720
|
+
};
|
|
721
|
+
|
|
722
|
+
if (headers) {
|
|
723
|
+
suite.headers = headers;
|
|
724
|
+
for (const t of tests) {
|
|
725
|
+
if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
|
|
726
|
+
delete (t as any).headers;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
suites.push(suite);
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// Positive smoke: path-param GETs with {{var}} placeholders + skip_if for unset env
|
|
735
|
+
if (pathParamGets.length > 0) {
|
|
736
|
+
const tests = pathParamGets.map(ep => {
|
|
737
|
+
const step = generateStep(ep, securitySchemes);
|
|
738
|
+
// Path stays as {{param}} so user-provided env values flow in
|
|
739
|
+
// Pick the first path param for skip_if guard (the resource ID)
|
|
740
|
+
const firstPathParam = ep.parameters.find(p => p.in === "path");
|
|
741
|
+
if (firstPathParam) {
|
|
742
|
+
step.skip_if = `{{${firstPathParam.name}}} ==`;
|
|
743
|
+
}
|
|
744
|
+
return step;
|
|
745
|
+
});
|
|
746
|
+
const headers = getSuiteHeaders(pathParamGets, securitySchemes);
|
|
747
|
+
|
|
748
|
+
const suite: RawSuite = {
|
|
749
|
+
name: `${tagSlug}-smoke-positive`,
|
|
750
|
+
tags: ["smoke", "positive", "needs-id"],
|
|
751
|
+
fileStem: `smoke-${tagSlug}-positive`,
|
|
752
|
+
base_url: "{{base_url}}",
|
|
753
|
+
tests,
|
|
754
|
+
};
|
|
755
|
+
|
|
756
|
+
if (headers) {
|
|
757
|
+
suite.headers = headers;
|
|
758
|
+
for (const t of tests) {
|
|
759
|
+
if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
|
|
760
|
+
delete (t as any).headers;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
suites.push(suite);
|
|
766
|
+
}
|
|
767
|
+
|
|
669
768
|
// Non-GET endpoints: split reset/system endpoints out of smoke-unsafe
|
|
670
769
|
const nonGetEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() !== "GET");
|
|
671
770
|
const resetEndpoints = nonGetEndpoints.filter(ep => RESET_PATH_RE.test(ep.path));
|
package/src/core/meta/types.ts
CHANGED
|
@@ -12,8 +12,6 @@ export interface ZondMeta {
|
|
|
12
12
|
zondVersion: string;
|
|
13
13
|
/** ISO timestamp of last sync/generate */
|
|
14
14
|
lastSyncedAt: string;
|
|
15
|
-
/** Spec URL or file path used for last generation */
|
|
16
|
-
specUrl: string;
|
|
17
15
|
/** SHA-256 hex of spec content at time of last generation */
|
|
18
16
|
specHash: string;
|
|
19
17
|
/** Per-file metadata, keyed by filename (e.g. "smoke-users.yaml") */
|
|
@@ -92,7 +92,7 @@ const AssertionRuleSchemaInner: z.ZodType<AssertionRule> = z.preprocess(
|
|
|
92
92
|
},
|
|
93
93
|
z.object({
|
|
94
94
|
capture: z.string().optional(),
|
|
95
|
-
type: z.enum(["string", "integer", "number", "boolean", "array", "object"]).optional(),
|
|
95
|
+
type: z.enum(["string", "integer", "number", "boolean", "array", "object", "null"]).optional(),
|
|
96
96
|
equals: z.unknown().optional(),
|
|
97
97
|
not_equals: z.unknown().optional(),
|
|
98
98
|
contains: z.string().optional(),
|
|
@@ -184,6 +184,7 @@ const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
|
|
|
184
184
|
retry_until: RetryUntilSchema.optional(),
|
|
185
185
|
for_each: ForEachSchema.optional(),
|
|
186
186
|
set: z.record(z.string(), z.unknown()).optional(),
|
|
187
|
+
always: z.boolean().optional(),
|
|
187
188
|
}),
|
|
188
189
|
) as z.ZodType<TestStep>;
|
|
189
190
|
|
|
@@ -220,6 +221,7 @@ const TestSuiteSchema = z.preprocess(
|
|
|
220
221
|
tags: z.array(z.string()).optional(),
|
|
221
222
|
base_url: z.string().optional(),
|
|
222
223
|
headers: z.record(z.string(), z.string()).optional(),
|
|
224
|
+
parameterize: z.record(z.string(), z.array(z.unknown()).min(1)).optional(),
|
|
223
225
|
config: SuiteConfigSchema,
|
|
224
226
|
tests: z.array(TestStepSchema).min(1),
|
|
225
227
|
}),
|
package/src/core/parser/types.ts
CHANGED
|
@@ -2,7 +2,7 @@ export type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
|
|
|
2
2
|
|
|
3
3
|
export interface AssertionRule {
|
|
4
4
|
capture?: string;
|
|
5
|
-
type?: "string" | "integer" | "number" | "boolean" | "array" | "object";
|
|
5
|
+
type?: "string" | "integer" | "number" | "boolean" | "array" | "object" | "null";
|
|
6
6
|
equals?: unknown;
|
|
7
7
|
not_equals?: unknown;
|
|
8
8
|
contains?: string;
|
|
@@ -63,6 +63,12 @@ export interface TestStep {
|
|
|
63
63
|
retry_until?: RetryUntil;
|
|
64
64
|
for_each?: ForEach;
|
|
65
65
|
set?: Record<string, unknown>;
|
|
66
|
+
/**
|
|
67
|
+
* Run this step even when prior steps in the suite have failed assertions
|
|
68
|
+
* (so their captures are "tainted"). Designed for cleanup steps. Still
|
|
69
|
+
* skips if a referenced capture is genuinely missing from a response.
|
|
70
|
+
*/
|
|
71
|
+
always?: boolean;
|
|
66
72
|
}
|
|
67
73
|
|
|
68
74
|
export interface SuiteConfig {
|
|
@@ -81,6 +87,9 @@ export interface TestSuite {
|
|
|
81
87
|
tags?: string[];
|
|
82
88
|
base_url?: string;
|
|
83
89
|
headers?: Record<string, string>;
|
|
90
|
+
/** Cross-product parameterisation: each key contributes one variable
|
|
91
|
+
* binding per array entry. Suite body runs once per combination. */
|
|
92
|
+
parameterize?: Record<string, unknown[]>;
|
|
84
93
|
config: SuiteConfig;
|
|
85
94
|
tests: TestStep[];
|
|
86
95
|
/** Absolute path to the source file, set by yaml-parser */
|
|
@@ -16,6 +16,23 @@ function randomChars(len: number): string {
|
|
|
16
16
|
return result;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
function lowerChars(len: number): string {
|
|
20
|
+
let result = "";
|
|
21
|
+
for (let i = 0; i < len; i++) {
|
|
22
|
+
const idx = Math.floor(Math.random() * 36);
|
|
23
|
+
result += CHARS[idx]!.toLowerCase();
|
|
24
|
+
}
|
|
25
|
+
return result;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function randomOctet(): number {
|
|
29
|
+
return Math.floor(Math.random() * 254) + 1;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function randomDate(): string {
|
|
33
|
+
return new Date().toISOString().slice(0, 10);
|
|
34
|
+
}
|
|
35
|
+
|
|
19
36
|
export const GENERATORS: Record<string, () => string | number> = {
|
|
20
37
|
"$uuid": () => crypto.randomUUID(),
|
|
21
38
|
"$timestamp": () => Math.floor(Date.now() / 1000),
|
|
@@ -24,10 +41,38 @@ export const GENERATORS: Record<string, () => string | number> = {
|
|
|
24
41
|
"$randomEmail": () => `${randomChars(8).toLowerCase()}@test.com`,
|
|
25
42
|
"$randomInt": () => Math.floor(Math.random() * 10000),
|
|
26
43
|
"$randomString": () => randomChars(8),
|
|
44
|
+
"$randomUrl": () => `https://example-${lowerChars(8)}.com/path`,
|
|
45
|
+
"$randomFqdn": () => `test-${lowerChars(8)}.example.com`,
|
|
46
|
+
"$randomIpv4": () => `10.${randomOctet()}.${randomOctet()}.${randomOctet()}`,
|
|
47
|
+
"$randomDate": randomDate,
|
|
48
|
+
"$randomIsoDate": () => new Date().toISOString(),
|
|
27
49
|
};
|
|
28
50
|
|
|
29
51
|
const VAR_PATTERN = /\{\{(.+?)\}\}/g;
|
|
30
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Suggest a known generator close to the misspelled name.
|
|
55
|
+
* Case-insensitive prefix match first, then case-insensitive exact match.
|
|
56
|
+
*/
|
|
57
|
+
function suggestGenerator(name: string): string | undefined {
|
|
58
|
+
const lower = name.toLowerCase();
|
|
59
|
+
const known = Object.keys(GENERATORS);
|
|
60
|
+
// Case-insensitive exact (catches case-only typos like $randomfqdn → $randomFqdn)
|
|
61
|
+
const ciExact = known.find((k) => k.toLowerCase() === lower);
|
|
62
|
+
if (ciExact) return ciExact;
|
|
63
|
+
// Prefix match
|
|
64
|
+
return known.find((k) => k.toLowerCase().startsWith(lower.slice(0, 6)));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function unknownGeneratorError(key: string): Error {
|
|
68
|
+
const suggestion = suggestGenerator(key);
|
|
69
|
+
const hint = suggestion ? ` (did you mean ${suggestion}?)` : "";
|
|
70
|
+
const available = Object.keys(GENERATORS).join(", ");
|
|
71
|
+
return new Error(
|
|
72
|
+
`Unknown generator: {{${key}}}${hint}. Available: ${available}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
31
76
|
export function substituteString(template: string, vars: Record<string, unknown>): unknown {
|
|
32
77
|
// If entire string is a single {{var}}, return raw value (number stays number)
|
|
33
78
|
const singleMatch = template.match(/^\{\{([^{}]+)\}\}$/);
|
|
@@ -35,6 +80,7 @@ export function substituteString(template: string, vars: Record<string, unknown>
|
|
|
35
80
|
const key = singleMatch[1]!;
|
|
36
81
|
if (key in vars) return vars[key];
|
|
37
82
|
if (key in GENERATORS) return GENERATORS[key]!();
|
|
83
|
+
if (key.startsWith("$")) throw unknownGeneratorError(key);
|
|
38
84
|
return template;
|
|
39
85
|
}
|
|
40
86
|
|
|
@@ -42,6 +88,7 @@ export function substituteString(template: string, vars: Record<string, unknown>
|
|
|
42
88
|
return template.replace(new RegExp(VAR_PATTERN.source, "g"), (_, key: string) => {
|
|
43
89
|
if (key in vars) return String(vars[key]);
|
|
44
90
|
if (key in GENERATORS) return String(GENERATORS[key]!());
|
|
91
|
+
if (key.startsWith("$")) throw unknownGeneratorError(key);
|
|
45
92
|
return `{{${key}}}`;
|
|
46
93
|
});
|
|
47
94
|
}
|
|
@@ -110,7 +157,7 @@ export function extractVariableReferences(step: TestStep): string[] {
|
|
|
110
157
|
return [...refs];
|
|
111
158
|
}
|
|
112
159
|
|
|
113
|
-
async function loadEnvFile(filePath: string): Promise<Record<string, string> | null> {
|
|
160
|
+
export async function loadEnvFile(filePath: string): Promise<Record<string, string> | null> {
|
|
114
161
|
try {
|
|
115
162
|
const text = await Bun.file(filePath).text();
|
|
116
163
|
const parsed = Bun.YAML.parse(text);
|
|
@@ -154,5 +201,46 @@ export async function loadEnvironment(envName?: string, searchDir: string = ".")
|
|
|
154
201
|
const fileVars = await loadEnvFile(`${searchDir}/${fileName}`);
|
|
155
202
|
const parentFileVars = await loadEnvFile(`${dirname(searchDir)}/${fileName}`);
|
|
156
203
|
|
|
157
|
-
|
|
204
|
+
const merged = { ...parentFileVars, ...fileVars };
|
|
205
|
+
// Strip reserved meta keys so they don't leak into variable substitution
|
|
206
|
+
for (const key of META_KEYS) {
|
|
207
|
+
delete merged[key];
|
|
208
|
+
}
|
|
209
|
+
return merged;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const META_KEYS = ["rateLimit"] as const;
|
|
213
|
+
|
|
214
|
+
export interface EnvMeta {
|
|
215
|
+
rateLimit?: number;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function readEnvMetaFile(filePath: string): Promise<EnvMeta | null> {
|
|
219
|
+
try {
|
|
220
|
+
const text = await Bun.file(filePath).text();
|
|
221
|
+
const parsed = Bun.YAML.parse(text);
|
|
222
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
223
|
+
const obj = parsed as Record<string, unknown>;
|
|
224
|
+
const meta: EnvMeta = {};
|
|
225
|
+
if ("rateLimit" in obj) {
|
|
226
|
+
const v = obj.rateLimit;
|
|
227
|
+
if (typeof v === "number" && Number.isFinite(v) && v > 0) {
|
|
228
|
+
meta.rateLimit = v;
|
|
229
|
+
} else if (typeof v === "string") {
|
|
230
|
+
const n = Number.parseFloat(v);
|
|
231
|
+
if (Number.isFinite(n) && n > 0) meta.rateLimit = n;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return meta;
|
|
235
|
+
} catch (err) {
|
|
236
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") return null;
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export async function loadEnvMeta(envName?: string, searchDir: string = "."): Promise<EnvMeta> {
|
|
242
|
+
const fileName = envName ? `.env.${envName}.yaml` : ".env.yaml";
|
|
243
|
+
const parent = await readEnvMetaFile(`${dirname(searchDir)}/${fileName}`);
|
|
244
|
+
const own = await readEnvMetaFile(`${searchDir}/${fileName}`);
|
|
245
|
+
return { ...(parent ?? {}), ...(own ?? {}) };
|
|
158
246
|
}
|
|
@@ -1,8 +1,46 @@
|
|
|
1
1
|
import { Glob } from "bun";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
|
+
import YAML from "yaml";
|
|
3
4
|
import { validateSuite } from "./schema.ts";
|
|
4
5
|
import type { TestSuite } from "./types.ts";
|
|
5
6
|
|
|
7
|
+
/** Convert a 0-based byte offset into a 1-based (line, col) position. */
|
|
8
|
+
function offsetToLineCol(text: string, offset: number): { line: number; col: number } {
|
|
9
|
+
let line = 1;
|
|
10
|
+
let col = 1;
|
|
11
|
+
for (let i = 0; i < offset && i < text.length; i++) {
|
|
12
|
+
if (text.charCodeAt(i) === 0x0a) {
|
|
13
|
+
line++;
|
|
14
|
+
col = 1;
|
|
15
|
+
} else {
|
|
16
|
+
col++;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return { line, col };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Format a YAML parse error as `file:line:col: <reason>` plus a snippet with
|
|
24
|
+
* a column pointer. Bun.YAML's SyntaxError exposes JS stack coordinates, not
|
|
25
|
+
* YAML positions, so on parse failure we re-parse with eemeli/yaml (which
|
|
26
|
+
* provides accurate `linePos`) just for diagnostics.
|
|
27
|
+
*
|
|
28
|
+
* Exported for tests.
|
|
29
|
+
*/
|
|
30
|
+
export function formatYamlParseError(filePath: string, text: string, primary: Error): Error {
|
|
31
|
+
const doc = YAML.parseDocument(text);
|
|
32
|
+
const e = doc.errors[0];
|
|
33
|
+
if (e?.linePos?.[0]) {
|
|
34
|
+
const { line, col } = e.linePos[0];
|
|
35
|
+
// eemeli's message reads "<reason> at line X, column Y:\n\n<snippet>".
|
|
36
|
+
// Strip the "at line ..." part since we surface line:col in the prefix.
|
|
37
|
+
const cleaned = e.message.replace(/\s+at line \d+, column \d+:/, ":");
|
|
38
|
+
return new Error(`Invalid YAML in ${filePath}:${line}:${col}: ${cleaned}`);
|
|
39
|
+
}
|
|
40
|
+
// eemeli accepted but Bun rejected — fall back to original message.
|
|
41
|
+
return new Error(`Invalid YAML in ${filePath}: ${primary.message}`);
|
|
42
|
+
}
|
|
43
|
+
|
|
6
44
|
export async function parseFile(filePath: string): Promise<TestSuite> {
|
|
7
45
|
let text: string;
|
|
8
46
|
try {
|
|
@@ -11,11 +49,22 @@ export async function parseFile(filePath: string): Promise<TestSuite> {
|
|
|
11
49
|
throw new Error(`Failed to read file ${filePath}: ${(err as Error).message}`);
|
|
12
50
|
}
|
|
13
51
|
|
|
52
|
+
// Both Bun.YAML and eemeli/yaml accept NUL bytes silently, but they corrupt
|
|
53
|
+
// downstream consumers (sqlite TEXT, JSON, terminals). Surface explicitly.
|
|
54
|
+
const nulIdx = text.indexOf("\x00");
|
|
55
|
+
if (nulIdx >= 0) {
|
|
56
|
+
const { line, col } = offsetToLineCol(text, nulIdx);
|
|
57
|
+
throw new Error(
|
|
58
|
+
`Invalid YAML in ${filePath}:${line}:${col}: NUL byte (\\x00) in source — ` +
|
|
59
|
+
`if you need a NUL in a request body, use the {{$nullByte}} generator instead of inlining the byte`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
14
63
|
let raw: unknown;
|
|
15
64
|
try {
|
|
16
65
|
raw = Bun.YAML.parse(text);
|
|
17
66
|
} catch (err) {
|
|
18
|
-
throw
|
|
67
|
+
throw formatYamlParseError(filePath, text, err as Error);
|
|
19
68
|
}
|
|
20
69
|
|
|
21
70
|
try {
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP method completeness probe (T48).
|
|
3
|
+
*
|
|
4
|
+
* Goal: catch the class of bugs where an API responds to *unsupported* HTTP
|
|
5
|
+
* methods with anything other than 405 / 404. A 500 here means an unhandled
|
|
6
|
+
* exception in the routing layer; a 200/201 means a forgotten or shadowed
|
|
7
|
+
* route; both are bug candidates.
|
|
8
|
+
*
|
|
9
|
+
* For every path declared in the spec, we look at which of {GET, POST, PUT,
|
|
10
|
+
* PATCH, DELETE} are *not* declared and emit one probe step per missing
|
|
11
|
+
* method. Each probe expects status in [404, 405, 401, 403] — anything else
|
|
12
|
+
* (notably 5xx, 200, 201) is a regular test failure surfaced via the
|
|
13
|
+
* existing runner / reporter / `zond db diagnose` flow.
|
|
14
|
+
*
|
|
15
|
+
* The probes are deterministic — same spec → same suites — so the generated
|
|
16
|
+
* YAML can be committed as a regression test.
|
|
17
|
+
*/
|
|
18
|
+
import type { OpenAPIV3 } from "openapi-types";
|
|
19
|
+
import type { EndpointInfo, SecuritySchemeInfo } from "../generator/types.ts";
|
|
20
|
+
import type { RawSuite, RawStep } from "../generator/serializer.ts";
|
|
21
|
+
|
|
22
|
+
// ──────────────────────────────────────────────
|
|
23
|
+
// Constants
|
|
24
|
+
// ──────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
const ALL_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const;
|
|
27
|
+
type Method = (typeof ALL_METHODS)[number];
|
|
28
|
+
|
|
29
|
+
/** Statuses we accept on a *missing* method. 405 is canonical, 404 is a
|
|
30
|
+
* common fallback (path not registered for that method), 401/403 are
|
|
31
|
+
* acceptable when auth is checked before routing. Anything else — notably
|
|
32
|
+
* 5xx (unhandled), 200/201 (silent acceptance) — is a probe failure. */
|
|
33
|
+
const ACCEPTABLE_STATUSES = [401, 403, 404, 405];
|
|
34
|
+
|
|
35
|
+
// ──────────────────────────────────────────────
|
|
36
|
+
// Types
|
|
37
|
+
// ──────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
export interface MethodProbeOptions {
|
|
40
|
+
endpoints: EndpointInfo[];
|
|
41
|
+
securitySchemes: SecuritySchemeInfo[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface MethodProbeResult {
|
|
45
|
+
suites: RawSuite[];
|
|
46
|
+
/** Number of distinct paths probed. */
|
|
47
|
+
probedPaths: number;
|
|
48
|
+
/** Paths skipped because every method in {GET,POST,PUT,PATCH,DELETE} is declared. */
|
|
49
|
+
skippedPaths: number;
|
|
50
|
+
/** Total generated probe steps. */
|
|
51
|
+
totalProbes: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ──────────────────────────────────────────────
|
|
55
|
+
// Helpers
|
|
56
|
+
// ──────────────────────────────────────────────
|
|
57
|
+
|
|
58
|
+
function convertPath(path: string): string {
|
|
59
|
+
return path.replace(/\{([^}]+)\}/g, "{{$1}}");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function slugify(s: string): string {
|
|
63
|
+
return s
|
|
64
|
+
.toLowerCase()
|
|
65
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
66
|
+
.replace(/^-|-$/g, "");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function pathStem(path: string): string {
|
|
70
|
+
const cleaned = path
|
|
71
|
+
.replace(/\{[^}]+\}/g, "by-id")
|
|
72
|
+
.replace(/^\//, "")
|
|
73
|
+
.replace(/\//g, "-");
|
|
74
|
+
return slugify(cleaned) || "root";
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Replace path params with valid-shape placeholders so the request can
|
|
78
|
+
* reach the routing layer without being rejected purely on path syntax. */
|
|
79
|
+
function pathWithPlaceholders(
|
|
80
|
+
path: string,
|
|
81
|
+
parameters: OpenAPIV3.ParameterObject[],
|
|
82
|
+
): string {
|
|
83
|
+
return path.replace(/\{([^}]+)\}/g, (_, name: string) => {
|
|
84
|
+
const param = parameters.find((p) => p.name === name && p.in === "path");
|
|
85
|
+
const schema = param?.schema as OpenAPIV3.SchemaObject | undefined;
|
|
86
|
+
if (schema?.format === "uuid") return "00000000-0000-0000-0000-000000000000";
|
|
87
|
+
if (schema?.type === "integer" || schema?.type === "number") return "999999999";
|
|
88
|
+
return "nonexistent-zzzzz";
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function getAuthHeaders(
|
|
93
|
+
ep: EndpointInfo,
|
|
94
|
+
schemes: SecuritySchemeInfo[],
|
|
95
|
+
): Record<string, string> | undefined {
|
|
96
|
+
if (ep.security.length === 0) return undefined;
|
|
97
|
+
for (const secName of ep.security) {
|
|
98
|
+
const scheme = schemes.find((s) => s.name === secName);
|
|
99
|
+
if (!scheme) continue;
|
|
100
|
+
if (scheme.type === "http") {
|
|
101
|
+
if (scheme.scheme === "bearer" || !scheme.scheme) {
|
|
102
|
+
return { Authorization: "Bearer {{auth_token}}" };
|
|
103
|
+
}
|
|
104
|
+
if (scheme.scheme === "basic") {
|
|
105
|
+
return { Authorization: "Basic {{auth_token}}" };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (scheme.type === "apiKey" && scheme.in === "header" && scheme.apiKeyName) {
|
|
109
|
+
if (scheme.apiKeyName === "Authorization") {
|
|
110
|
+
return { Authorization: "Bearer {{auth_token}}" };
|
|
111
|
+
}
|
|
112
|
+
return { [scheme.apiKeyName]: "{{api_key}}" };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
interface PathBucket {
|
|
119
|
+
path: string;
|
|
120
|
+
/** Methods declared on this path, normalized to upper-case. */
|
|
121
|
+
declared: Set<string>;
|
|
122
|
+
/** A representative endpoint we can borrow auth/path-param shape from. */
|
|
123
|
+
sample: EndpointInfo;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function bucketByPath(endpoints: EndpointInfo[]): PathBucket[] {
|
|
127
|
+
const map = new Map<string, PathBucket>();
|
|
128
|
+
for (const ep of endpoints) {
|
|
129
|
+
if (ep.deprecated) continue;
|
|
130
|
+
let bucket = map.get(ep.path);
|
|
131
|
+
if (!bucket) {
|
|
132
|
+
bucket = { path: ep.path, declared: new Set(), sample: ep };
|
|
133
|
+
map.set(ep.path, bucket);
|
|
134
|
+
}
|
|
135
|
+
bucket.declared.add(ep.method.toUpperCase());
|
|
136
|
+
}
|
|
137
|
+
return Array.from(map.values());
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ──────────────────────────────────────────────
|
|
141
|
+
// Public API
|
|
142
|
+
// ──────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
export function generateMethodProbes(opts: MethodProbeOptions): MethodProbeResult {
|
|
145
|
+
const { endpoints, securitySchemes } = opts;
|
|
146
|
+
const methodSet: readonly Method[] = ALL_METHODS;
|
|
147
|
+
|
|
148
|
+
const buckets = bucketByPath(endpoints);
|
|
149
|
+
const suites: RawSuite[] = [];
|
|
150
|
+
let probedPaths = 0;
|
|
151
|
+
let skippedPaths = 0;
|
|
152
|
+
let totalProbes = 0;
|
|
153
|
+
|
|
154
|
+
for (const bucket of buckets) {
|
|
155
|
+
const missing = methodSet.filter((m) => !bucket.declared.has(m));
|
|
156
|
+
if (missing.length === 0) {
|
|
157
|
+
skippedPaths++;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const concretePath = pathWithPlaceholders(
|
|
162
|
+
bucket.path,
|
|
163
|
+
bucket.sample.parameters,
|
|
164
|
+
);
|
|
165
|
+
const headers = getAuthHeaders(bucket.sample, securitySchemes);
|
|
166
|
+
|
|
167
|
+
const steps: RawStep[] = missing.map((method) => {
|
|
168
|
+
const step: RawStep = {
|
|
169
|
+
name: `${method} ${bucket.path} — undeclared method must reject (no 5xx, no 2xx)`,
|
|
170
|
+
[method]: convertPath(concretePath),
|
|
171
|
+
expect: { status: ACCEPTABLE_STATUSES },
|
|
172
|
+
};
|
|
173
|
+
// Body-bearing methods on an undeclared route — send a minimal valid
|
|
174
|
+
// JSON object to provoke any body-parsing path while the router is
|
|
175
|
+
// still expected to reject the method first.
|
|
176
|
+
if (method === "POST" || method === "PUT" || method === "PATCH") {
|
|
177
|
+
(step as any).json = {};
|
|
178
|
+
}
|
|
179
|
+
return step;
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
probedPaths++;
|
|
183
|
+
totalProbes += steps.length;
|
|
184
|
+
|
|
185
|
+
const stem = pathStem(bucket.path);
|
|
186
|
+
suites.push({
|
|
187
|
+
name: `probe methods ${bucket.path}`,
|
|
188
|
+
tags: ["probe-methods", "negative-method", "no-5xx", "smoke"],
|
|
189
|
+
fileStem: `probe-methods-${stem}`,
|
|
190
|
+
base_url: "{{base_url}}",
|
|
191
|
+
...(headers ? { headers } : {}),
|
|
192
|
+
tests: steps,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return { suites, probedPaths, skippedPaths, totalProbes };
|
|
197
|
+
}
|