@kirrosh/zond 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -3
- package/README.md +26 -15
- package/package.json +10 -6
- package/src/cli/commands/catalog.ts +62 -0
- package/src/cli/commands/ci-init.ts +12 -6
- package/src/cli/commands/completions.ts +176 -0
- package/src/cli/commands/db.ts +2 -1
- package/src/cli/commands/generate.ts +18 -2
- package/src/cli/commands/init/agents-md.ts +61 -0
- package/src/cli/commands/init/bootstrap.ts +79 -0
- package/src/cli/commands/init/skills.ts +45 -0
- package/src/cli/commands/init/templates/agents.md +73 -0
- package/src/cli/commands/init/templates/markdown.d.ts +4 -0
- package/src/cli/commands/init/templates/skills/scenarios.md +97 -0
- package/src/cli/commands/init/templates/skills/zond.md +184 -0
- package/src/cli/commands/init/templates/zond-config.yml +15 -0
- package/src/cli/commands/init.ts +124 -31
- package/src/cli/commands/probe-methods.ts +108 -0
- package/src/cli/commands/probe-validation.ts +124 -0
- package/src/cli/commands/run.ts +99 -10
- package/src/cli/commands/serve.ts +52 -19
- package/src/cli/commands/sync.ts +28 -1
- package/src/cli/commands/update.ts +1 -1
- package/src/cli/commands/use.ts +57 -0
- package/src/cli/index.ts +21 -591
- package/src/cli/program.ts +655 -0
- package/src/cli/version.ts +3 -0
- package/src/core/context/current.ts +35 -0
- package/src/core/diagnostics/db-analysis.ts +11 -2
- package/src/core/diagnostics/render-md.ts +112 -0
- package/src/core/generator/catalog-builder.ts +179 -0
- package/src/core/generator/chunker.ts +14 -2
- package/src/core/generator/data-factory.ts +50 -19
- package/src/core/generator/guide-builder.ts +1 -1
- package/src/core/generator/index.ts +2 -0
- package/src/core/generator/openapi-reader.ts +18 -0
- package/src/core/generator/serializer.ts +11 -2
- package/src/core/generator/suite-generator.ts +106 -7
- package/src/core/meta/types.ts +0 -2
- package/src/core/parser/schema.ts +3 -1
- package/src/core/parser/types.ts +10 -1
- package/src/core/parser/variables.ts +90 -2
- package/src/core/parser/yaml-parser.ts +50 -1
- package/src/core/probe/method-probe.ts +197 -0
- package/src/core/probe/negative-probe.ts +657 -0
- package/src/core/reporter/console.ts +29 -3
- package/src/core/reporter/index.ts +2 -2
- package/src/core/reporter/json.ts +5 -2
- package/src/core/runner/assertions.ts +4 -1
- package/src/core/runner/executor.ts +132 -37
- package/src/core/runner/http-client.ts +40 -5
- package/src/core/runner/rate-limiter.ts +131 -0
- package/src/core/setup-api.ts +4 -1
- package/src/core/workspace/root.ts +94 -0
- package/src/db/schema.ts +4 -1
- package/src/web/routes/api.ts +80 -0
- package/src/web/routes/dashboard.ts +15 -0
- package/src/web/static/style.css +290 -0
- package/src/web/views/explorer-tab.ts +402 -0
|
@@ -8,11 +8,25 @@ const DIM = "\x1b[2m";
|
|
|
8
8
|
const GREEN = "\x1b[32m";
|
|
9
9
|
const RED = "\x1b[31m";
|
|
10
10
|
const GRAY = "\x1b[90m";
|
|
11
|
+
const YELLOW = "\x1b[33m";
|
|
11
12
|
|
|
12
13
|
const PASS_ICON = "\u2713"; // ✓
|
|
13
14
|
const FAIL_ICON = "\u2717"; // ✗
|
|
14
15
|
const SKIP_ICON = "\u25CB"; // ○
|
|
15
16
|
|
|
17
|
+
export function is5xx(step: StepResult): boolean {
|
|
18
|
+
const status = step.response?.status;
|
|
19
|
+
return typeof status === "number" && status >= 500 && status < 600;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function count5xx(steps: StepResult[]): number {
|
|
23
|
+
let n = 0;
|
|
24
|
+
for (const s of steps) {
|
|
25
|
+
if ((s.status === "fail" || s.status === "error") && is5xx(s)) n++;
|
|
26
|
+
}
|
|
27
|
+
return n;
|
|
28
|
+
}
|
|
29
|
+
|
|
16
30
|
export function formatDuration(ms: number): string {
|
|
17
31
|
if (ms < 1000) return `${Math.round(ms)}ms`;
|
|
18
32
|
if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
|
|
@@ -33,11 +47,13 @@ export function formatStep(step: StepResult, color: boolean): string {
|
|
|
33
47
|
case "fail": {
|
|
34
48
|
const icon = color ? `${RED}${FAIL_ICON}${RESET}` : FAIL_ICON;
|
|
35
49
|
const dim = color ? `${DIM}(${duration})${RESET}` : `(${duration})`;
|
|
36
|
-
|
|
50
|
+
const tag = is5xx(step) ? (color ? ` ${BOLD}${YELLOW}[5xx ${step.response?.status}]${RESET}` : ` [5xx ${step.response?.status}]`) : "";
|
|
51
|
+
return ` ${icon} ${step.name}${tag} ${dim}`;
|
|
37
52
|
}
|
|
38
53
|
case "skip": {
|
|
39
54
|
const icon = color ? `${GRAY}${SKIP_ICON}${RESET}` : SKIP_ICON;
|
|
40
|
-
const
|
|
55
|
+
const reason = step.error ? `skipped: ${step.error}` : "skipped";
|
|
56
|
+
const label = color ? `${GRAY}(${reason})${RESET}` : `(${reason})`;
|
|
41
57
|
return ` ${icon} ${step.name} ${label}`;
|
|
42
58
|
}
|
|
43
59
|
case "error": {
|
|
@@ -105,6 +121,11 @@ export function formatSuiteResult(result: TestRunResult, color: boolean): string
|
|
|
105
121
|
if (result.failed > 0) {
|
|
106
122
|
parts.push(color ? `${RED}${result.failed} failed${RESET}` : `${result.failed} failed`);
|
|
107
123
|
}
|
|
124
|
+
const fiveXx = count5xx(result.steps);
|
|
125
|
+
if (fiveXx > 0) {
|
|
126
|
+
const label = `${fiveXx} 5xx`;
|
|
127
|
+
parts.push(color ? `${BOLD}${YELLOW}${label}${RESET}` : label);
|
|
128
|
+
}
|
|
108
129
|
if (result.skipped > 0) {
|
|
109
130
|
parts.push(color ? `${GRAY}${result.skipped} skipped${RESET}` : `${result.skipped} skipped`);
|
|
110
131
|
}
|
|
@@ -119,7 +140,7 @@ export function formatSuiteResult(result: TestRunResult, color: boolean): string
|
|
|
119
140
|
}
|
|
120
141
|
|
|
121
142
|
export function formatGrandTotal(results: TestRunResult[], color: boolean): string {
|
|
122
|
-
const totals = { passed: 0, failed: 0, skipped: 0, total: 0 };
|
|
143
|
+
const totals = { passed: 0, failed: 0, skipped: 0, total: 0, fiveXx: 0 };
|
|
123
144
|
let minStart = Infinity;
|
|
124
145
|
let maxEnd = -Infinity;
|
|
125
146
|
|
|
@@ -128,6 +149,7 @@ export function formatGrandTotal(results: TestRunResult[], color: boolean): stri
|
|
|
128
149
|
totals.failed += r.failed;
|
|
129
150
|
totals.skipped += r.skipped;
|
|
130
151
|
totals.total += r.total;
|
|
152
|
+
totals.fiveXx += count5xx(r.steps);
|
|
131
153
|
const start = Date.parse(r.started_at);
|
|
132
154
|
const end = Date.parse(r.finished_at);
|
|
133
155
|
if (start < minStart) minStart = start;
|
|
@@ -144,6 +166,10 @@ export function formatGrandTotal(results: TestRunResult[], color: boolean): stri
|
|
|
144
166
|
if (totals.failed > 0) {
|
|
145
167
|
parts.push(color ? `${RED}${totals.failed} failed${RESET}` : `${totals.failed} failed`);
|
|
146
168
|
}
|
|
169
|
+
if (totals.fiveXx > 0) {
|
|
170
|
+
const label = `${totals.fiveXx} 5xx`;
|
|
171
|
+
parts.push(color ? `${BOLD}${YELLOW}${label}${RESET}` : label);
|
|
172
|
+
}
|
|
147
173
|
if (totals.skipped > 0) {
|
|
148
174
|
parts.push(color ? `${GRAY}${totals.skipped} skipped${RESET}` : `${totals.skipped} skipped`);
|
|
149
175
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export type { Reporter, ReporterOptions, ReporterName } from "./types.ts";
|
|
2
2
|
export { consoleReporter, formatDuration, formatStep, formatFailures, formatSuiteResult, formatGrandTotal } from "./console.ts";
|
|
3
|
-
export { jsonReporter } from "./json.ts";
|
|
4
|
-
export { junitReporter } from "./junit.ts";
|
|
3
|
+
export { jsonReporter, generateJsonReport } from "./json.ts";
|
|
4
|
+
export { junitReporter, generateJunitXml } from "./junit.ts";
|
|
5
5
|
|
|
6
6
|
import type { Reporter, ReporterName } from "./types.ts";
|
|
7
7
|
import { consoleReporter } from "./console.ts";
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import type { TestRunResult } from "../runner/types.ts";
|
|
2
2
|
import type { Reporter, ReporterOptions } from "./types.ts";
|
|
3
3
|
|
|
4
|
+
export function generateJsonReport(results: TestRunResult[]): string {
|
|
5
|
+
return JSON.stringify(results, null, 2);
|
|
6
|
+
}
|
|
7
|
+
|
|
4
8
|
export const jsonReporter: Reporter = {
|
|
5
9
|
report(results: TestRunResult[], _options?: ReporterOptions): void {
|
|
6
|
-
|
|
7
|
-
console.log(json);
|
|
10
|
+
console.log(generateJsonReport(results));
|
|
8
11
|
},
|
|
9
12
|
};
|
|
@@ -10,6 +10,7 @@ function checkType(value: unknown, expectedType: string): boolean {
|
|
|
10
10
|
case "boolean": return typeof value === "boolean";
|
|
11
11
|
case "array": return Array.isArray(value);
|
|
12
12
|
case "object": return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13
|
+
case "null": return value === null;
|
|
13
14
|
default: return false;
|
|
14
15
|
}
|
|
15
16
|
}
|
|
@@ -37,7 +38,9 @@ function checkRule(path: string, rule: AssertionRule, actual: unknown): Assertio
|
|
|
37
38
|
const field = `body.${path}`;
|
|
38
39
|
|
|
39
40
|
if (rule.exists !== undefined) {
|
|
40
|
-
|
|
41
|
+
// Key-presence semantics: null counts as "exists" (key present in response).
|
|
42
|
+
// Use `not_equals: null` or `type: "null"` to assert non-null specifically.
|
|
43
|
+
const doesExist = actual !== undefined;
|
|
41
44
|
results.push({
|
|
42
45
|
field, rule: `exists ${rule.exists}`,
|
|
43
46
|
passed: doesExist === rule.exists, actual: doesExist, expected: rule.exists,
|
|
@@ -3,6 +3,7 @@ import type { TestSuite, TestStep, Environment } from "../parser/types.ts";
|
|
|
3
3
|
import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
|
|
4
4
|
import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
|
|
5
5
|
import { executeRequest, type FetchOptions } from "./http-client.ts";
|
|
6
|
+
import type { RateLimiter } from "./rate-limiter.ts";
|
|
6
7
|
import { checkAssertions, extractCaptures } from "./assertions.ts";
|
|
7
8
|
import { evaluateExpr } from "./expr-eval.ts";
|
|
8
9
|
import { applyTransform } from "./transforms.ts";
|
|
@@ -28,19 +29,77 @@ function makeSkippedResult(stepName: string, reason: string): StepResult {
|
|
|
28
29
|
};
|
|
29
30
|
}
|
|
30
31
|
|
|
31
|
-
|
|
32
|
+
/** Interpolate {{var}} placeholders inside a test/step name. Falls back to
|
|
33
|
+
* the raw name string if substitution returns a non-string value. */
|
|
34
|
+
function interpolateName(name: string, vars: Record<string, unknown>): string {
|
|
35
|
+
try {
|
|
36
|
+
const out = substituteString(name, vars);
|
|
37
|
+
return typeof out === "string" ? out : String(out);
|
|
38
|
+
} catch {
|
|
39
|
+
return name;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Expand a `parameterize: { key: [val, ...] }` map into the cross-product of
|
|
45
|
+
* iteration variable bindings. No `parameterize` (or an empty map) yields a
|
|
46
|
+
* single empty iteration so the existing single-pass behaviour is preserved.
|
|
47
|
+
*
|
|
48
|
+
* Exported for tests.
|
|
49
|
+
*/
|
|
50
|
+
export function expandParameterize(params?: Record<string, unknown[]>): Record<string, unknown>[] {
|
|
51
|
+
if (!params) return [{}];
|
|
52
|
+
const keys = Object.keys(params).filter(k => Array.isArray(params[k]) && (params[k] as unknown[]).length > 0);
|
|
53
|
+
if (keys.length === 0) return [{}];
|
|
54
|
+
let combos: Record<string, unknown>[] = [{}];
|
|
55
|
+
for (const k of keys) {
|
|
56
|
+
const values = params[k] as unknown[];
|
|
57
|
+
const next: Record<string, unknown>[] = [];
|
|
58
|
+
for (const combo of combos) {
|
|
59
|
+
for (const v of values) {
|
|
60
|
+
next.push({ ...combo, [k]: v });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
combos = next;
|
|
64
|
+
}
|
|
65
|
+
return combos;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface RunSuiteOptions {
|
|
69
|
+
rateLimiter?: RateLimiter;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function runSuite(
|
|
73
|
+
suite: TestSuite,
|
|
74
|
+
env: Environment = {},
|
|
75
|
+
dryRun = false,
|
|
76
|
+
options: RunSuiteOptions = {},
|
|
77
|
+
): Promise<TestRunResult> {
|
|
32
78
|
const startedAt = new Date().toISOString();
|
|
33
79
|
const steps: StepResult[] = [];
|
|
34
|
-
const variables: Record<string, unknown> = { ...env };
|
|
35
|
-
const failedCaptures = new Set<string>();
|
|
36
80
|
|
|
37
81
|
const fetchOptions: Partial<FetchOptions> = {
|
|
38
82
|
timeout: suite.config.timeout,
|
|
39
83
|
retries: suite.config.retries,
|
|
40
84
|
retry_delay: suite.config.retry_delay,
|
|
41
85
|
follow_redirects: suite.config.follow_redirects,
|
|
86
|
+
rate_limiter: options.rateLimiter,
|
|
42
87
|
};
|
|
43
88
|
|
|
89
|
+
// parameterize cross-product → N iterations of the suite body.
|
|
90
|
+
// Captures and tainted/missing sets are reset per iteration so that
|
|
91
|
+
// values from one binding never leak into the next.
|
|
92
|
+
const iterations = expandParameterize(suite.parameterize);
|
|
93
|
+
|
|
94
|
+
for (const iterVars of iterations) {
|
|
95
|
+
const variables: Record<string, unknown> = { ...env, ...iterVars };
|
|
96
|
+
// Captures whose source step's assertions partially failed, but the value
|
|
97
|
+
// itself was extracted. Cleanup/always steps may still consume them.
|
|
98
|
+
const taintedCaptures = new Set<string>();
|
|
99
|
+
// Captures that were never extracted (response missing the field). Even
|
|
100
|
+
// always-steps can't run if their referenced capture is missing.
|
|
101
|
+
const missingCaptures = new Set<string>();
|
|
102
|
+
|
|
44
103
|
// Expand steps lazily (for_each needs current variables)
|
|
45
104
|
let stepIndex = 0;
|
|
46
105
|
const rawSteps = [...suite.tests];
|
|
@@ -86,7 +145,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
86
145
|
variables[key] = applyTransform(substituted);
|
|
87
146
|
}
|
|
88
147
|
steps.push({
|
|
89
|
-
name: step.name,
|
|
148
|
+
name: interpolateName(step.name, variables),
|
|
90
149
|
status: "pass",
|
|
91
150
|
duration_ms: 0,
|
|
92
151
|
request: { method: "", url: "", headers: {} },
|
|
@@ -96,37 +155,65 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
96
155
|
continue;
|
|
97
156
|
}
|
|
98
157
|
|
|
99
|
-
// Skip check: if step references a failed capture
|
|
158
|
+
// Skip check: if step references a failed capture, skip — unless
|
|
159
|
+
// step is `always: true` AND the capture is just tainted (still extracted).
|
|
100
160
|
const referencedVars = extractVariableReferences(step);
|
|
101
|
-
const
|
|
102
|
-
if (
|
|
103
|
-
steps.push(makeSkippedResult(step.name, `Depends on missing capture: ${
|
|
161
|
+
const missing = referencedVars.find((v) => missingCaptures.has(v));
|
|
162
|
+
if (missing) {
|
|
163
|
+
steps.push(makeSkippedResult(interpolateName(step.name, variables), `Depends on missing capture: ${missing}`));
|
|
104
164
|
continue;
|
|
105
165
|
}
|
|
166
|
+
if (!step.always) {
|
|
167
|
+
const tainted = referencedVars.find((v) => taintedCaptures.has(v));
|
|
168
|
+
if (tainted) {
|
|
169
|
+
steps.push(makeSkippedResult(interpolateName(step.name, variables), `Depends on tainted capture: ${tainted} (use always: true on cleanup steps)`));
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
106
173
|
|
|
107
174
|
// skip_if evaluation
|
|
108
175
|
if (step.skip_if) {
|
|
109
176
|
const exprAfterSubst = String(substituteString(step.skip_if, variables));
|
|
110
177
|
if (evaluateExpr(exprAfterSubst)) {
|
|
111
|
-
steps.push(makeSkippedResult(step.name, `Skipped: ${step.skip_if}`));
|
|
178
|
+
steps.push(makeSkippedResult(interpolateName(step.name, variables), `Skipped: ${step.skip_if}`));
|
|
112
179
|
continue;
|
|
113
180
|
}
|
|
114
181
|
}
|
|
115
182
|
|
|
116
|
-
// Process set: on HTTP steps — evaluate generators once before building request
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
183
|
+
// Process set: on HTTP steps — evaluate generators once before building request.
|
|
184
|
+
// Substitution can throw on unknown {{$generator}} — fail this step, not the suite.
|
|
185
|
+
let resolved: TestStep;
|
|
186
|
+
let resolvedBaseUrl: string | undefined;
|
|
187
|
+
let resolvedSuiteHeaders: Record<string, string> | undefined;
|
|
188
|
+
try {
|
|
189
|
+
if (step.set) {
|
|
190
|
+
for (const [key, rawDirective] of Object.entries(step.set)) {
|
|
191
|
+
const substituted = substituteDeep(rawDirective, variables);
|
|
192
|
+
variables[key] = applyTransform(substituted);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
resolved = substituteStep(step, variables);
|
|
196
|
+
resolvedBaseUrl = suite.base_url ? substituteString(suite.base_url, variables) as string : undefined;
|
|
197
|
+
resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
|
|
198
|
+
} catch (err) {
|
|
199
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
200
|
+
steps.push({
|
|
201
|
+
name: interpolateName(step.name, variables),
|
|
202
|
+
status: "error",
|
|
203
|
+
duration_ms: 0,
|
|
204
|
+
request: { method: step.method, url: step.path, headers: {} },
|
|
205
|
+
assertions: [],
|
|
206
|
+
captures: {},
|
|
207
|
+
error: errorMsg,
|
|
208
|
+
});
|
|
209
|
+
// Substitution never produced a request → capture truly missing.
|
|
210
|
+
if (step.expect.body) {
|
|
211
|
+
for (const rule of Object.values(step.expect.body)) {
|
|
212
|
+
if (rule.capture) missingCaptures.add(rule.capture);
|
|
213
|
+
}
|
|
121
214
|
}
|
|
215
|
+
continue;
|
|
122
216
|
}
|
|
123
|
-
|
|
124
|
-
// Substitute variables
|
|
125
|
-
const resolved = substituteStep(step, variables);
|
|
126
|
-
|
|
127
|
-
// Build request — substitute base_url and suite headers with current variables
|
|
128
|
-
const resolvedBaseUrl = suite.base_url ? substituteString(suite.base_url, variables) as string : undefined;
|
|
129
|
-
const resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
|
|
130
217
|
const url = buildUrl(resolvedBaseUrl, resolved.path, resolved.query);
|
|
131
218
|
const headers: Record<string, string> = { ...resolvedSuiteHeaders, ...resolved.headers };
|
|
132
219
|
let body: string | undefined;
|
|
@@ -163,7 +250,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
163
250
|
// Validate absolute URL before attempting fetch
|
|
164
251
|
if (!url.startsWith("http://") && !url.startsWith("https://")) {
|
|
165
252
|
steps.push({
|
|
166
|
-
name: step.name,
|
|
253
|
+
name: interpolateName(step.name, variables),
|
|
167
254
|
status: "error",
|
|
168
255
|
duration_ms: 0,
|
|
169
256
|
request,
|
|
@@ -173,7 +260,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
173
260
|
});
|
|
174
261
|
if (step.expect.body) {
|
|
175
262
|
for (const rule of Object.values(step.expect.body)) {
|
|
176
|
-
if (rule.capture)
|
|
263
|
+
if (rule.capture) missingCaptures.add(rule.capture);
|
|
177
264
|
}
|
|
178
265
|
}
|
|
179
266
|
continue;
|
|
@@ -184,7 +271,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
184
271
|
? ` [multipart: ${[...formData.keys()].length} field(s)]`
|
|
185
272
|
: body ? ` ${body.slice(0, 200)}` : "";
|
|
186
273
|
steps.push({
|
|
187
|
-
name: step.name,
|
|
274
|
+
name: interpolateName(step.name, variables),
|
|
188
275
|
status: "pass",
|
|
189
276
|
duration_ms: 0,
|
|
190
277
|
request,
|
|
@@ -207,7 +294,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
207
294
|
const allPassed = assertions.every((a) => a.passed);
|
|
208
295
|
|
|
209
296
|
lastStepResult = {
|
|
210
|
-
name: step.name,
|
|
297
|
+
name: interpolateName(step.name, variables),
|
|
211
298
|
status: allPassed ? "pass" : "fail",
|
|
212
299
|
duration_ms: response.duration_ms,
|
|
213
300
|
request,
|
|
@@ -234,7 +321,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
234
321
|
}
|
|
235
322
|
} catch (err) {
|
|
236
323
|
lastStepResult = {
|
|
237
|
-
name: step.name,
|
|
324
|
+
name: interpolateName(step.name, variables),
|
|
238
325
|
status: "error",
|
|
239
326
|
duration_ms: 0,
|
|
240
327
|
request,
|
|
@@ -255,11 +342,11 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
255
342
|
const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
|
|
256
343
|
Object.assign(variables, captures);
|
|
257
344
|
|
|
258
|
-
// Track expected captures that weren't obtained
|
|
345
|
+
// Track expected captures that weren't obtained — these are missing.
|
|
259
346
|
if (resolved.expect.body) {
|
|
260
347
|
for (const rule of Object.values(resolved.expect.body)) {
|
|
261
348
|
if (rule.capture && !(rule.capture in captures)) {
|
|
262
|
-
|
|
349
|
+
missingCaptures.add(rule.capture);
|
|
263
350
|
}
|
|
264
351
|
}
|
|
265
352
|
}
|
|
@@ -269,7 +356,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
269
356
|
const allPassed = assertions.every((a) => a.passed);
|
|
270
357
|
|
|
271
358
|
steps.push({
|
|
272
|
-
name: step.name,
|
|
359
|
+
name: interpolateName(step.name, variables),
|
|
273
360
|
status: allPassed ? "pass" : "fail",
|
|
274
361
|
duration_ms: response.duration_ms,
|
|
275
362
|
request,
|
|
@@ -278,18 +365,20 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
278
365
|
captures,
|
|
279
366
|
});
|
|
280
367
|
|
|
281
|
-
// If step failed,
|
|
368
|
+
// If step failed, captures that did extract are tainted (value is real
|
|
369
|
+
// but came from a step whose other assertions failed). Always-steps may
|
|
370
|
+
// still consume them; non-always steps cascade-skip.
|
|
282
371
|
if (!allPassed && resolved.expect.body) {
|
|
283
372
|
for (const rule of Object.values(resolved.expect.body)) {
|
|
284
|
-
if (rule.capture) {
|
|
285
|
-
|
|
373
|
+
if (rule.capture && rule.capture in captures) {
|
|
374
|
+
taintedCaptures.add(rule.capture);
|
|
286
375
|
}
|
|
287
376
|
}
|
|
288
377
|
}
|
|
289
378
|
} catch (err) {
|
|
290
379
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
291
380
|
steps.push({
|
|
292
|
-
name: step.name,
|
|
381
|
+
name: interpolateName(step.name, variables),
|
|
293
382
|
status: "error",
|
|
294
383
|
duration_ms: 0,
|
|
295
384
|
request,
|
|
@@ -298,14 +387,15 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
298
387
|
error: errorMsg,
|
|
299
388
|
});
|
|
300
389
|
|
|
301
|
-
//
|
|
390
|
+
// Network/runtime error → no response → capture truly missing.
|
|
302
391
|
if (step.expect.body) {
|
|
303
392
|
for (const rule of Object.values(step.expect.body)) {
|
|
304
|
-
if (rule.capture)
|
|
393
|
+
if (rule.capture) missingCaptures.add(rule.capture);
|
|
305
394
|
}
|
|
306
395
|
}
|
|
307
396
|
}
|
|
308
397
|
}
|
|
398
|
+
} // end of parameterize iteration loop
|
|
309
399
|
|
|
310
400
|
const finishedAt = new Date().toISOString();
|
|
311
401
|
return {
|
|
@@ -323,6 +413,11 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
323
413
|
};
|
|
324
414
|
}
|
|
325
415
|
|
|
326
|
-
export async function runSuites(
|
|
327
|
-
|
|
416
|
+
export async function runSuites(
|
|
417
|
+
suites: TestSuite[],
|
|
418
|
+
env: Environment = {},
|
|
419
|
+
dryRun = false,
|
|
420
|
+
options: RunSuiteOptions = {},
|
|
421
|
+
): Promise<TestRunResult[]> {
|
|
422
|
+
return Promise.all(suites.map((suite) => runSuite(suite, env, dryRun, options)));
|
|
328
423
|
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { HttpRequest, HttpResponse } from "./types.ts";
|
|
2
|
+
import { type RateLimiter, parseRetryAfter, parseRateLimitHeaders } from "./rate-limiter.ts";
|
|
2
3
|
|
|
3
4
|
export interface FetchOptions {
|
|
4
5
|
timeout: number;
|
|
5
6
|
retries: number;
|
|
6
7
|
retry_delay: number;
|
|
7
8
|
follow_redirects: boolean;
|
|
9
|
+
rate_limiter?: RateLimiter;
|
|
10
|
+
rate_limit_retries: number;
|
|
11
|
+
rate_limit_max_delay_ms: number;
|
|
8
12
|
}
|
|
9
13
|
|
|
10
14
|
export const DEFAULT_FETCH_OPTIONS: FetchOptions = {
|
|
@@ -12,6 +16,8 @@ export const DEFAULT_FETCH_OPTIONS: FetchOptions = {
|
|
|
12
16
|
retries: 0,
|
|
13
17
|
retry_delay: 1000,
|
|
14
18
|
follow_redirects: true,
|
|
19
|
+
rate_limit_retries: 5,
|
|
20
|
+
rate_limit_max_delay_ms: 30000,
|
|
15
21
|
};
|
|
16
22
|
|
|
17
23
|
export async function executeRequest(
|
|
@@ -20,10 +26,12 @@ export async function executeRequest(
|
|
|
20
26
|
): Promise<HttpResponse> {
|
|
21
27
|
const opts = { ...DEFAULT_FETCH_OPTIONS, ...options };
|
|
22
28
|
let lastError: Error | undefined;
|
|
29
|
+
let networkAttempt = 0;
|
|
30
|
+
let rate429Attempt = 0;
|
|
23
31
|
|
|
24
|
-
|
|
25
|
-
if (
|
|
26
|
-
await
|
|
32
|
+
while (true) {
|
|
33
|
+
if (opts.rate_limiter) {
|
|
34
|
+
await opts.rate_limiter.acquire();
|
|
27
35
|
}
|
|
28
36
|
|
|
29
37
|
try {
|
|
@@ -43,6 +51,20 @@ export async function executeRequest(
|
|
|
43
51
|
clearTimeout(timeoutId);
|
|
44
52
|
const duration_ms = Math.round(performance.now() - start);
|
|
45
53
|
|
|
54
|
+
if (response.status === 429 && rate429Attempt < opts.rate_limit_retries) {
|
|
55
|
+
const retryAfterMs = parseRetryAfter(response.headers.get("retry-after"));
|
|
56
|
+
const backoffMs = Math.min(
|
|
57
|
+
opts.retry_delay * 2 ** rate429Attempt,
|
|
58
|
+
opts.rate_limit_max_delay_ms,
|
|
59
|
+
);
|
|
60
|
+
const waitMs = Math.min(retryAfterMs ?? backoffMs, opts.rate_limit_max_delay_ms);
|
|
61
|
+
rate429Attempt++;
|
|
62
|
+
// Drain body so the connection can be reused
|
|
63
|
+
await response.text().catch(() => undefined);
|
|
64
|
+
await Bun.sleep(waitMs);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
46
68
|
const bodyText = await response.text();
|
|
47
69
|
let body_parsed: unknown = undefined;
|
|
48
70
|
const contentType = response.headers.get("content-type") ?? "";
|
|
@@ -69,11 +91,24 @@ export async function executeRequest(
|
|
|
69
91
|
headers[k] = v;
|
|
70
92
|
});
|
|
71
93
|
|
|
94
|
+
// Feed ratelimit-* headers back into the limiter so it can pause the
|
|
95
|
+
// stream proactively when the window is nearly exhausted (TASK-81).
|
|
96
|
+
if (opts.rate_limiter?.note) {
|
|
97
|
+
const meta = parseRateLimitHeaders(headers);
|
|
98
|
+
if (meta.remaining !== undefined || meta.reset !== undefined) {
|
|
99
|
+
opts.rate_limiter.note(meta);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
72
103
|
return { status: response.status, headers, body: bodyText, body_parsed, duration_ms };
|
|
73
104
|
} catch (err) {
|
|
74
105
|
lastError = err instanceof Error ? err : new Error(String(err));
|
|
106
|
+
if (networkAttempt < opts.retries) {
|
|
107
|
+
networkAttempt++;
|
|
108
|
+
await Bun.sleep(opts.retry_delay);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
throw lastError;
|
|
75
112
|
}
|
|
76
113
|
}
|
|
77
|
-
|
|
78
|
-
throw lastError!;
|
|
79
114
|
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
export interface RateLimiter {
|
|
2
|
+
acquire(): Promise<void>;
|
|
3
|
+
/**
|
|
4
|
+
* Feed rate-limit metadata from the latest response back into the limiter.
|
|
5
|
+
* When `remaining` falls at or below the threshold, the limiter postpones
|
|
6
|
+
* the next acquire until the API's reset window expires. No-op when the
|
|
7
|
+
* server reports plenty of headroom.
|
|
8
|
+
*
|
|
9
|
+
* Optional so existing callers / mocks need not implement it.
|
|
10
|
+
*/
|
|
11
|
+
note?(meta: RateLimitMeta, now?: number): void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface RateLimitMeta {
|
|
15
|
+
/** Requests remaining in the current window. */
|
|
16
|
+
remaining?: number;
|
|
17
|
+
/** Either seconds-until-reset (RFC draft) or a Unix epoch in seconds (GitHub style). */
|
|
18
|
+
reset?: number;
|
|
19
|
+
/** Window cap; used only for diagnostics. */
|
|
20
|
+
limit?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** When `remaining` is at or below this number we proactively pause until reset. */
|
|
24
|
+
const THROTTLE_THRESHOLD = 5;
|
|
25
|
+
|
|
26
|
+
/** Magnitudes above this are treated as Unix timestamps; below as relative
|
|
27
|
+
* seconds. 10^9 seconds ≈ Sep 2001, so any real reset window is far below. */
|
|
28
|
+
const UNIX_TS_BOUNDARY = 1_000_000_000;
|
|
29
|
+
|
|
30
|
+
function applyMeta(prevNextAvailable: number, meta: RateLimitMeta, now: number): number {
|
|
31
|
+
if (meta.remaining === undefined) return prevNextAvailable;
|
|
32
|
+
if (meta.remaining > THROTTLE_THRESHOLD) return prevNextAvailable;
|
|
33
|
+
if (meta.reset === undefined || !Number.isFinite(meta.reset)) return prevNextAvailable;
|
|
34
|
+
const resetMs = meta.reset > UNIX_TS_BOUNDARY ? meta.reset * 1000 : now + Math.max(0, meta.reset) * 1000;
|
|
35
|
+
return Math.max(prevNextAvailable, resetMs);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
class IntervalRateLimiter implements RateLimiter {
|
|
39
|
+
private nextAvailable = 0;
|
|
40
|
+
private readonly intervalMs: number;
|
|
41
|
+
|
|
42
|
+
constructor(reqPerSec: number) {
|
|
43
|
+
if (!Number.isFinite(reqPerSec) || reqPerSec <= 0) {
|
|
44
|
+
throw new Error(`Invalid rate limit: ${reqPerSec}`);
|
|
45
|
+
}
|
|
46
|
+
this.intervalMs = 1000 / reqPerSec;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async acquire(): Promise<void> {
|
|
50
|
+
const now = Date.now();
|
|
51
|
+
const slot = Math.max(now, this.nextAvailable);
|
|
52
|
+
const waitMs = slot - now;
|
|
53
|
+
this.nextAvailable = slot + this.intervalMs;
|
|
54
|
+
if (waitMs > 0) {
|
|
55
|
+
await Bun.sleep(waitMs);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
note(meta: RateLimitMeta, now: number = Date.now()): void {
|
|
60
|
+
this.nextAvailable = applyMeta(this.nextAvailable, meta, now);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
class AdaptiveRateLimiter implements RateLimiter {
|
|
65
|
+
private nextAvailable = 0;
|
|
66
|
+
|
|
67
|
+
async acquire(): Promise<void> {
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
const wait = this.nextAvailable - now;
|
|
70
|
+
if (wait > 0) await Bun.sleep(wait);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
note(meta: RateLimitMeta, now: number = Date.now()): void {
|
|
74
|
+
this.nextAvailable = applyMeta(this.nextAvailable, meta, now);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function createRateLimiter(reqPerSec: number | undefined): RateLimiter | undefined {
|
|
79
|
+
if (reqPerSec === undefined || reqPerSec === null) return undefined;
|
|
80
|
+
if (!Number.isFinite(reqPerSec) || reqPerSec <= 0) return undefined;
|
|
81
|
+
return new IntervalRateLimiter(reqPerSec);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Adaptive limiter for `--rate-limit auto`. Issues no proactive throttling on
|
|
86
|
+
* its own, but reacts to ratelimit-* response headers via `note()` and pauses
|
|
87
|
+
* the request stream until the API's reset window elapses when headroom drops.
|
|
88
|
+
*/
|
|
89
|
+
export function createAdaptiveRateLimiter(): RateLimiter {
|
|
90
|
+
return new AdaptiveRateLimiter();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Read RFC draft-ietf-httpapi-ratelimit-headers (`ratelimit-*`) plus the
|
|
95
|
+
* GitHub / Stripe style `x-ratelimit-*` aliases out of a response header bag.
|
|
96
|
+
* All keys are matched case-insensitively. Unparseable values are dropped.
|
|
97
|
+
*/
|
|
98
|
+
export function parseRateLimitHeaders(headers: Record<string, string>): RateLimitMeta {
|
|
99
|
+
const lower: Record<string, string> = {};
|
|
100
|
+
for (const [k, v] of Object.entries(headers)) lower[k.toLowerCase()] = v;
|
|
101
|
+
const num = (v: string | undefined): number | undefined => {
|
|
102
|
+
if (v === undefined) return undefined;
|
|
103
|
+
// RFC draft `ratelimit-remaining` may carry `q="value"` quoted-string form;
|
|
104
|
+
// strip leading numeric run.
|
|
105
|
+
const match = v.match(/-?\d+(?:\.\d+)?/);
|
|
106
|
+
if (!match) return undefined;
|
|
107
|
+
const n = Number.parseFloat(match[0]);
|
|
108
|
+
return Number.isFinite(n) ? n : undefined;
|
|
109
|
+
};
|
|
110
|
+
return {
|
|
111
|
+
limit: num(lower["ratelimit-limit"] ?? lower["x-ratelimit-limit"]),
|
|
112
|
+
remaining: num(lower["ratelimit-remaining"] ?? lower["x-ratelimit-remaining"]),
|
|
113
|
+
reset: num(lower["ratelimit-reset"] ?? lower["x-ratelimit-reset"]),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function parseRetryAfter(header: string | null | undefined, now: number = Date.now()): number | undefined {
|
|
118
|
+
if (!header) return undefined;
|
|
119
|
+
const trimmed = header.trim();
|
|
120
|
+
if (trimmed === "") return undefined;
|
|
121
|
+
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
|
122
|
+
const seconds = Number.parseFloat(trimmed);
|
|
123
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000);
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
const date = Date.parse(trimmed);
|
|
127
|
+
if (!Number.isNaN(date)) {
|
|
128
|
+
return Math.max(0, date - now);
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
package/src/core/setup-api.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync, existsSync, readFileSync } from "fs";
|
|
|
3
3
|
import { getDb } from "../db/schema.ts";
|
|
4
4
|
import { createCollection, deleteCollection, findCollectionByNameOrId, normalizePath } from "../db/queries.ts";
|
|
5
5
|
import { readOpenApiSpec, extractEndpoints } from "./generator/index.ts";
|
|
6
|
+
import { findWorkspaceRoot } from "./workspace/root.ts";
|
|
6
7
|
|
|
7
8
|
function toYaml(vars: Record<string, string>): string {
|
|
8
9
|
const lines: string[] = [];
|
|
@@ -90,7 +91,9 @@ export async function setupApi(options: SetupApiOptions): Promise<SetupApiResult
|
|
|
90
91
|
|
|
91
92
|
// Sanitize name for directory use
|
|
92
93
|
const dirName = name.replace(/[^a-zA-Z0-9_\-\.]/g, "-").toLowerCase();
|
|
93
|
-
const baseDir =
|
|
94
|
+
const baseDir = options.dir
|
|
95
|
+
? resolve(options.dir)
|
|
96
|
+
: resolve(findWorkspaceRoot().root, `apis/${dirName}/`);
|
|
94
97
|
const testPath = join(baseDir, "tests");
|
|
95
98
|
|
|
96
99
|
// Create directories
|