@kirrosh/zond 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -1
- package/src/cli/commands/coverage.ts +47 -42
- package/src/cli/commands/db.ts +20 -5
- package/src/cli/commands/describe.ts +26 -1
- package/src/cli/commands/generate.ts +16 -5
- package/src/cli/commands/run.ts +21 -1
- package/src/cli/commands/update.ts +28 -13
- package/src/cli/index.ts +25 -3
- package/src/core/diagnostics/db-analysis.ts +4 -4
- package/src/core/generator/describe.ts +52 -0
- package/src/core/generator/endpoint-warnings.ts +10 -1
- package/src/core/generator/suite-generator.ts +31 -12
- package/src/core/parser/filter.ts +26 -0
- package/src/db/queries.ts +24 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kirrosh/zond",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "index.ts",
|
|
@@ -37,6 +37,9 @@
|
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@types/bun": "latest"
|
|
39
39
|
},
|
|
40
|
+
"engines": {
|
|
41
|
+
"bun": ">=1.1.0"
|
|
42
|
+
},
|
|
40
43
|
"peerDependencies": {
|
|
41
44
|
"typescript": "^5"
|
|
42
45
|
},
|
|
@@ -50,6 +50,10 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
|
|
|
50
50
|
const color = useColor();
|
|
51
51
|
|
|
52
52
|
// Enriched mode with run results
|
|
53
|
+
let passing = 0;
|
|
54
|
+
let apiError = 0;
|
|
55
|
+
let testFailed = 0;
|
|
56
|
+
|
|
53
57
|
if (options.runId != null) {
|
|
54
58
|
getDb();
|
|
55
59
|
const run = getRunById(options.runId);
|
|
@@ -88,62 +92,63 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
|
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
94
|
|
|
91
|
-
let passing = 0;
|
|
92
|
-
let apiError = 0;
|
|
93
|
-
let testFailed = 0;
|
|
94
95
|
for (const status of endpointStatus.values()) {
|
|
95
96
|
if (status === "passing") passing++;
|
|
96
97
|
else if (status === "api_error") apiError++;
|
|
97
98
|
else if (status === "test_failed") testFailed++;
|
|
98
99
|
}
|
|
100
|
+
}
|
|
99
101
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
+
if (!options.json) {
|
|
103
|
+
if (options.runId != null) {
|
|
104
|
+
console.log(`Coverage: ${coveredCount}/${allEndpoints.length} endpoints (${percentage}%) — Run #${options.runId}`);
|
|
105
|
+
console.log("");
|
|
102
106
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
107
|
+
if (passing > 0) {
|
|
108
|
+
console.log(` ${color ? GREEN : ""}✅ ${passing} covered and passing${color ? RESET : ""}`);
|
|
109
|
+
}
|
|
110
|
+
if (apiError > 0) {
|
|
111
|
+
console.log(` ${color ? YELLOW : ""}⚠️ ${apiError} covered but returning 5xx (possibly broken API)${color ? RESET : ""}`);
|
|
112
|
+
}
|
|
113
|
+
if (testFailed > 0) {
|
|
114
|
+
console.log(` ${color ? RED : ""}❌ ${testFailed} covered, test assertions failed${color ? RESET : ""}`);
|
|
115
|
+
}
|
|
116
|
+
if (uncovered.length > 0) {
|
|
117
|
+
console.log(` ${color ? DIM : ""}⬜ ${uncovered.length} not covered${color ? RESET : ""}`);
|
|
118
|
+
}
|
|
119
|
+
} else {
|
|
120
|
+
// Standard mode
|
|
121
|
+
console.log(`Coverage: ${coveredCount}/${allEndpoints.length} endpoints (${percentage}%)`);
|
|
122
|
+
console.log("");
|
|
119
123
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
124
|
+
// Covered endpoints
|
|
125
|
+
if (coveredCount > 0) {
|
|
126
|
+
console.log(`${color ? GREEN : ""}Covered:${color ? RESET : ""}`);
|
|
127
|
+
for (const ep of allEndpoints) {
|
|
128
|
+
if (!uncovered.includes(ep)) {
|
|
129
|
+
console.log(` ${color ? GREEN : ""}✓${color ? RESET : ""} ${ep.method.padEnd(7)} ${ep.path}`);
|
|
130
|
+
}
|
|
126
131
|
}
|
|
132
|
+
console.log("");
|
|
127
133
|
}
|
|
128
|
-
console.log("");
|
|
129
|
-
}
|
|
130
134
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
135
|
+
// Uncovered endpoints
|
|
136
|
+
if (uncovered.length > 0) {
|
|
137
|
+
console.log(`${color ? RED : ""}Uncovered:${color ? RESET : ""}`);
|
|
138
|
+
for (const ep of uncovered) {
|
|
139
|
+
console.log(` ${color ? RED : ""}✗${color ? RESET : ""} ${ep.method.padEnd(7)} ${ep.path}`);
|
|
140
|
+
}
|
|
136
141
|
}
|
|
137
142
|
}
|
|
138
|
-
}
|
|
139
143
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
144
|
+
// Static warnings (always shown in human-readable mode)
|
|
145
|
+
const warnings = analyzeEndpoints(allEndpoints);
|
|
146
|
+
if (warnings.length > 0) {
|
|
147
|
+
console.log("");
|
|
148
|
+
console.log(`${color ? YELLOW : ""}Spec warnings:${color ? RESET : ""}`);
|
|
149
|
+
for (const w of warnings) {
|
|
150
|
+
console.log(` ${color ? YELLOW : ""}⚠${color ? RESET : ""} ${w.method.padEnd(7)} ${w.path}: ${w.warnings.join(", ")}`);
|
|
151
|
+
}
|
|
147
152
|
}
|
|
148
153
|
}
|
|
149
154
|
|
package/src/cli/commands/db.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { getCollections, getRuns, getRunDetail, diagnoseRun, compareRuns } from "../../core/diagnostics/db-analysis.ts";
|
|
2
|
+
import { getFilteredResults } from "../../db/queries.ts";
|
|
3
|
+
import { getDb } from "../../db/schema.ts";
|
|
2
4
|
import { printError } from "../output.ts";
|
|
3
5
|
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
4
6
|
|
|
@@ -9,6 +11,8 @@ export interface DbOptions {
|
|
|
9
11
|
verbose?: boolean;
|
|
10
12
|
dbPath?: string;
|
|
11
13
|
json?: boolean;
|
|
14
|
+
method?: string;
|
|
15
|
+
status?: number;
|
|
12
16
|
}
|
|
13
17
|
|
|
14
18
|
export async function dbCommand(options: DbOptions): Promise<number> {
|
|
@@ -58,11 +62,22 @@ export async function dbCommand(options: DbOptions): Promise<number> {
|
|
|
58
62
|
else printError(msg);
|
|
59
63
|
return 2;
|
|
60
64
|
}
|
|
61
|
-
|
|
62
|
-
if (
|
|
63
|
-
|
|
65
|
+
// If filtering by method/status, show filtered results instead of full detail
|
|
66
|
+
if (options.method || options.status !== undefined) {
|
|
67
|
+
getDb(options.dbPath);
|
|
68
|
+
const results = getFilteredResults(id, { method: options.method, status: options.status });
|
|
69
|
+
if (json) {
|
|
70
|
+
printJson(jsonOk("db run", { run_id: id, count: results.length, results }));
|
|
71
|
+
} else {
|
|
72
|
+
console.log(JSON.stringify({ run_id: id, count: results.length, results }, null, 2));
|
|
73
|
+
}
|
|
64
74
|
} else {
|
|
65
|
-
|
|
75
|
+
const detail = getRunDetail(id, options.verbose, options.dbPath);
|
|
76
|
+
if (json) {
|
|
77
|
+
printJson(jsonOk("db run", detail));
|
|
78
|
+
} else {
|
|
79
|
+
console.log(JSON.stringify(detail, null, 2));
|
|
80
|
+
}
|
|
66
81
|
}
|
|
67
82
|
return 0;
|
|
68
83
|
}
|
|
@@ -75,7 +90,7 @@ export async function dbCommand(options: DbOptions): Promise<number> {
|
|
|
75
90
|
else printError(msg);
|
|
76
91
|
return 2;
|
|
77
92
|
}
|
|
78
|
-
const result = diagnoseRun(id, options.verbose, options.dbPath);
|
|
93
|
+
const result = diagnoseRun(id, options.verbose, options.dbPath, options.limit);
|
|
79
94
|
if (json) {
|
|
80
95
|
printJson(jsonOk("db diagnose", result));
|
|
81
96
|
} else {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describeEndpoint, describeCompact } from "../../core/generator/describe.ts";
|
|
1
|
+
import { describeEndpoint, describeCompact, describeAllParams } from "../../core/generator/describe.ts";
|
|
2
2
|
import { printError } from "../output.ts";
|
|
3
3
|
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
4
4
|
|
|
@@ -7,11 +7,36 @@ export interface DescribeOptions {
|
|
|
7
7
|
method?: string;
|
|
8
8
|
path?: string;
|
|
9
9
|
compact?: boolean;
|
|
10
|
+
listParams?: boolean;
|
|
10
11
|
json?: boolean;
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
export async function describeCommand(options: DescribeOptions): Promise<number> {
|
|
14
15
|
try {
|
|
16
|
+
if (options.listParams) {
|
|
17
|
+
const params = await describeAllParams(options.specPath);
|
|
18
|
+
if (options.json) {
|
|
19
|
+
printJson(jsonOk("describe", { params }));
|
|
20
|
+
} else {
|
|
21
|
+
const grouped = new Map<string, typeof params>();
|
|
22
|
+
for (const p of params) {
|
|
23
|
+
const arr = grouped.get(p.in) ?? [];
|
|
24
|
+
arr.push(p);
|
|
25
|
+
grouped.set(p.in, arr);
|
|
26
|
+
}
|
|
27
|
+
for (const [location, locationParams] of grouped) {
|
|
28
|
+
console.log(`\n${location.toUpperCase()} parameters:`);
|
|
29
|
+
for (const p of locationParams) {
|
|
30
|
+
const req = p.required ? " (required)" : "";
|
|
31
|
+
const type = p.type ? ` [${p.type}]` : "";
|
|
32
|
+
console.log(` ${p.name}${type}${req} — used in ${p.usedBy.length} endpoint(s)`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
console.log(`\n${params.length} unique parameter(s)`);
|
|
36
|
+
}
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
|
|
15
40
|
if (options.compact) {
|
|
16
41
|
const endpoints = await describeCompact(options.specPath);
|
|
17
42
|
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
filterUncoveredEndpoints,
|
|
9
9
|
serializeSuite,
|
|
10
10
|
} from "../../core/generator/index.ts";
|
|
11
|
-
import { generateSuites } from "../../core/generator/suite-generator.ts";
|
|
11
|
+
import { generateSuites, findUnresolvedVars } from "../../core/generator/suite-generator.ts";
|
|
12
12
|
import { filterByTag } from "../../core/generator/chunker.ts";
|
|
13
13
|
import { parse } from "../../core/parser/yaml-parser.ts";
|
|
14
14
|
import { decycleSchema } from "../../core/generator/schema-utils.ts";
|
|
@@ -104,12 +104,23 @@ export async function generateCommand(options: GenerateOptions): Promise<number>
|
|
|
104
104
|
// DB unavailable — not fatal
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
// Create .env.yaml with base_url
|
|
107
|
+
// Create .env.yaml with base_url and unresolved variables as placeholders
|
|
108
108
|
const envPath = join(options.output, ".env.yaml");
|
|
109
109
|
const envFile = Bun.file(envPath);
|
|
110
|
-
if (!(await envFile.exists())
|
|
111
|
-
|
|
112
|
-
|
|
110
|
+
if (!(await envFile.exists())) {
|
|
111
|
+
const unresolvedVars = new Set<string>();
|
|
112
|
+
for (const suite of suites) {
|
|
113
|
+
for (const v of findUnresolvedVars(suite)) unresolvedVars.add(v);
|
|
114
|
+
}
|
|
115
|
+
const lines: string[] = [];
|
|
116
|
+
if (baseUrl) lines.push(`base_url: ${baseUrl}`);
|
|
117
|
+
for (const v of [...unresolvedVars].sort()) {
|
|
118
|
+
lines.push(`${v}: "" # TODO: fill in`);
|
|
119
|
+
}
|
|
120
|
+
if (lines.length > 0) {
|
|
121
|
+
await Bun.write(envPath, lines.join("\n") + "\n");
|
|
122
|
+
warnings.push(`Created ${envPath} with ${unresolvedVars.size} placeholder variable(s)`);
|
|
123
|
+
}
|
|
113
124
|
}
|
|
114
125
|
|
|
115
126
|
// Validate generated files
|
package/src/cli/commands/run.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { dirname } from "path";
|
|
|
2
2
|
import { stat } from "node:fs/promises";
|
|
3
3
|
import { parse } from "../../core/parser/yaml-parser.ts";
|
|
4
4
|
import { loadEnvironment } from "../../core/parser/variables.ts";
|
|
5
|
-
import { filterSuitesByTags } from "../../core/parser/filter.ts";
|
|
5
|
+
import { filterSuitesByTags, excludeSuitesByTags, filterSuitesByMethod } from "../../core/parser/filter.ts";
|
|
6
6
|
import { runSuite } from "../../core/runner/executor.ts";
|
|
7
7
|
import { getReporter } from "../../core/reporter/index.ts";
|
|
8
8
|
import type { ReporterName } from "../../core/reporter/types.ts";
|
|
@@ -25,6 +25,8 @@ export interface RunOptions {
|
|
|
25
25
|
authToken?: string;
|
|
26
26
|
safe?: boolean;
|
|
27
27
|
tag?: string[];
|
|
28
|
+
excludeTag?: string[];
|
|
29
|
+
method?: string;
|
|
28
30
|
envVars?: string[];
|
|
29
31
|
dryRun?: boolean;
|
|
30
32
|
json?: boolean;
|
|
@@ -54,6 +56,24 @@ export async function runCommand(options: RunOptions): Promise<number> {
|
|
|
54
56
|
}
|
|
55
57
|
}
|
|
56
58
|
|
|
59
|
+
// 1b2. Exclude-tag filter
|
|
60
|
+
if (options.excludeTag && options.excludeTag.length > 0) {
|
|
61
|
+
suites = excludeSuitesByTags(suites, options.excludeTag);
|
|
62
|
+
if (suites.length === 0) {
|
|
63
|
+
printWarning("All suites excluded by --exclude-tag");
|
|
64
|
+
return 0;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// 1b3. Method filter
|
|
69
|
+
if (options.method) {
|
|
70
|
+
suites = filterSuitesByMethod(suites, options.method);
|
|
71
|
+
if (suites.length === 0) {
|
|
72
|
+
printWarning(`No tests found with method ${options.method.toUpperCase()}`);
|
|
73
|
+
return 0;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
57
77
|
// 1c. Safe mode: keep GET, set-only steps, and auth-related requests
|
|
58
78
|
if (options.safe) {
|
|
59
79
|
for (const suite of suites) {
|
|
@@ -39,13 +39,13 @@ async function fetchLatestRelease(): Promise<GitHubRelease> {
|
|
|
39
39
|
export async function updateCommand(options: UpdateOptions): Promise<number> {
|
|
40
40
|
try {
|
|
41
41
|
if (!isCompiledBinary()) {
|
|
42
|
-
const msg = "Self-update is only available for standalone binaries.
|
|
42
|
+
const msg = "Self-update is only available for standalone binaries. Install binary: curl -fsSL https://raw.githubusercontent.com/kirrosh/zond/master/install.sh | sh";
|
|
43
43
|
if (options.json) {
|
|
44
|
-
printJson(jsonOk("update", { action: "skip", reason: "not-standalone" }, [msg]));
|
|
44
|
+
printJson(jsonOk("update", { action: "skip", reason: "not-standalone", installHint: "curl -fsSL https://raw.githubusercontent.com/kirrosh/zond/master/install.sh | sh" }, [msg]));
|
|
45
45
|
} else {
|
|
46
46
|
printWarning(msg);
|
|
47
47
|
}
|
|
48
|
-
return
|
|
48
|
+
return 3;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
const target = getTarget();
|
|
@@ -141,16 +141,31 @@ export async function updateCommand(options: UpdateOptions): Promise<number> {
|
|
|
141
141
|
}
|
|
142
142
|
|
|
143
143
|
// Replace current binary
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
144
|
+
try {
|
|
145
|
+
if (process.platform === "win32") {
|
|
146
|
+
// Windows: rename current to .old, move new, clean up
|
|
147
|
+
const oldBinary = currentBinary + ".old";
|
|
148
|
+
try { await rm(oldBinary, { force: true }); } catch {}
|
|
149
|
+
await rename(currentBinary, oldBinary);
|
|
150
|
+
await rename(newBinary, currentBinary);
|
|
151
|
+
try { await rm(oldBinary, { force: true }); } catch {}
|
|
152
|
+
} else {
|
|
153
|
+
await rename(newBinary, currentBinary);
|
|
154
|
+
await chmod(currentBinary, 0o755);
|
|
155
|
+
}
|
|
156
|
+
} catch (replaceErr: any) {
|
|
157
|
+
if (replaceErr?.code === "EACCES" || replaceErr?.code === "EPERM") {
|
|
158
|
+
const hint = process.platform === "win32"
|
|
159
|
+
? `Permission denied. Run the terminal as Administrator.`
|
|
160
|
+
: `Permission denied writing to ${currentBinary}. Run: sudo zond update`;
|
|
161
|
+
if (options.json) {
|
|
162
|
+
printJson(jsonError("update", [hint]));
|
|
163
|
+
} else {
|
|
164
|
+
printError(hint);
|
|
165
|
+
}
|
|
166
|
+
return 2;
|
|
167
|
+
}
|
|
168
|
+
throw replaceErr;
|
|
154
169
|
}
|
|
155
170
|
|
|
156
171
|
if (options.json) {
|
package/src/cli/index.ts
CHANGED
|
@@ -123,6 +123,8 @@ Options for 'run':
|
|
|
123
123
|
--auth-token <token> Auth token injected as {{auth_token}} variable
|
|
124
124
|
--safe Run only GET tests (read-only, safe mode)
|
|
125
125
|
--tag <tag> Filter suites by tag (repeatable, comma-separated, OR logic)
|
|
126
|
+
--exclude-tag <tag> Exclude suites by tag (repeatable, comma-separated)
|
|
127
|
+
--method <method> Filter tests by HTTP method (e.g. GET, POST)
|
|
126
128
|
|
|
127
129
|
Options for 'init':
|
|
128
130
|
--name <name> API name (auto-detected from spec title if omitted)
|
|
@@ -132,14 +134,15 @@ Options for 'init':
|
|
|
132
134
|
|
|
133
135
|
Options for 'describe':
|
|
134
136
|
--compact List all endpoints briefly
|
|
137
|
+
--list-params List all unique parameters across all endpoints
|
|
135
138
|
--method <method> HTTP method for single endpoint detail
|
|
136
139
|
--path <path> Endpoint path for single endpoint detail
|
|
137
140
|
|
|
138
141
|
Options for 'db':
|
|
139
142
|
zond db collections List all API collections
|
|
140
143
|
zond db runs [--limit N] List recent test runs
|
|
141
|
-
zond db run <id> [--verbose] Show run details
|
|
142
|
-
zond db diagnose <id> Diagnose run failures
|
|
144
|
+
zond db run <id> [--verbose] Show run details (--method GET, --status 403 to filter)
|
|
145
|
+
zond db diagnose <id> Diagnose run failures (--limit N examples per group, --verbose for all)
|
|
143
146
|
zond db compare <idA> <idB> Compare two runs
|
|
144
147
|
|
|
145
148
|
Options for 'request':
|
|
@@ -256,8 +259,9 @@ async function main(): Promise<number> {
|
|
|
256
259
|
}
|
|
257
260
|
}
|
|
258
261
|
|
|
259
|
-
// Collect all --tag and --env-var flags (parseArgs only stores last one, so re-parse)
|
|
262
|
+
// Collect all --tag, --exclude-tag, and --env-var flags (parseArgs only stores last one, so re-parse)
|
|
260
263
|
const tagValues: string[] = [];
|
|
264
|
+
const excludeTagValues: string[] = [];
|
|
261
265
|
const envVarValues: string[] = [];
|
|
262
266
|
const rawRunArgs = process.argv.slice(2);
|
|
263
267
|
for (let i = 0; i < rawRunArgs.length; i++) {
|
|
@@ -267,6 +271,11 @@ async function main(): Promise<number> {
|
|
|
267
271
|
i++;
|
|
268
272
|
} else if (arg.startsWith("--tag=")) {
|
|
269
273
|
tagValues.push(arg.slice("--tag=".length));
|
|
274
|
+
} else if (arg === "--exclude-tag" && rawRunArgs[i + 1]) {
|
|
275
|
+
excludeTagValues.push(rawRunArgs[i + 1]!);
|
|
276
|
+
i++;
|
|
277
|
+
} else if (arg.startsWith("--exclude-tag=")) {
|
|
278
|
+
excludeTagValues.push(arg.slice("--exclude-tag=".length));
|
|
270
279
|
} else if (arg === "--env-var" && rawRunArgs[i + 1]) {
|
|
271
280
|
envVarValues.push(rawRunArgs[i + 1]!);
|
|
272
281
|
i++;
|
|
@@ -276,6 +285,7 @@ async function main(): Promise<number> {
|
|
|
276
285
|
}
|
|
277
286
|
// Support comma-separated: --tag smoke,crud → ["smoke", "crud"]
|
|
278
287
|
const tags = tagValues.flatMap(v => v.split(",")).filter(Boolean);
|
|
288
|
+
const excludeTags = excludeTagValues.flatMap(v => v.split(",")).filter(Boolean);
|
|
279
289
|
|
|
280
290
|
return runCommand({
|
|
281
291
|
path,
|
|
@@ -288,6 +298,8 @@ async function main(): Promise<number> {
|
|
|
288
298
|
authToken: typeof flags["auth-token"] === "string" ? flags["auth-token"] : undefined,
|
|
289
299
|
safe: flags["safe"] === true,
|
|
290
300
|
tag: tags.length > 0 ? tags : undefined,
|
|
301
|
+
excludeTag: excludeTags.length > 0 ? excludeTags : undefined,
|
|
302
|
+
method: typeof flags["method"] === "string" ? flags["method"] : undefined,
|
|
291
303
|
envVars: envVarValues.length > 0 ? envVarValues : undefined,
|
|
292
304
|
dryRun: flags["dry-run"] === true,
|
|
293
305
|
json: jsonFlag,
|
|
@@ -410,6 +422,7 @@ async function main(): Promise<number> {
|
|
|
410
422
|
return describeCommand({
|
|
411
423
|
specPath,
|
|
412
424
|
compact: flags["compact"] === true,
|
|
425
|
+
listParams: flags["list-params"] === true,
|
|
413
426
|
method: typeof flags["method"] === "string" ? flags["method"] : undefined,
|
|
414
427
|
path: typeof flags["path"] === "string" ? flags["path"] : undefined,
|
|
415
428
|
json: jsonFlag,
|
|
@@ -428,6 +441,13 @@ async function main(): Promise<number> {
|
|
|
428
441
|
limit = parseInt(limitRaw, 10);
|
|
429
442
|
if (isNaN(limit) || limit <= 0) limit = undefined;
|
|
430
443
|
}
|
|
444
|
+
const statusRaw = flags["status"];
|
|
445
|
+
let statusFilter: number | undefined;
|
|
446
|
+
if (typeof statusRaw === "string") {
|
|
447
|
+
statusFilter = parseInt(statusRaw, 10);
|
|
448
|
+
if (isNaN(statusFilter)) statusFilter = undefined;
|
|
449
|
+
}
|
|
450
|
+
|
|
431
451
|
return dbCommand({
|
|
432
452
|
subcommand: dbSub,
|
|
433
453
|
positional: positional.slice(1),
|
|
@@ -435,6 +455,8 @@ async function main(): Promise<number> {
|
|
|
435
455
|
verbose: flags["verbose"] === true,
|
|
436
456
|
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
437
457
|
json: jsonFlag,
|
|
458
|
+
method: typeof flags["method"] === "string" ? flags["method"] : undefined,
|
|
459
|
+
status: statusFilter,
|
|
438
460
|
});
|
|
439
461
|
}
|
|
440
462
|
|
|
@@ -169,7 +169,7 @@ export interface DiagnoseResult {
|
|
|
169
169
|
grouped_failures?: FailureGroup[];
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
-
export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string): DiagnoseResult {
|
|
172
|
+
export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, maxExamples?: number): DiagnoseResult {
|
|
173
173
|
getDb(dbPath);
|
|
174
174
|
const diagRun = getRunById(runId);
|
|
175
175
|
if (!diagRun) throw new Error(`Run ${runId} not found`);
|
|
@@ -272,7 +272,7 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
|
|
|
272
272
|
|
|
273
273
|
const { grouped_failures, compactFailures } = verbose
|
|
274
274
|
? { grouped_failures: undefined, compactFailures: failures }
|
|
275
|
-
: groupFailures(failures);
|
|
275
|
+
: groupFailures(failures, maxExamples);
|
|
276
276
|
|
|
277
277
|
return {
|
|
278
278
|
run: {
|
|
@@ -301,7 +301,7 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
|
|
|
301
301
|
type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; hint?: string; response_status: number | null };
|
|
302
302
|
|
|
303
303
|
/** Group similar failures for compact output. Exported for testing. */
|
|
304
|
-
export function groupFailures<T extends FailureItem>(failures: T[]): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
|
|
304
|
+
export function groupFailures<T extends FailureItem>(failures: T[], maxExamples = 2): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
|
|
305
305
|
if (failures.length <= 5) {
|
|
306
306
|
return { compactFailures: failures };
|
|
307
307
|
}
|
|
@@ -341,7 +341,7 @@ export function groupFailures<T extends FailureItem>(failures: T[]): { grouped_f
|
|
|
341
341
|
failure_type: group.failure_type,
|
|
342
342
|
recommended_action: group.items[0]!.recommended_action,
|
|
343
343
|
hint: group.hint,
|
|
344
|
-
examples: group.items.slice(0,
|
|
344
|
+
examples: (maxExamples === 0 ? group.items : group.items.slice(0, maxExamples)).map(f => `${f.suite_name}/${f.test_name}`),
|
|
345
345
|
response_status: group.response_status,
|
|
346
346
|
});
|
|
347
347
|
compactFailures.push(group.items[0]!);
|
|
@@ -248,3 +248,55 @@ export async function describeCompact(
|
|
|
248
248
|
|
|
249
249
|
return result;
|
|
250
250
|
}
|
|
251
|
+
|
|
252
|
+
export interface ParamInfo {
|
|
253
|
+
name: string;
|
|
254
|
+
in: string;
|
|
255
|
+
type?: string;
|
|
256
|
+
required: boolean;
|
|
257
|
+
usedBy: string[];
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export async function describeAllParams(
|
|
261
|
+
specPath: string,
|
|
262
|
+
options?: { insecure?: boolean },
|
|
263
|
+
): Promise<ParamInfo[]> {
|
|
264
|
+
const doc = await readOpenApiSpec(specPath, options) as OpenAPIV3.Document;
|
|
265
|
+
const paths = doc.paths ?? {};
|
|
266
|
+
const paramMap = new Map<string, ParamInfo>();
|
|
267
|
+
|
|
268
|
+
for (const [path, pathItem] of Object.entries(paths)) {
|
|
269
|
+
const pathParams = ((pathItem as any)?.parameters ?? []) as OpenAPIV3.ParameterObject[];
|
|
270
|
+
|
|
271
|
+
for (const method of ["get", "post", "put", "patch", "delete"]) {
|
|
272
|
+
const op = (pathItem as any)?.[method] as OpenAPIV3.OperationObject | undefined;
|
|
273
|
+
if (!op) continue;
|
|
274
|
+
|
|
275
|
+
const allParams = [...pathParams, ...((op.parameters ?? []) as OpenAPIV3.ParameterObject[])];
|
|
276
|
+
const endpoint = `${method.toUpperCase()} ${path}`;
|
|
277
|
+
|
|
278
|
+
for (const p of allParams) {
|
|
279
|
+
const key = `${p.in}:${p.name}`;
|
|
280
|
+
const existing = paramMap.get(key);
|
|
281
|
+
const schema = p.schema as OpenAPIV3.SchemaObject | undefined;
|
|
282
|
+
if (existing) {
|
|
283
|
+
existing.usedBy.push(endpoint);
|
|
284
|
+
if (p.required) existing.required = true;
|
|
285
|
+
} else {
|
|
286
|
+
paramMap.set(key, {
|
|
287
|
+
name: p.name,
|
|
288
|
+
in: p.in,
|
|
289
|
+
type: schema?.type,
|
|
290
|
+
required: p.required ?? false,
|
|
291
|
+
usedBy: [endpoint],
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
return [...paramMap.values()].sort((a, b) => {
|
|
299
|
+
if (a.in !== b.in) return a.in.localeCompare(b.in);
|
|
300
|
+
return a.name.localeCompare(b.name);
|
|
301
|
+
});
|
|
302
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EndpointInfo } from "./types.ts";
|
|
2
2
|
|
|
3
|
-
export type WarningCode = "deprecated" | "no_response_schema" | "no_responses_defined" | "required_params_no_examples";
|
|
3
|
+
export type WarningCode = "deprecated" | "no_response_schema" | "no_responses_defined" | "required_params_no_examples" | "post_body_as_query";
|
|
4
4
|
|
|
5
5
|
export interface EndpointWarning {
|
|
6
6
|
method: string;
|
|
@@ -34,6 +34,15 @@ export function analyzeEndpoints(endpoints: EndpointInfo[]): EndpointWarning[] {
|
|
|
34
34
|
warnings.push(`required_params_no_examples: ${missingExamples.join(", ")}`);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
// SpringDoc quirk: POST/PUT/PATCH with query param named "body" or single complex object query param
|
|
38
|
+
if (["POST", "PUT", "PATCH"].includes(ep.method)) {
|
|
39
|
+
const queryParams = ep.parameters.filter(p => p.in === "query");
|
|
40
|
+
const hasBodyQuery = queryParams.some(p => p.name.toLowerCase() === "body");
|
|
41
|
+
if (hasBodyQuery) {
|
|
42
|
+
warnings.push("post_body_as_query: query param 'body' on POST/PUT/PATCH likely means request body (SpringDoc quirk)");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
37
46
|
if (warnings.length > 0) {
|
|
38
47
|
result.push({ method: ep.method, path: ep.path, warnings });
|
|
39
48
|
}
|
|
@@ -15,8 +15,8 @@ function convertPath(path: string): string {
|
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Convert path params to seed values for smoke suites (no capture context).
|
|
18
|
-
* Uses the parameter's example/default from the spec, or falls back to
|
|
19
|
-
*
|
|
18
|
+
* Uses the parameter's example/default from the spec, or falls back to
|
|
19
|
+
* {{placeholder}} form so the user fills them in via .env.yaml.
|
|
20
20
|
*/
|
|
21
21
|
function convertPathWithSeeds(path: string, ep: EndpointInfo): string {
|
|
22
22
|
return path.replace(/\{([^}]+)\}/g, (_, name: string) => {
|
|
@@ -24,7 +24,6 @@ function convertPathWithSeeds(path: string, ep: EndpointInfo): string {
|
|
|
24
24
|
const schema = param?.schema as OpenAPIV3.SchemaObject | undefined;
|
|
25
25
|
const example = (param as any)?.example ?? schema?.example ?? schema?.default;
|
|
26
26
|
if (example !== undefined) return String(example);
|
|
27
|
-
if (schema?.type === "integer" || /^id$|_id$|Id$/i.test(name)) return "1";
|
|
28
27
|
return `{{${name}}}`;
|
|
29
28
|
});
|
|
30
29
|
}
|
|
@@ -81,6 +80,20 @@ function getBodyAssertions(ep: EndpointInfo): Record<string, Record<string, stri
|
|
|
81
80
|
return undefined;
|
|
82
81
|
}
|
|
83
82
|
|
|
83
|
+
/** Derive a variable name for a security scheme's token */
|
|
84
|
+
function schemeVarName(scheme: SecuritySchemeInfo, allSchemes: SecuritySchemeInfo[]): string {
|
|
85
|
+
// Count how many bearer-like schemes exist
|
|
86
|
+
const bearerSchemes = allSchemes.filter(s =>
|
|
87
|
+
(s.type === "http" && (s.scheme === "bearer" || !s.scheme)) ||
|
|
88
|
+
(s.type === "apiKey" && s.in === "header" && s.apiKeyName === "Authorization")
|
|
89
|
+
);
|
|
90
|
+
// If only one bearer scheme → keep generic auth_token for backward compat
|
|
91
|
+
if (bearerSchemes.length <= 1) return "auth_token";
|
|
92
|
+
// Multiple → derive from scheme name (e.g. "platformAuth" → "platform_auth_token")
|
|
93
|
+
const slug = scheme.name.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/_token$|_auth$/, "");
|
|
94
|
+
return `${slug}_token`;
|
|
95
|
+
}
|
|
96
|
+
|
|
84
97
|
function getAuthHeaders(
|
|
85
98
|
ep: EndpointInfo,
|
|
86
99
|
schemes: SecuritySchemeInfo[],
|
|
@@ -93,16 +106,15 @@ function getAuthHeaders(
|
|
|
93
106
|
|
|
94
107
|
if (scheme.type === "http") {
|
|
95
108
|
if (scheme.scheme === "bearer" || !scheme.scheme) {
|
|
96
|
-
return { Authorization:
|
|
109
|
+
return { Authorization: `Bearer {{${schemeVarName(scheme, schemes)}}}` };
|
|
97
110
|
}
|
|
98
111
|
if (scheme.scheme === "basic") {
|
|
99
|
-
return { Authorization:
|
|
112
|
+
return { Authorization: `Basic {{${schemeVarName(scheme, schemes)}}}` };
|
|
100
113
|
}
|
|
101
114
|
}
|
|
102
115
|
if (scheme.type === "apiKey" && scheme.in === "header" && scheme.apiKeyName) {
|
|
103
|
-
// When apiKey scheme uses Authorization header, it's typically a Bearer token
|
|
104
116
|
if (scheme.apiKeyName === "Authorization") {
|
|
105
|
-
return { Authorization:
|
|
117
|
+
return { Authorization: `Bearer {{${schemeVarName(scheme, schemes)}}}` };
|
|
106
118
|
}
|
|
107
119
|
return { [scheme.apiKeyName]: "{{api_key}}" };
|
|
108
120
|
}
|
|
@@ -111,14 +123,15 @@ function getAuthHeaders(
|
|
|
111
123
|
return undefined;
|
|
112
124
|
}
|
|
113
125
|
|
|
114
|
-
function getRequiredQueryParams(ep: EndpointInfo): Record<string,
|
|
126
|
+
function getRequiredQueryParams(ep: EndpointInfo): Record<string, string> | undefined {
|
|
115
127
|
const queryParams = ep.parameters.filter(p => p.in === "query" && p.required);
|
|
116
128
|
if (queryParams.length === 0) return undefined;
|
|
117
129
|
|
|
118
|
-
const query: Record<string,
|
|
130
|
+
const query: Record<string, string> = {};
|
|
119
131
|
for (const p of queryParams) {
|
|
120
132
|
if (p.schema) {
|
|
121
|
-
|
|
133
|
+
const val = generateFromSchema(p.schema as OpenAPIV3.SchemaObject, p.name);
|
|
134
|
+
query[p.name] = typeof val === "object" ? JSON.stringify(val) : String(val);
|
|
122
135
|
} else {
|
|
123
136
|
query[p.name] = "{{$randomString}}";
|
|
124
137
|
}
|
|
@@ -387,9 +400,10 @@ export function generateCrudSuite(
|
|
|
387
400
|
}
|
|
388
401
|
|
|
389
402
|
/** Find unresolved template variables in a suite (excluding known globals, captured vars, and env keys) */
|
|
390
|
-
export function findUnresolvedVars(suite: RawSuite, envKeys?: Set<string>): string[] {
|
|
403
|
+
export function findUnresolvedVars(suite: RawSuite, envKeys?: Set<string>, extraKnown?: Set<string>): string[] {
|
|
391
404
|
const KNOWN = new Set(["base_url", "auth_token", "api_key"]);
|
|
392
405
|
if (envKeys) for (const k of envKeys) KNOWN.add(k);
|
|
406
|
+
if (extraKnown) for (const k of extraKnown) KNOWN.add(k);
|
|
393
407
|
const captured = new Set<string>();
|
|
394
408
|
for (const step of suite.tests) {
|
|
395
409
|
if (step.expect?.body) {
|
|
@@ -518,8 +532,13 @@ function generateConsistentAuthSuite(
|
|
|
518
532
|
/token|access_token|accessToken|jwt/i.test(k)
|
|
519
533
|
);
|
|
520
534
|
if (tokenField) {
|
|
535
|
+
// Determine the capture variable name based on the login endpoint's security scheme
|
|
536
|
+
const loginScheme = loginEp.security.length > 0
|
|
537
|
+
? securitySchemes.find(s => s.name === loginEp.security[0])
|
|
538
|
+
: undefined;
|
|
539
|
+
const captureVar = loginScheme ? schemeVarName(loginScheme, securitySchemes) : "auth_token";
|
|
521
540
|
if (!loginStep.expect.body) loginStep.expect.body = {};
|
|
522
|
-
loginStep.expect.body[tokenField] = { capture:
|
|
541
|
+
loginStep.expect.body[tokenField] = { capture: captureVar };
|
|
523
542
|
}
|
|
524
543
|
}
|
|
525
544
|
tests.push(loginStep);
|
|
@@ -12,3 +12,29 @@ export function filterSuitesByTags(suites: TestSuite[], tags: string[]): TestSui
|
|
|
12
12
|
return suite.tags.some(t => normalizedTags.includes(t.toLowerCase()));
|
|
13
13
|
});
|
|
14
14
|
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Exclude suites whose tags intersect with the exclusion set (OR logic, case-insensitive).
|
|
18
|
+
* Suites without tags are kept.
|
|
19
|
+
*/
|
|
20
|
+
export function excludeSuitesByTags(suites: TestSuite[], excludeTags: string[]): TestSuite[] {
|
|
21
|
+
if (excludeTags.length === 0) return suites;
|
|
22
|
+
const normalizedTags = excludeTags.map(t => t.toLowerCase());
|
|
23
|
+
return suites.filter(suite => {
|
|
24
|
+
if (!suite.tags || suite.tags.length === 0) return true;
|
|
25
|
+
return !suite.tags.some(t => normalizedTags.includes(t.toLowerCase()));
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Filter test steps within suites by HTTP method (case-insensitive).
|
|
31
|
+
* Suites with no remaining tests are removed.
|
|
32
|
+
*/
|
|
33
|
+
export function filterSuitesByMethod(suites: TestSuite[], method: string): TestSuite[] {
|
|
34
|
+
const upperMethod = method.toUpperCase();
|
|
35
|
+
const filtered = suites.map(suite => ({
|
|
36
|
+
...suite,
|
|
37
|
+
tests: suite.tests.filter(t => (t.method ?? "GET").toUpperCase() === upperMethod),
|
|
38
|
+
}));
|
|
39
|
+
return filtered.filter(s => s.tests.length > 0);
|
|
40
|
+
}
|
package/src/db/queries.ts
CHANGED
|
@@ -292,6 +292,30 @@ export function getResultsByRunId(runId: number): StoredStepResult[] {
|
|
|
292
292
|
}));
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
+
export function getFilteredResults(runId: number, filters: { method?: string; status?: number }): StoredStepResult[] {
|
|
296
|
+
const db = getDb();
|
|
297
|
+
const conditions = ["run_id = ?"];
|
|
298
|
+
const params: (string | number)[] = [runId];
|
|
299
|
+
|
|
300
|
+
if (filters.method) {
|
|
301
|
+
conditions.push("request_method = ?");
|
|
302
|
+
params.push(filters.method.toUpperCase());
|
|
303
|
+
}
|
|
304
|
+
if (filters.status !== undefined) {
|
|
305
|
+
conditions.push("response_status = ?");
|
|
306
|
+
params.push(filters.status);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const rows = db.query(`SELECT * FROM results WHERE ${conditions.join(" AND ")} ORDER BY id`).all(...params) as Array<
|
|
310
|
+
Omit<StoredStepResult, "assertions" | "captures"> & { assertions: string | null; captures: string | null }
|
|
311
|
+
>;
|
|
312
|
+
return rows.map((row) => ({
|
|
313
|
+
...row,
|
|
314
|
+
assertions: row.assertions ? JSON.parse(row.assertions) : [],
|
|
315
|
+
captures: row.captures ? JSON.parse(row.captures) : {},
|
|
316
|
+
}));
|
|
317
|
+
}
|
|
318
|
+
|
|
295
319
|
// ──────────────────────────────────────────────
|
|
296
320
|
// Dashboard metrics
|
|
297
321
|
// ──────────────────────────────────────────────
|