@kirrosh/zond 0.23.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +176 -1
- package/README.md +8 -7
- package/package.json +2 -3
- package/src/CLAUDE.md +112 -0
- package/src/cli/commands/add-api.ts +19 -7
- package/src/cli/commands/api/annotate/index.ts +359 -4
- package/src/cli/commands/api/annotate/lifecycle.ts +1 -1
- package/src/cli/commands/api/annotate/overlay.ts +1 -1
- package/src/cli/commands/api/annotate/pagination.ts +10 -6
- package/src/cli/commands/api/annotate/prompts.ts +39 -2
- package/src/cli/commands/audit.ts +360 -54
- package/src/cli/commands/check.ts +15 -2
- package/src/cli/commands/checks.ts +352 -36
- package/src/cli/commands/cleanup.ts +4 -30
- package/src/cli/commands/coverage.ts +275 -57
- package/src/cli/commands/db.ts +311 -8
- package/src/cli/commands/discover.ts +281 -161
- package/src/cli/commands/doctor.ts +57 -3
- package/src/cli/commands/fixtures.ts +1 -1
- package/src/cli/commands/generate.ts +24 -7
- package/src/cli/commands/init/bootstrap.ts +4 -1
- package/src/cli/commands/init/index.ts +2 -2
- package/src/cli/commands/init/skills.ts +43 -0
- package/src/cli/commands/init/templates/agents.md +12 -3
- package/src/cli/commands/init/templates/skills/warm-up-target.md +122 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +268 -44
- package/src/cli/commands/init/templates/skills/zond-seed.md +114 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +88 -26
- package/src/cli/commands/init/templates/skills/zond.md +274 -64
- package/src/cli/commands/init/templates/zond-config.yml +1 -1
- package/src/cli/commands/prepare-fixtures.ts +14 -52
- package/src/cli/commands/probe/_seed-bodies.ts +52 -0
- package/src/cli/commands/probe/mass-assignment.ts +101 -10
- package/src/cli/commands/probe/security.ts +95 -12
- package/src/cli/commands/probe/webhooks.ts +2 -0
- package/src/cli/commands/probe.ts +87 -11
- package/src/cli/commands/refresh-api.ts +59 -1
- package/src/cli/commands/report-bundle.ts +3 -11
- package/src/cli/commands/request.ts +116 -0
- package/src/cli/commands/run.ts +53 -5
- package/src/cli/commands/schema-from-runs.ts +128 -0
- package/src/cli/commands/secrets.ts +133 -0
- package/src/cli/json-envelope.ts +0 -20
- package/src/cli/json-schemas.ts +51 -0
- package/src/cli/output.ts +17 -1
- package/src/cli/program.ts +5 -4
- package/src/cli/safe-live.ts +24 -0
- package/src/cli/status-filter.ts +0 -10
- package/src/core/audit/persist.ts +183 -0
- package/src/core/checks/budget.ts +59 -0
- package/src/core/checks/checks/cross_call_references.ts +17 -4
- package/src/core/checks/checks/cursor_boundary_fuzzing.ts +219 -0
- package/src/core/checks/checks/idempotency_replay.ts +1 -5
- package/src/core/checks/checks/ignored_auth.ts +44 -1
- package/src/core/checks/checks/index.ts +3 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +169 -26
- package/src/core/checks/checks/negative_data_rejection.ts +119 -16
- package/src/core/checks/checks/not_a_server_error.ts +8 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +47 -18
- package/src/core/checks/checks/pagination_invariants.ts +298 -117
- package/src/core/checks/checks/positive_data_acceptance.ts +1 -4
- package/src/core/checks/checks/status_code_conformance.ts +78 -7
- package/src/core/checks/mode.ts +3 -0
- package/src/core/checks/recommended-action.ts +5 -1
- package/src/core/checks/runner.ts +614 -27
- package/src/core/checks/spec-findings.ts +308 -0
- package/src/core/checks/types.ts +117 -1
- package/src/core/checks/zond-extensions.ts +73 -0
- package/src/core/classifier/recommended-action.ts +35 -6
- package/src/core/coverage/loader.ts +31 -0
- package/src/core/diagnostics/db-analysis.ts +200 -106
- package/src/core/diagnostics/failure-class.ts +21 -1
- package/src/core/diagnostics/failure-hints.ts +4 -208
- package/src/core/diagnostics/suggested-fixes.ts +2 -3
- package/src/core/generator/chunker.ts +1 -8
- package/src/core/generator/data-factory.ts +199 -61
- package/src/core/generator/fixtures-builder.ts +38 -31
- package/src/core/generator/index.ts +0 -2
- package/src/core/generator/openapi-reader.ts +98 -4
- package/src/core/generator/path-param-disambig.ts +30 -4
- package/src/core/generator/resources-builder.ts +276 -26
- package/src/core/generator/schema-utils.ts +22 -0
- package/src/core/generator/suite-generator.ts +168 -15
- package/src/core/generator/types.ts +6 -0
- package/src/core/identity/identity-file.ts +0 -0
- package/src/core/output/README.md +11 -29
- package/src/core/output/index.ts +1 -1
- package/src/core/output/run.ts +0 -35
- package/src/core/output/types.ts +0 -7
- package/src/core/parser/dynamic-values.ts +160 -0
- package/src/core/parser/variables.ts +0 -0
- package/src/core/probe/dry-run-envelope.ts +4 -0
- package/src/core/probe/mass-assignment/classify.ts +175 -0
- package/src/core/probe/mass-assignment/cleanup.ts +52 -0
- package/src/core/probe/mass-assignment/digest.ts +114 -0
- package/src/core/probe/mass-assignment/orchestrator.ts +459 -0
- package/src/core/probe/mass-assignment/regression.ts +141 -0
- package/src/core/probe/mass-assignment/suspects.ts +92 -0
- package/src/core/probe/mass-assignment/types.ts +135 -0
- package/src/core/probe/mass-assignment-probe.ts +23 -1118
- package/src/core/probe/mass-assignment-template.ts +32 -4
- package/src/core/probe/path-discovery.ts +3 -4
- package/src/core/probe/probe-harness.ts +21 -22
- package/src/core/probe/security/baseline.ts +174 -0
- package/src/core/probe/security/classify.ts +341 -0
- package/src/core/probe/security/cleanup.ts +125 -0
- package/src/core/probe/security/detectors.ts +71 -0
- package/src/core/probe/security/digest.ts +104 -0
- package/src/core/probe/security/orchestrator.ts +398 -0
- package/src/core/probe/security/regression.ts +103 -0
- package/src/core/probe/security/types.ts +151 -0
- package/src/core/probe/security-probe-class.ts +8 -2
- package/src/core/probe/security-probe.ts +28 -1449
- package/src/core/probe/shared.ts +26 -0
- package/src/core/probe/webhooks-probe.ts +5 -7
- package/src/core/runner/assertions.ts +1 -1
- package/src/core/runner/executor.ts +3 -18
- package/src/core/runner/form-encode.ts +8 -18
- package/src/core/runner/http-client.ts +38 -1
- package/src/core/runner/preflight-vars.ts +19 -15
- package/src/core/runner/rate-limiter.ts +11 -29
- package/src/core/runner/run-kind.ts +7 -1
- package/src/core/runner/schema-validator.ts +2 -6
- package/src/core/runner/send-request.ts +11 -6
- package/src/core/runner/types.ts +6 -0
- package/src/core/setup-api.ts +53 -15
- package/src/core/severity/index.ts +0 -63
- package/src/core/spec/infer-schema.ts +102 -0
- package/src/core/spec/merge-specs.ts +156 -0
- package/src/core/spec/schema-from-runs.ts +117 -0
- package/src/core/spec/schema-overlay.ts +130 -0
- package/src/core/util/ajv.ts +13 -0
- package/src/core/util/headers.ts +9 -0
- package/src/core/util/url.ts +24 -0
- package/src/core/workspace/fixture-gap-report.ts +84 -0
- package/src/core/workspace/fixture-gaps.ts +71 -0
- package/src/core/workspace/root.ts +13 -11
- package/src/db/migrate.ts +2 -0
- package/src/db/migrations/0002_run_kind_request.sql +59 -0
- package/src/db/queries/collections.ts +2 -2
- package/src/db/queries/results.ts +88 -0
- package/src/db/queries/runs.ts +56 -2
- package/src/db/queries.ts +3 -0
- package/src/db/schema.ts +7 -7
- package/src/cli/commands/bootstrap.ts +0 -710
- package/src/core/anti-fp/bootstrap.ts +0 -34
- package/src/core/anti-fp/index.ts +0 -33
- package/src/core/anti-fp/registry.ts +0 -44
- package/src/core/anti-fp/rules/baseline-echo.ts +0 -74
- package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +0 -52
- package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +0 -38
- package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +0 -35
- package/src/core/anti-fp/rules/schemathesis/index.ts +0 -24
- package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +0 -53
- package/src/core/anti-fp/rules/subscription-gated/index.ts +0 -11
- package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +0 -75
- package/src/core/anti-fp/types.ts +0 -68
- package/src/core/generator/create-body.ts +0 -89
|
@@ -22,7 +22,15 @@ import {
|
|
|
22
22
|
SPEC_SNAPSHOT_FILENAME,
|
|
23
23
|
} from "../../core/setup-api.ts";
|
|
24
24
|
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
25
|
-
import { printError, printSuccess } from "../output.ts";
|
|
25
|
+
import { printError, printSuccess, printWarning } from "../output.ts";
|
|
26
|
+
import { readFileSync as readFileSyncNode } from "node:fs";
|
|
27
|
+
import {
|
|
28
|
+
loadSchemaOverlay,
|
|
29
|
+
mergePatch,
|
|
30
|
+
saveSchemaOverlay,
|
|
31
|
+
applySchemaOverlay,
|
|
32
|
+
type ResponseSchemaPatch,
|
|
33
|
+
} from "../../core/spec/schema-overlay.ts";
|
|
26
34
|
|
|
27
35
|
export interface RefreshApiOptions {
|
|
28
36
|
api: string;
|
|
@@ -32,6 +40,11 @@ export interface RefreshApiOptions {
|
|
|
32
40
|
insecure?: boolean;
|
|
33
41
|
json?: boolean;
|
|
34
42
|
dbPath?: string;
|
|
43
|
+
/** ARV-176: path to a patch.schema.json (from `schema-from-runs`) to fold
|
|
44
|
+
* into the response-schema overlay before rebuilding spec.json. */
|
|
45
|
+
mergeSchema?: string;
|
|
46
|
+
/** ARV-176: overwrite response schemas already declared upstream. */
|
|
47
|
+
force?: boolean;
|
|
35
48
|
}
|
|
36
49
|
|
|
37
50
|
export async function refreshApiCommand(opts: RefreshApiOptions): Promise<number> {
|
|
@@ -92,6 +105,40 @@ export async function refreshApiCommand(opts: RefreshApiOptions): Promise<number
|
|
|
92
105
|
// 3. Determine baseUrl from the doc
|
|
93
106
|
const baseUrl = ((doc as any).servers?.[0]?.url as string | undefined) ?? "";
|
|
94
107
|
|
|
108
|
+
// ARV-176: response-schema overlay. Fold a --merge-schema patch into the
|
|
109
|
+
// persistent overlay first, then apply the (possibly-updated) overlay onto
|
|
110
|
+
// the freshly-pulled doc *before* it's serialised to spec.json. Applying on
|
|
111
|
+
// every refresh (not only when --merge-schema is passed) is what makes the
|
|
112
|
+
// overlay survive an upstream re-pull (AC#2).
|
|
113
|
+
let schemaReport: {
|
|
114
|
+
merged_from?: string;
|
|
115
|
+
applied: string[];
|
|
116
|
+
preserved: string[];
|
|
117
|
+
conflicts: string[];
|
|
118
|
+
} | undefined;
|
|
119
|
+
if (opts.mergeSchema) {
|
|
120
|
+
let incoming: ResponseSchemaPatch;
|
|
121
|
+
try {
|
|
122
|
+
incoming = JSON.parse(readFileSyncNode(opts.mergeSchema, "utf-8")) as ResponseSchemaPatch;
|
|
123
|
+
} catch (err) {
|
|
124
|
+
const m = `Failed to read --merge-schema "${opts.mergeSchema}": ${(err as Error).message}`;
|
|
125
|
+
if (opts.json) printJson(jsonError("refresh-api", [m])); else printError(m);
|
|
126
|
+
return 2;
|
|
127
|
+
}
|
|
128
|
+
const merged = mergePatch(loadSchemaOverlay(baseDir), incoming);
|
|
129
|
+
saveSchemaOverlay(baseDir, merged);
|
|
130
|
+
}
|
|
131
|
+
const overlay = loadSchemaOverlay(baseDir);
|
|
132
|
+
if (overlay) {
|
|
133
|
+
const r = applySchemaOverlay(doc, overlay, { force: opts.force === true });
|
|
134
|
+
schemaReport = {
|
|
135
|
+
...(opts.mergeSchema ? { merged_from: opts.mergeSchema } : {}),
|
|
136
|
+
applied: r.applied,
|
|
137
|
+
preserved: r.preserved,
|
|
138
|
+
conflicts: r.conflicts,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
95
142
|
// 4. Write spec.json + 3 artifacts
|
|
96
143
|
writeArtifactsFromDoc({
|
|
97
144
|
doc,
|
|
@@ -117,6 +164,7 @@ export async function refreshApiCommand(opts: RefreshApiOptions): Promise<number
|
|
|
117
164
|
pulledFrom: usedExternal ? specSource : null,
|
|
118
165
|
endpointCount,
|
|
119
166
|
artifacts: [".api-catalog.yaml", ".api-resources.yaml", ".api-fixtures.yaml"],
|
|
167
|
+
...(schemaReport ? { schema_overlay: schemaReport } : {}),
|
|
120
168
|
};
|
|
121
169
|
|
|
122
170
|
if (opts.json) {
|
|
@@ -125,6 +173,12 @@ export async function refreshApiCommand(opts: RefreshApiOptions): Promise<number
|
|
|
125
173
|
printSuccess(`Refreshed '${collection.name}' (${endpointCount} endpoints)${usedExternal ? ` from ${specSource}` : ""}`);
|
|
126
174
|
process.stdout.write(` spec: ${localSpec}\n`);
|
|
127
175
|
process.stdout.write(` artifacts: ${result.artifacts.join(", ")}\n`);
|
|
176
|
+
if (schemaReport) {
|
|
177
|
+
process.stdout.write(` schema overlay: ${schemaReport.applied.length} applied, ${schemaReport.preserved.length} preserved, ${schemaReport.conflicts.length} conflict(s)\n`);
|
|
178
|
+
for (const c of schemaReport.conflicts) {
|
|
179
|
+
printWarning(`schema overlay conflict (endpoint not in upstream, skipped): ${c}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
128
182
|
process.stdout.write(` Run \`zond doctor --api ${collection.name}\` to verify fixtures.\n`);
|
|
129
183
|
}
|
|
130
184
|
return 0;
|
|
@@ -156,12 +210,16 @@ export function registerRefreshApi(program: Command): void {
|
|
|
156
210
|
.description("Re-snapshot the OpenAPI spec into apis/<name>/spec.json and regenerate the 3 artifacts (catalog/resources/fixtures)")
|
|
157
211
|
.option("--spec <path>", "Pull fresh from this path or URL (overrides registered source)")
|
|
158
212
|
.option("--insecure", "Allow self-signed TLS when --spec is an https URL")
|
|
213
|
+
.option("--merge-schema <path>", "ARV-176: fold a patch.schema.json (from `zond schema-from-runs`) into the persistent response-schema overlay (.api-schema.local.yaml) and apply it to spec.json. The overlay survives future refreshes.")
|
|
214
|
+
.option("--force", "ARV-176: overwrite response schemas already declared upstream (default: only fill gaps).")
|
|
159
215
|
.option("--db <path>", "Path to SQLite database file")
|
|
160
216
|
.action(async (name: string, opts, cmd: Command) => {
|
|
161
217
|
process.exitCode = await refreshApiCommand({
|
|
162
218
|
api: name,
|
|
163
219
|
spec: opts.spec,
|
|
164
220
|
insecure: opts.insecure === true,
|
|
221
|
+
mergeSchema: typeof opts.mergeSchema === "string" ? opts.mergeSchema : undefined,
|
|
222
|
+
force: opts.force === true,
|
|
165
223
|
dbPath: typeof opts.db === "string" ? opts.db : undefined,
|
|
166
224
|
json: globalJson(cmd),
|
|
167
225
|
});
|
|
@@ -49,7 +49,6 @@ interface BundleEntry {
|
|
|
49
49
|
caseStudy?: string;
|
|
50
50
|
htmlReport?: string;
|
|
51
51
|
diagnose?: string;
|
|
52
|
-
agentDirective?: string | null;
|
|
53
52
|
}
|
|
54
53
|
|
|
55
54
|
export function parseBundleRange(input: string): number[] | { error: string } {
|
|
@@ -186,7 +185,6 @@ export async function reportBundleCommand(options: ReportBundleOptions): Promise
|
|
|
186
185
|
const file = join(runDir, "diagnose.json");
|
|
187
186
|
await Bun.write(file, JSON.stringify(diag, null, 2));
|
|
188
187
|
entry.diagnose = file;
|
|
189
|
-
entry.agentDirective = (diag as unknown as { agent_directive?: string }).agent_directive ?? null;
|
|
190
188
|
}
|
|
191
189
|
|
|
192
190
|
entries.push(entry);
|
|
@@ -214,7 +212,6 @@ export async function reportBundleCommand(options: ReportBundleOptions): Promise
|
|
|
214
212
|
caseStudy: e.caseStudy ?? null,
|
|
215
213
|
htmlReport: e.htmlReport ?? null,
|
|
216
214
|
diagnose: e.diagnose ?? null,
|
|
217
|
-
agentDirective: e.agentDirective ?? null,
|
|
218
215
|
})),
|
|
219
216
|
skipped,
|
|
220
217
|
}),
|
|
@@ -284,8 +281,8 @@ function renderIndex(entries: BundleEntry[], skipped: Array<{ runId: number; rea
|
|
|
284
281
|
lines.push(`Generated: ${new Date().toISOString()}`);
|
|
285
282
|
lines.push(`Runs: ${entries.length}` + (skipped.length > 0 ? ` (skipped: ${skipped.length})` : ""));
|
|
286
283
|
lines.push("");
|
|
287
|
-
lines.push("| Run | Spec | Total | Pass | Fail | Skip | Artefacts |
|
|
288
|
-
lines.push("
|
|
284
|
+
lines.push("| Run | Spec | Total | Pass | Fail | Skip | Artefacts |");
|
|
285
|
+
lines.push("|----:|------|------:|-----:|-----:|-----:|-----------|");
|
|
289
286
|
|
|
290
287
|
for (const e of entries) {
|
|
291
288
|
const links: string[] = [];
|
|
@@ -293,7 +290,7 @@ function renderIndex(entries: BundleEntry[], skipped: Array<{ runId: number; rea
|
|
|
293
290
|
if (e.htmlReport) links.push(`[html](${rel(e.htmlReport)})`);
|
|
294
291
|
if (e.diagnose) links.push(`[diagnose](${rel(e.diagnose)})`);
|
|
295
292
|
lines.push(
|
|
296
|
-
`| ${e.runId} | ${e.spec ?? "—"} | ${e.totals.total} | ${e.totals.passed} | ${e.totals.failed} | ${e.totals.skipped} | ${links.join(" · ") || "—"}
|
|
293
|
+
`| ${e.runId} | ${e.spec ?? "—"} | ${e.totals.total} | ${e.totals.passed} | ${e.totals.failed} | ${e.totals.skipped} | ${links.join(" · ") || "—"} |`,
|
|
297
294
|
);
|
|
298
295
|
}
|
|
299
296
|
|
|
@@ -311,8 +308,3 @@ function rel(p: string): string {
|
|
|
311
308
|
const tail = seg.slice(-2);
|
|
312
309
|
return tail.join("/");
|
|
313
310
|
}
|
|
314
|
-
|
|
315
|
-
function truncate(s: string, n: number): string {
|
|
316
|
-
if (s.length <= n) return s.replace(/\n/g, " ");
|
|
317
|
-
return s.slice(0, n - 1).replace(/\n/g, " ") + "…";
|
|
318
|
-
}
|
|
@@ -9,6 +9,7 @@ import { resolveCollectionSpec } from "../../core/setup-api.ts";
|
|
|
9
9
|
import { findCollectionByNameOrId } from "../../db/queries.ts";
|
|
10
10
|
import { getDb } from "../../db/schema.ts";
|
|
11
11
|
import type { AssertionResult } from "../../core/runner/types.ts";
|
|
12
|
+
import { applyEnvWrites } from "./fixtures.ts";
|
|
12
13
|
|
|
13
14
|
// TASK-272: when the request fails authentication (401/403) and the user
|
|
14
15
|
// did NOT pass `--api <name>`, surface a one-liner pointing at auto-auth via
|
|
@@ -86,6 +87,10 @@ export interface RequestOptions {
|
|
|
86
87
|
env?: string;
|
|
87
88
|
api?: string;
|
|
88
89
|
jsonPath?: string;
|
|
90
|
+
/** ARV-355: write the --json-path scalar into apis/<api>/.env.yaml as this
|
|
91
|
+
* var, so a POST-create's returned id lands in a fixture in one command
|
|
92
|
+
* (the "execute create+capture" primitive of the agent seed loop). */
|
|
93
|
+
capture?: string;
|
|
89
94
|
dbPath?: string;
|
|
90
95
|
json?: boolean;
|
|
91
96
|
/** TASK-142: validate the response body against the OpenAPI response schema. */
|
|
@@ -109,6 +114,18 @@ interface SchemaValidationOutcome {
|
|
|
109
114
|
|
|
110
115
|
export async function requestCommand(options: RequestOptions): Promise<number> {
|
|
111
116
|
try {
|
|
117
|
+
// ARV-355: --capture needs a value to extract (--json-path) and a target
|
|
118
|
+
// .env.yaml (--api). Fail fast with an actionable message rather than
|
|
119
|
+
// silently no-op'ing mid-seed-loop.
|
|
120
|
+
if (options.capture && (!options.jsonPath || !options.api)) {
|
|
121
|
+
printError(
|
|
122
|
+
"--capture requires --json-path <path> (the field to extract) and --api <name> " +
|
|
123
|
+
"(the .env.yaml to write). E.g. `zond request POST /v1/accounts --api stripe " +
|
|
124
|
+
"--json-path id --capture account`.",
|
|
125
|
+
);
|
|
126
|
+
return 2;
|
|
127
|
+
}
|
|
128
|
+
|
|
112
129
|
const headers: Record<string, string> = {};
|
|
113
130
|
if (options.headers) {
|
|
114
131
|
for (const h of options.headers) {
|
|
@@ -141,6 +158,20 @@ export async function requestCommand(options: RequestOptions): Promise<number> {
|
|
|
141
158
|
form: useForm,
|
|
142
159
|
});
|
|
143
160
|
|
|
161
|
+
// ARV-265 (B3): persist this ad-hoc call into runs/results when a
|
|
162
|
+
// session is active, so `zond coverage --scope audit` attributes it.
|
|
163
|
+
// No session → no DB write (mirrors `curl`-replacement intent).
|
|
164
|
+
await maybePersistAuditedRequest({
|
|
165
|
+
options,
|
|
166
|
+
method: options.method.toUpperCase(),
|
|
167
|
+
url: options.url,
|
|
168
|
+
headers,
|
|
169
|
+
body: options.body,
|
|
170
|
+
result,
|
|
171
|
+
}).catch((err) => {
|
|
172
|
+
process.stderr.write(`zond: audit persistence failed (${(err as Error).message}).\n`);
|
|
173
|
+
});
|
|
174
|
+
|
|
144
175
|
let validation: SchemaValidationOutcome | null = null;
|
|
145
176
|
if (options.validateSchema || options.validateAgainst) {
|
|
146
177
|
validation = await runSchemaValidation(options, result);
|
|
@@ -173,6 +204,33 @@ export async function requestCommand(options: RequestOptions): Promise<number> {
|
|
|
173
204
|
if (validation) printSchemaValidation(validation);
|
|
174
205
|
}
|
|
175
206
|
|
|
207
|
+
// ARV-355: create+capture. Write the extracted json-path scalar into the
|
|
208
|
+
// api's .env.yaml so the agent seed loop can chain a created id into
|
|
209
|
+
// dependent fixtures with one command. Deterministic plumbing only — the
|
|
210
|
+
// agent decides the body/endpoint/var; zond just POSTs and captures.
|
|
211
|
+
// Gated to a 2xx response (capturing an id from an error body is a bug —
|
|
212
|
+
// the agent reads the note, revises the body, and retries).
|
|
213
|
+
if (options.capture) {
|
|
214
|
+
const v = result.body;
|
|
215
|
+
const ok2xx = result.status >= 200 && result.status < 300;
|
|
216
|
+
const scalar =
|
|
217
|
+
typeof v === "string" || typeof v === "number" || typeof v === "boolean" ? String(v) : null;
|
|
218
|
+
if (!ok2xx || scalar === null || scalar.length === 0) {
|
|
219
|
+
process.stderr.write(
|
|
220
|
+
`zond: --capture ${options.capture} skipped — need a non-empty scalar from a 2xx response ` +
|
|
221
|
+
`(status=${result.status}, --json-path '${options.jsonPath}' → ${scalar === null ? "non-scalar/null" : "empty"}). ` +
|
|
222
|
+
`Revise the request and retry.\n`,
|
|
223
|
+
);
|
|
224
|
+
} else {
|
|
225
|
+
const envPath = join(`apis/${options.api}`, ".env.yaml");
|
|
226
|
+
const { backup } = await applyEnvWrites(envPath, { [options.capture]: scalar });
|
|
227
|
+
process.stderr.write(
|
|
228
|
+
`zond: captured ${options.capture}=${scalar} → ${envPath}` +
|
|
229
|
+
(backup ? ` (backup: ${backup})` : "") + "\n",
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
176
234
|
// TASK-272: post-response auto-auth hint on 401/403 without --api
|
|
177
235
|
if (
|
|
178
236
|
!options.json
|
|
@@ -211,6 +269,56 @@ export async function requestCommand(options: RequestOptions): Promise<number> {
|
|
|
211
269
|
}
|
|
212
270
|
}
|
|
213
271
|
|
|
272
|
+
// ──────────────────────────────────────────────
|
|
273
|
+
// ARV-265 (B3): persist an ad-hoc `zond request` call into runs/results
|
|
274
|
+
// so `zond coverage --scope audit` sees the HTTP touch. Only fires when
|
|
275
|
+
// a session is active — outside a session, the command stays a pure
|
|
276
|
+
// curl-replacement and the DB doesn't grow with one-off requests.
|
|
277
|
+
// ──────────────────────────────────────────────
|
|
278
|
+
|
|
279
|
+
async function maybePersistAuditedRequest(args: {
|
|
280
|
+
options: RequestOptions;
|
|
281
|
+
method: string;
|
|
282
|
+
url: string;
|
|
283
|
+
headers: Record<string, string>;
|
|
284
|
+
body: string | undefined;
|
|
285
|
+
result: { status: number; headers: Record<string, string>; body: unknown; duration_ms: number };
|
|
286
|
+
}): Promise<void> {
|
|
287
|
+
const { readCurrentSession } = await import("../../core/context/session.ts");
|
|
288
|
+
const session = readCurrentSession();
|
|
289
|
+
if (!session) return; // B3: outside-session calls leave no trace.
|
|
290
|
+
|
|
291
|
+
const { checksPersistEnabled, beginAuditRun, finalizeAuditRun } =
|
|
292
|
+
await import("../../core/audit/persist.ts");
|
|
293
|
+
if (!checksPersistEnabled()) return;
|
|
294
|
+
|
|
295
|
+
getDb(args.options.dbPath);
|
|
296
|
+
const collectionId = args.options.api ? findCollectionByNameOrId(args.options.api)?.id : undefined;
|
|
297
|
+
const runId = beginAuditRun({
|
|
298
|
+
runKind: "request",
|
|
299
|
+
...(collectionId != null ? { collectionId } : {}),
|
|
300
|
+
sessionId: session.id,
|
|
301
|
+
tags: ["request", "ad-hoc"],
|
|
302
|
+
});
|
|
303
|
+
const status = args.result.status >= 200 && args.result.status < 400 ? "pass" : "fail";
|
|
304
|
+
finalizeAuditRun(runId, [
|
|
305
|
+
{
|
|
306
|
+
suiteName: "request/ad-hoc",
|
|
307
|
+
suiteFile: `apis/${args.options.api ?? "_"}/request.yaml`,
|
|
308
|
+
testName: `request::${args.method} ${args.url}`,
|
|
309
|
+
status,
|
|
310
|
+
request: { method: args.method, url: args.url, headers: args.headers, body: args.body },
|
|
311
|
+
response: {
|
|
312
|
+
status: args.result.status,
|
|
313
|
+
headers: args.result.headers,
|
|
314
|
+
body: typeof args.result.body === "string" ? args.result.body : JSON.stringify(args.result.body ?? ""),
|
|
315
|
+
duration_ms: args.result.duration_ms,
|
|
316
|
+
},
|
|
317
|
+
durationMs: args.result.duration_ms,
|
|
318
|
+
},
|
|
319
|
+
]);
|
|
320
|
+
}
|
|
321
|
+
|
|
214
322
|
// ──────────────────────────────────────────────
|
|
215
323
|
// TASK-142: --validate-schema / --validate-against
|
|
216
324
|
// ──────────────────────────────────────────────
|
|
@@ -398,6 +506,13 @@ export function registerRequest(program: Command): void {
|
|
|
398
506
|
"(`id=$(zond request --json-path data.id ...)`), objects/arrays as compact JSON. " +
|
|
399
507
|
"With --json, embeds the extracted value as the envelope's `body` field.",
|
|
400
508
|
)
|
|
509
|
+
.option(
|
|
510
|
+
"--capture <var>",
|
|
511
|
+
"ARV-355: write the --json-path scalar into apis/<api>/.env.yaml as <var> " +
|
|
512
|
+
"(requires --json-path and --api). Captures a POST-create's returned id into " +
|
|
513
|
+
"a fixture in one command, so the agent seed loop can chain dependent creates. " +
|
|
514
|
+
"Only writes on a 2xx response; a .env.yaml.bak backup is made.",
|
|
515
|
+
)
|
|
401
516
|
.option("--db <path>", "Path to SQLite database file")
|
|
402
517
|
.option("--validate-schema", "TASK-142: validate the response body against the OpenAPI response schema (requires --api). Endpoint is auto-resolved from the request method + URL.path; templated paths like /users/{id} are matched via regex. Falls back gracefully if no endpoint matches — pass --validate-against to override.")
|
|
403
518
|
.option("--validate-against <method:path>", "TASK-142: explicit endpoint override for --validate-schema, e.g. \"GET:/users/{id}\". Use the spec template form (with \"{...}\" placeholders).")
|
|
@@ -422,6 +537,7 @@ export function registerRequest(program: Command): void {
|
|
|
422
537
|
env: opts.env,
|
|
423
538
|
api,
|
|
424
539
|
jsonPath: opts.jsonPath,
|
|
540
|
+
capture: opts.capture,
|
|
425
541
|
dbPath: opts.db,
|
|
426
542
|
json: globalJson(cmd),
|
|
427
543
|
validateSchema: opts.validateSchema === true || typeof opts.validateAgainst === "string",
|
package/src/cli/commands/run.ts
CHANGED
|
@@ -160,6 +160,26 @@ export async function runCommand(options: RunOptions): Promise<number> {
|
|
|
160
160
|
// the rogue character. Mutating options.paths so downstream lookups
|
|
161
161
|
// (collection by test_path, env-file search) see the cleaned form.
|
|
162
162
|
options.paths = options.paths.map((p) => p.trim());
|
|
163
|
+
|
|
164
|
+
// ARV-383: a path that does not exist on disk is a hard error, not an
|
|
165
|
+
// empty (exit-0) run. Without this, `parseDirectorySafe` globs a missing
|
|
166
|
+
// cwd -> 0 suites and run falls through to the ARV-357 empty-report path,
|
|
167
|
+
// reporting a green 0-test "pass". Real-world trigger: a probe that matched
|
|
168
|
+
// 0 fields never created its output dir, then `zond run <that-dir>` went
|
|
169
|
+
// silently green. An existing-but-empty dir still flows to ARV-357.
|
|
170
|
+
const missingPaths: string[] = [];
|
|
171
|
+
for (const p of options.paths) {
|
|
172
|
+
try {
|
|
173
|
+
await stat(p);
|
|
174
|
+
} catch {
|
|
175
|
+
missingPaths.push(p);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (missingPaths.length > 0) {
|
|
179
|
+
printError(`No such file or directory: ${missingPaths.join(", ")} — nothing to run`);
|
|
180
|
+
return 2;
|
|
181
|
+
}
|
|
182
|
+
|
|
163
183
|
const primaryPath = options.paths[0]!;
|
|
164
184
|
|
|
165
185
|
// 1. Parse test files from every input path (collect parse errors instead
|
|
@@ -189,6 +209,24 @@ export async function runCommand(options: RunOptions): Promise<number> {
|
|
|
189
209
|
return 2;
|
|
190
210
|
}
|
|
191
211
|
printWarning(`No test files found in ${pathList}`);
|
|
212
|
+
// ARV-357: an empty dir used to exit 0 and write NO --output file, so a
|
|
213
|
+
// scripted pipeline saw a missing file with no error to key on. Still
|
|
214
|
+
// write the requested report (empty JSON `[]` / junit tests=0 envelope)
|
|
215
|
+
// so downstream stages parse "zero tests" instead of silently skipping.
|
|
216
|
+
if (options.output) {
|
|
217
|
+
try {
|
|
218
|
+
const spec = resolveOutput(RUN_OUTPUT_SPEC, { report: options.report, output: options.output });
|
|
219
|
+
if (spec.channel === "file") {
|
|
220
|
+
const content = spec.format === "junit" ? generateJunitXml([]) : generateJsonReport([]);
|
|
221
|
+
await mkdir(pathDirname(spec.path!), { recursive: true });
|
|
222
|
+
await writeFile(spec.path!, content, "utf-8");
|
|
223
|
+
process.stderr.write(`zond: empty report written to ${spec.path!} (0 tests)\n`);
|
|
224
|
+
}
|
|
225
|
+
} catch (err) {
|
|
226
|
+
printError(`Failed to write --output file ${options.output}: ${(err as Error).message}`);
|
|
227
|
+
return 2;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
192
230
|
return 0;
|
|
193
231
|
}
|
|
194
232
|
|
|
@@ -447,9 +485,11 @@ export async function runCommand(options: RunOptions): Promise<number> {
|
|
|
447
485
|
}
|
|
448
486
|
|
|
449
487
|
// TASK-282: validate --learn flag combinations early (before run).
|
|
488
|
+
// ARV-199: --learn-apply implies --learn — cf. `git stash apply` (no
|
|
489
|
+
// explicit `git stash` toggle required). The earlier hard error here
|
|
490
|
+
// tripped up scripted callers that only thought to pass --learn-apply.
|
|
450
491
|
if (options.learnApply && !options.learn) {
|
|
451
|
-
|
|
452
|
-
return 2;
|
|
492
|
+
options.learn = true;
|
|
453
493
|
}
|
|
454
494
|
if (options.learnApply && !options.learnTarget) {
|
|
455
495
|
printError("--learn-apply requires --learn-target=test or --learn-target=drifts");
|
|
@@ -830,8 +870,16 @@ export async function runCommand(options: RunOptions): Promise<number> {
|
|
|
830
870
|
// --json so the JSON envelope stays alone on stdout (this is stderr
|
|
831
871
|
// anyway, but skip avoids confusing parsers that capture both streams).
|
|
832
872
|
if (hasFailures && !options.json) {
|
|
833
|
-
|
|
834
|
-
|
|
873
|
+
// ARV-318: error steps (env_issue/network — couldn't execute) also drive
|
|
874
|
+
// the non-zero exit (see hasFailures above). The old line counted only
|
|
875
|
+
// `failed`, so an all-errored run printed "0 test step(s) failed —
|
|
876
|
+
// exiting with code 1", a contradiction. Name both classes.
|
|
877
|
+
const failed = results.reduce((s, r) => s + r.failed, 0);
|
|
878
|
+
const errored = results.reduce((s, r) => s + r.steps.filter((st) => st.status === "error").length, 0);
|
|
879
|
+
const parts: string[] = [];
|
|
880
|
+
if (failed > 0) parts.push(`${failed} step(s) failed`);
|
|
881
|
+
if (errored > 0) parts.push(`${errored} step(s) errored (couldn't execute — see failure_class)`);
|
|
882
|
+
process.stderr.write(`zond: ${parts.join(", ")} — exiting with code 1 (pass --no-fail-on-failures to suppress, e.g. for advisory runs).\n`);
|
|
835
883
|
}
|
|
836
884
|
|
|
837
885
|
if (hasFailures && options.failOnFailures === false) {
|
|
@@ -1000,7 +1048,7 @@ export function registerRun(program: Command): void {
|
|
|
1000
1048
|
.option("--spec <path>", "Path or URL to OpenAPI spec used for --validate-schema (overrides the collection's openapi_spec)")
|
|
1001
1049
|
.option("--session-id <id>", "Group this run under a session. Resolution order: --session-id flag > ZOND_SESSION_ID env > .zond/current-session file (set by 'zond session start')")
|
|
1002
1050
|
.option("--learn", "TASK-282: detect status-code drift (passing-test-but-wrong-status). Implies --validate-schema. Without --learn-apply prints the plan; combine with --learn-apply --learn-target=test|drifts to mutate.")
|
|
1003
|
-
.option("--learn-apply", "Apply the drift plan instead of printing it.
|
|
1051
|
+
.option("--learn-apply", "Apply the drift plan instead of printing it. Implies --learn; still requires --learn-target=test|drifts.")
|
|
1004
1052
|
.addOption(
|
|
1005
1053
|
new Option("--learn-target <where>", "What to mutate when --learn-apply is set: rewrite expect.status in YAML (test) or append to apis/<name>/tolerated-drifts.yaml (drifts)")
|
|
1006
1054
|
.choices(["test", "drifts"]),
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ARV-175: `zond schema-from-runs` — mine 2xx response bodies from a
|
|
3
|
+
* persisted run and emit `patch.schema.json` (inferred JSON Schema per
|
|
4
|
+
* endpoint+status). Pairs with `refresh-api --merge-schema` (ARV-176).
|
|
5
|
+
*/
|
|
6
|
+
import { writeFile } from "node:fs/promises";
|
|
7
|
+
import type { Command } from "commander";
|
|
8
|
+
import { readOpenApiSpec, extractEndpoints } from "../../core/generator/index.ts";
|
|
9
|
+
import { schemaFromRuns, type ResultRow } from "../../core/spec/schema-from-runs.ts";
|
|
10
|
+
import { getResultsByRunId, getLatestRunId } from "../../db/queries.ts";
|
|
11
|
+
import { getDb } from "../../db/schema.ts";
|
|
12
|
+
import { resolveApiCollection, globalJson } from "../resolve.ts";
|
|
13
|
+
import { getApi } from "../util/api-context.ts";
|
|
14
|
+
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
15
|
+
import { printError, printSuccess, printWarning } from "../output.ts";
|
|
16
|
+
import { parsePositiveInt } from "../argv.ts";
|
|
17
|
+
|
|
18
|
+
export interface SchemaFromRunsCliOptions {
|
|
19
|
+
api?: string;
|
|
20
|
+
spec?: string;
|
|
21
|
+
db?: string;
|
|
22
|
+
run?: number;
|
|
23
|
+
minSamples?: number;
|
|
24
|
+
out?: string;
|
|
25
|
+
engine?: string;
|
|
26
|
+
json?: boolean;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function schemaFromRunsCommand(
|
|
30
|
+
opts: SchemaFromRunsCliOptions,
|
|
31
|
+
cmd: Command,
|
|
32
|
+
): Promise<number> {
|
|
33
|
+
const json = opts.json === true || globalJson(cmd);
|
|
34
|
+
const label = "schema-from-runs";
|
|
35
|
+
|
|
36
|
+
// ARV-175 AC#2: only `builtin` is wired. quicktype/genson would each pull
|
|
37
|
+
// a large dependency tree, which contradicts zond's minimal-deps charter.
|
|
38
|
+
const engine = (opts.engine ?? "builtin").toLowerCase();
|
|
39
|
+
if (engine !== "builtin") {
|
|
40
|
+
const msg = `--engine "${opts.engine}" is not available. Only 'builtin' is wired (quicktype/genson would add heavy deps; see src/CLAUDE.md).`;
|
|
41
|
+
if (json) printJson(jsonError(label, [msg])); else printError(msg);
|
|
42
|
+
return 2;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Resolve spec: --spec wins, else --api / current-api collection.
|
|
46
|
+
let spec = opts.spec;
|
|
47
|
+
const apiFlag = opts.spec ? opts.api : getApi(cmd, opts as Record<string, unknown>);
|
|
48
|
+
if (!spec && apiFlag) {
|
|
49
|
+
const resolved = resolveApiCollection(apiFlag, opts.db);
|
|
50
|
+
if ("error" in resolved) {
|
|
51
|
+
if (json) printJson(jsonError(label, [resolved.error])); else printError(resolved.error);
|
|
52
|
+
return resolved.error.startsWith("Failed") ? 2 : 1;
|
|
53
|
+
}
|
|
54
|
+
if (resolved.spec) spec = resolved.spec;
|
|
55
|
+
}
|
|
56
|
+
if (!spec) {
|
|
57
|
+
const msg = "No spec: pass --spec <path> or --api <name> (or set a current API).";
|
|
58
|
+
if (json) printJson(jsonError(label, [msg])); else printError(msg);
|
|
59
|
+
return 2;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
getDb(opts.db);
|
|
63
|
+
const runId = opts.run ?? getLatestRunId();
|
|
64
|
+
if (runId == null) {
|
|
65
|
+
const msg = "No runs in the database. Run `zond run` / `zond checks run` first, then re-run schema-from-runs.";
|
|
66
|
+
if (json) printJson(jsonError(label, [msg])); else printError(msg);
|
|
67
|
+
return 2;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
let doc;
|
|
71
|
+
try {
|
|
72
|
+
doc = await readOpenApiSpec(spec);
|
|
73
|
+
} catch (err) {
|
|
74
|
+
const msg = `Failed to read spec at "${spec}": ${(err as Error).message}`;
|
|
75
|
+
if (json) printJson(jsonError(label, [msg])); else printError(msg);
|
|
76
|
+
return 2;
|
|
77
|
+
}
|
|
78
|
+
const endpoints = extractEndpoints(doc);
|
|
79
|
+
|
|
80
|
+
const rows = getResultsByRunId(runId) as unknown as ResultRow[];
|
|
81
|
+
const minSamples = opts.minSamples ?? 2;
|
|
82
|
+
const result = schemaFromRuns({ results: rows, endpoints, minSamples });
|
|
83
|
+
|
|
84
|
+
const emitted = result.groups.filter((g) => g.emitted);
|
|
85
|
+
const skipped = result.groups.filter((g) => !g.emitted);
|
|
86
|
+
const out = opts.out ?? "patch.schema.json";
|
|
87
|
+
|
|
88
|
+
if (emitted.length > 0) {
|
|
89
|
+
await writeFile(out, JSON.stringify(result.patch, null, 2) + "\n", "utf-8");
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (json) {
|
|
93
|
+
printJson(jsonOk(label, {
|
|
94
|
+
run_id: runId,
|
|
95
|
+
out: emitted.length > 0 ? out : null,
|
|
96
|
+
min_samples: minSamples,
|
|
97
|
+
emitted: emitted.map((g) => ({ endpoint: g.endpoint, status: g.status, samples: g.samples })),
|
|
98
|
+
skipped: skipped.map((g) => ({ endpoint: g.endpoint, status: g.status, samples: g.samples, reason: g.reason })),
|
|
99
|
+
}));
|
|
100
|
+
return 0;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (emitted.length === 0) {
|
|
104
|
+
printWarning(`No schemas emitted from run #${runId}: no (endpoint, status) group reached --min-samples ${minSamples}. ${skipped.length} group(s) had too few 2xx JSON samples.`);
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
printSuccess(`Wrote ${emitted.length} response schema(s) from run #${runId} to ${out}`);
|
|
108
|
+
for (const g of emitted) console.log(` ${g.endpoint} → ${g.status} (${g.samples} samples)`);
|
|
109
|
+
for (const g of skipped) printWarning(`skipped ${g.endpoint} → ${g.status}: ${g.reason} (${g.samples} sample(s))`);
|
|
110
|
+
console.log(`\nNext: zond refresh-api --api <name> --merge-schema ${out}`);
|
|
111
|
+
return 0;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function registerSchemaFromRuns(program: Command): void {
|
|
115
|
+
program
|
|
116
|
+
.command("schema-from-runs")
|
|
117
|
+
.description("ARV-175: infer response JSON Schemas from a run's 2xx bodies → patch.schema.json (pairs with refresh-api --merge-schema). Revives response_schema_conformance on specs that declare no response schemas.")
|
|
118
|
+
.option("--api <name>", "Registered API (spec + DB lookup)")
|
|
119
|
+
.option("--spec <path>", "Explicit OpenAPI spec path (overrides --api)")
|
|
120
|
+
.option("--run <id>", "Run id to mine (default: latest)", parsePositiveInt("--run"))
|
|
121
|
+
.option("--min-samples <n>", "Minimum 2xx samples per endpoint+status to emit (default 2)", parsePositiveInt("--min-samples"))
|
|
122
|
+
.option("--out <path>", "Output path for patch.schema.json (default: patch.schema.json)")
|
|
123
|
+
.option("--engine <name>", "Schema engine. Only 'builtin' is wired (default).", "builtin")
|
|
124
|
+
.option("--db <path>", "SQLite path")
|
|
125
|
+
.action(async (opts, cmd: Command) => {
|
|
126
|
+
process.exitCode = await schemaFromRunsCommand(opts, cmd);
|
|
127
|
+
});
|
|
128
|
+
}
|