@kirrosh/zond 0.23.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +176 -1
- package/README.md +8 -7
- package/package.json +2 -3
- package/src/CLAUDE.md +112 -0
- package/src/cli/commands/add-api.ts +19 -7
- package/src/cli/commands/api/annotate/index.ts +359 -4
- package/src/cli/commands/api/annotate/lifecycle.ts +1 -1
- package/src/cli/commands/api/annotate/overlay.ts +1 -1
- package/src/cli/commands/api/annotate/pagination.ts +10 -6
- package/src/cli/commands/api/annotate/prompts.ts +39 -2
- package/src/cli/commands/audit.ts +360 -54
- package/src/cli/commands/check.ts +15 -2
- package/src/cli/commands/checks.ts +352 -36
- package/src/cli/commands/cleanup.ts +4 -30
- package/src/cli/commands/coverage.ts +275 -57
- package/src/cli/commands/db.ts +311 -8
- package/src/cli/commands/discover.ts +281 -161
- package/src/cli/commands/doctor.ts +57 -3
- package/src/cli/commands/fixtures.ts +1 -1
- package/src/cli/commands/generate.ts +24 -7
- package/src/cli/commands/init/bootstrap.ts +4 -1
- package/src/cli/commands/init/index.ts +2 -2
- package/src/cli/commands/init/skills.ts +43 -0
- package/src/cli/commands/init/templates/agents.md +12 -3
- package/src/cli/commands/init/templates/skills/warm-up-target.md +122 -0
- package/src/cli/commands/init/templates/skills/zond-checks.md +268 -44
- package/src/cli/commands/init/templates/skills/zond-seed.md +114 -0
- package/src/cli/commands/init/templates/skills/zond-triage.md +88 -26
- package/src/cli/commands/init/templates/skills/zond.md +274 -64
- package/src/cli/commands/init/templates/zond-config.yml +1 -1
- package/src/cli/commands/prepare-fixtures.ts +14 -52
- package/src/cli/commands/probe/_seed-bodies.ts +52 -0
- package/src/cli/commands/probe/mass-assignment.ts +101 -10
- package/src/cli/commands/probe/security.ts +95 -12
- package/src/cli/commands/probe/webhooks.ts +2 -0
- package/src/cli/commands/probe.ts +87 -11
- package/src/cli/commands/refresh-api.ts +59 -1
- package/src/cli/commands/report-bundle.ts +3 -11
- package/src/cli/commands/request.ts +116 -0
- package/src/cli/commands/run.ts +53 -5
- package/src/cli/commands/schema-from-runs.ts +128 -0
- package/src/cli/commands/secrets.ts +133 -0
- package/src/cli/json-envelope.ts +0 -20
- package/src/cli/json-schemas.ts +51 -0
- package/src/cli/output.ts +17 -1
- package/src/cli/program.ts +5 -4
- package/src/cli/safe-live.ts +24 -0
- package/src/cli/status-filter.ts +0 -10
- package/src/core/audit/persist.ts +183 -0
- package/src/core/checks/budget.ts +59 -0
- package/src/core/checks/checks/cross_call_references.ts +17 -4
- package/src/core/checks/checks/cursor_boundary_fuzzing.ts +219 -0
- package/src/core/checks/checks/idempotency_replay.ts +1 -5
- package/src/core/checks/checks/ignored_auth.ts +44 -1
- package/src/core/checks/checks/index.ts +3 -0
- package/src/core/checks/checks/lifecycle_transitions.ts +169 -26
- package/src/core/checks/checks/negative_data_rejection.ts +119 -16
- package/src/core/checks/checks/not_a_server_error.ts +8 -0
- package/src/core/checks/checks/open_cors_on_sensitive.ts +47 -18
- package/src/core/checks/checks/pagination_invariants.ts +298 -117
- package/src/core/checks/checks/positive_data_acceptance.ts +1 -4
- package/src/core/checks/checks/status_code_conformance.ts +78 -7
- package/src/core/checks/mode.ts +3 -0
- package/src/core/checks/recommended-action.ts +5 -1
- package/src/core/checks/runner.ts +614 -27
- package/src/core/checks/spec-findings.ts +308 -0
- package/src/core/checks/types.ts +117 -1
- package/src/core/checks/zond-extensions.ts +73 -0
- package/src/core/classifier/recommended-action.ts +35 -6
- package/src/core/coverage/loader.ts +31 -0
- package/src/core/diagnostics/db-analysis.ts +200 -106
- package/src/core/diagnostics/failure-class.ts +21 -1
- package/src/core/diagnostics/failure-hints.ts +4 -208
- package/src/core/diagnostics/suggested-fixes.ts +2 -3
- package/src/core/generator/chunker.ts +1 -8
- package/src/core/generator/data-factory.ts +199 -61
- package/src/core/generator/fixtures-builder.ts +38 -31
- package/src/core/generator/index.ts +0 -2
- package/src/core/generator/openapi-reader.ts +98 -4
- package/src/core/generator/path-param-disambig.ts +30 -4
- package/src/core/generator/resources-builder.ts +276 -26
- package/src/core/generator/schema-utils.ts +22 -0
- package/src/core/generator/suite-generator.ts +168 -15
- package/src/core/generator/types.ts +6 -0
- package/src/core/identity/identity-file.ts +0 -0
- package/src/core/output/README.md +11 -29
- package/src/core/output/index.ts +1 -1
- package/src/core/output/run.ts +0 -35
- package/src/core/output/types.ts +0 -7
- package/src/core/parser/dynamic-values.ts +160 -0
- package/src/core/parser/variables.ts +0 -0
- package/src/core/probe/dry-run-envelope.ts +4 -0
- package/src/core/probe/mass-assignment/classify.ts +175 -0
- package/src/core/probe/mass-assignment/cleanup.ts +52 -0
- package/src/core/probe/mass-assignment/digest.ts +114 -0
- package/src/core/probe/mass-assignment/orchestrator.ts +459 -0
- package/src/core/probe/mass-assignment/regression.ts +141 -0
- package/src/core/probe/mass-assignment/suspects.ts +92 -0
- package/src/core/probe/mass-assignment/types.ts +135 -0
- package/src/core/probe/mass-assignment-probe.ts +23 -1118
- package/src/core/probe/mass-assignment-template.ts +32 -4
- package/src/core/probe/path-discovery.ts +3 -4
- package/src/core/probe/probe-harness.ts +21 -22
- package/src/core/probe/security/baseline.ts +174 -0
- package/src/core/probe/security/classify.ts +341 -0
- package/src/core/probe/security/cleanup.ts +125 -0
- package/src/core/probe/security/detectors.ts +71 -0
- package/src/core/probe/security/digest.ts +104 -0
- package/src/core/probe/security/orchestrator.ts +398 -0
- package/src/core/probe/security/regression.ts +103 -0
- package/src/core/probe/security/types.ts +151 -0
- package/src/core/probe/security-probe-class.ts +8 -2
- package/src/core/probe/security-probe.ts +28 -1449
- package/src/core/probe/shared.ts +26 -0
- package/src/core/probe/webhooks-probe.ts +5 -7
- package/src/core/runner/assertions.ts +1 -1
- package/src/core/runner/executor.ts +3 -18
- package/src/core/runner/form-encode.ts +8 -18
- package/src/core/runner/http-client.ts +38 -1
- package/src/core/runner/preflight-vars.ts +19 -15
- package/src/core/runner/rate-limiter.ts +11 -29
- package/src/core/runner/run-kind.ts +7 -1
- package/src/core/runner/schema-validator.ts +2 -6
- package/src/core/runner/send-request.ts +11 -6
- package/src/core/runner/types.ts +6 -0
- package/src/core/setup-api.ts +53 -15
- package/src/core/severity/index.ts +0 -63
- package/src/core/spec/infer-schema.ts +102 -0
- package/src/core/spec/merge-specs.ts +156 -0
- package/src/core/spec/schema-from-runs.ts +117 -0
- package/src/core/spec/schema-overlay.ts +130 -0
- package/src/core/util/ajv.ts +13 -0
- package/src/core/util/headers.ts +9 -0
- package/src/core/util/url.ts +24 -0
- package/src/core/workspace/fixture-gap-report.ts +84 -0
- package/src/core/workspace/fixture-gaps.ts +71 -0
- package/src/core/workspace/root.ts +13 -11
- package/src/db/migrate.ts +2 -0
- package/src/db/migrations/0002_run_kind_request.sql +59 -0
- package/src/db/queries/collections.ts +2 -2
- package/src/db/queries/results.ts +88 -0
- package/src/db/queries/runs.ts +56 -2
- package/src/db/queries.ts +3 -0
- package/src/db/schema.ts +7 -7
- package/src/cli/commands/bootstrap.ts +0 -710
- package/src/core/anti-fp/bootstrap.ts +0 -34
- package/src/core/anti-fp/index.ts +0 -33
- package/src/core/anti-fp/registry.ts +0 -44
- package/src/core/anti-fp/rules/baseline-echo.ts +0 -74
- package/src/core/anti-fp/rules/schemathesis/body_negation_becomes_valid.ts +0 -52
- package/src/core/anti-fp/rules/schemathesis/coverage_phase_boundary_positive.ts +0 -38
- package/src/core/anti-fp/rules/schemathesis/has_unverifiable_mutations.ts +0 -35
- package/src/core/anti-fp/rules/schemathesis/index.ts +0 -24
- package/src/core/anti-fp/rules/schemathesis/string_type_mutation_becomes_valid.ts +0 -53
- package/src/core/anti-fp/rules/subscription-gated/index.ts +0 -11
- package/src/core/anti-fp/rules/subscription-gated/paid-plan-403.ts +0 -75
- package/src/core/anti-fp/types.ts +0 -68
- package/src/core/generator/create-body.ts +0 -89
package/src/core/probe/shared.ts
CHANGED
|
@@ -173,6 +173,32 @@ export function renderPath(
|
|
|
173
173
|
});
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
+
/** ARV-244/ARV-245: percent-encode unsafe characters per path segment,
|
|
177
|
+
* preserving anything already percent-encoded (`%XX`). Slashes, the
|
|
178
|
+
* unreserved set, and a conservative slice of sub-delims are kept
|
|
179
|
+
* verbatim. Used both for orphan-cleanup DELETE URLs and paste-ready
|
|
180
|
+
* manual repro lines in the security-probe digest. */
|
|
181
|
+
export function encodePathForRepro(deletePath: string): string {
|
|
182
|
+
const SAFE = /[A-Za-z0-9._~!$&'()*+,;=:@-]/;
|
|
183
|
+
return deletePath
|
|
184
|
+
.split("/")
|
|
185
|
+
.map((segment) => {
|
|
186
|
+
if (segment.length === 0) return segment;
|
|
187
|
+
let out = "";
|
|
188
|
+
for (let i = 0; i < segment.length; i++) {
|
|
189
|
+
const ch = segment.charAt(i);
|
|
190
|
+
if (ch === "%" && /^[0-9A-Fa-f]{2}$/.test(segment.slice(i + 1, i + 3))) {
|
|
191
|
+
out += segment.slice(i, i + 3);
|
|
192
|
+
i += 2;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
out += SAFE.test(ch) ? ch : encodeURIComponent(ch);
|
|
196
|
+
}
|
|
197
|
+
return out;
|
|
198
|
+
})
|
|
199
|
+
.join("/");
|
|
200
|
+
}
|
|
201
|
+
|
|
176
202
|
export function isMutating(method: string): boolean {
|
|
177
203
|
const m = method.toUpperCase();
|
|
178
204
|
return m === "POST" || m === "PUT" || m === "PATCH";
|
|
@@ -34,10 +34,8 @@
|
|
|
34
34
|
* categories, not contract bugs the API owner promised against.
|
|
35
35
|
*/
|
|
36
36
|
import type { OpenAPIV3, OpenAPIV3_1 } from "openapi-types";
|
|
37
|
-
import Ajv2020 from "ajv/dist/2020.js";
|
|
38
|
-
import Ajv from "ajv";
|
|
39
|
-
import addFormats from "ajv-formats";
|
|
40
37
|
import type { ValidateFunction, ErrorObject } from "ajv";
|
|
38
|
+
import { makeAjv } from "../util/ajv.ts";
|
|
41
39
|
|
|
42
40
|
interface SingleSchemaValidator {
|
|
43
41
|
validate(value: unknown): boolean;
|
|
@@ -48,10 +46,7 @@ interface SingleSchemaValidator {
|
|
|
48
46
|
* 3.1-flavour by default (`webhooks:` is OpenAPI 3.1) but tolerates
|
|
49
47
|
* pre-3.1 specs using `x-webhooks` — they ship Draft-7-ish schemas. */
|
|
50
48
|
function compileSingleSchema(schema: OpenAPIV3.SchemaObject, isV31: boolean): SingleSchemaValidator {
|
|
51
|
-
const ajv = isV31
|
|
52
|
-
? new (Ajv2020 as unknown as typeof Ajv)({ strict: false, allErrors: true })
|
|
53
|
-
: new Ajv({ strict: false, allErrors: true });
|
|
54
|
-
addFormats(ajv);
|
|
49
|
+
const ajv = makeAjv(isV31, { strict: false, allErrors: true });
|
|
55
50
|
const validate: ValidateFunction = ajv.compile(schema);
|
|
56
51
|
return {
|
|
57
52
|
validate(value) { return validate(value) as boolean; },
|
|
@@ -73,6 +68,9 @@ export interface WebhookFinding {
|
|
|
73
68
|
event_type: string | null;
|
|
74
69
|
message: string;
|
|
75
70
|
evidence: Record<string, unknown>;
|
|
71
|
+
/** Reserved suppression trace (audit-trail). Currently unset — the
|
|
72
|
+
* agent triages severity from the raw finding. */
|
|
73
|
+
suppressed_by?: { source: string; rule_index: number; reason: string };
|
|
76
74
|
}
|
|
77
75
|
|
|
78
76
|
export interface WebhookProbeResult {
|
|
@@ -30,7 +30,7 @@ function deepEquals(a: unknown, b: unknown): boolean {
|
|
|
30
30
|
if (typeof a === "string" && typeof b === "number") return Number(a) === b;
|
|
31
31
|
if (typeof a !== typeof b) return false;
|
|
32
32
|
if (typeof a !== "object" || a === null || b === null) return false;
|
|
33
|
-
return
|
|
33
|
+
return Bun.deepEquals(a, b);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
function checkRule(path: string, rule: AssertionRule, actual: unknown): AssertionResult[] {
|
|
@@ -9,15 +9,7 @@ import { evaluateExpr } from "./expr-eval.ts";
|
|
|
9
9
|
import { applyTransform } from "./transforms.ts";
|
|
10
10
|
import type { SchemaValidator } from "./schema-validator.ts";
|
|
11
11
|
import { classifyFailure } from "../diagnostics/failure-class.ts";
|
|
12
|
-
|
|
13
|
-
function buildUrl(baseUrl: string | undefined, path: string, query?: Record<string, string>): string {
|
|
14
|
-
let url = baseUrl ? `${baseUrl.replace(/\/+$/, "")}${path}` : path;
|
|
15
|
-
if (query && Object.keys(query).length > 0) {
|
|
16
|
-
const params = new URLSearchParams(query);
|
|
17
|
-
url += `?${params.toString()}`;
|
|
18
|
-
}
|
|
19
|
-
return url;
|
|
20
|
-
}
|
|
12
|
+
import { buildUrl } from "../util/url.ts";
|
|
21
13
|
|
|
22
14
|
/** Shallow-merge suite-level и step-level provenance. Step перекрывает suite. */
|
|
23
15
|
function mergeProvenance(
|
|
@@ -653,15 +645,8 @@ export async function runSuite(
|
|
|
653
645
|
passed: steps.filter((s) => s.status === "pass").length,
|
|
654
646
|
failed: steps.filter((s) => s.status === "fail").length,
|
|
655
647
|
skipped: steps.filter((s) => s.status === "skip").length,
|
|
648
|
+
// ARV-318: surface error steps so total = passed+failed+skipped+errored.
|
|
649
|
+
errored: steps.filter((s) => s.status === "error").length,
|
|
656
650
|
steps,
|
|
657
651
|
};
|
|
658
652
|
}
|
|
659
|
-
|
|
660
|
-
export async function runSuites(
|
|
661
|
-
suites: TestSuite[],
|
|
662
|
-
env: Environment = {},
|
|
663
|
-
dryRun = false,
|
|
664
|
-
options: RunSuiteOptions = {},
|
|
665
|
-
): Promise<TestRunResult[]> {
|
|
666
|
-
return Promise.all(suites.map((suite) => runSuite(suite, env, dryRun, options)));
|
|
667
|
-
}
|
|
@@ -29,23 +29,13 @@ export function encodeFormBody(body: Record<string, unknown>): string {
|
|
|
29
29
|
return params.toString();
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
/** Flatten a nested JS object to a `Record<string, string>` using
|
|
33
|
-
* notation
|
|
34
|
-
* as `Record<string, string>` and
|
|
32
|
+
/** Flatten a nested JS object to a `Record<string, string>` using the same
|
|
33
|
+
* bracket-notation walk as `encodeFormBody`, suitable for the YAML
|
|
34
|
+
* runner's `form:` step (which is typed as `Record<string, string>` and
|
|
35
|
+
* serialised via `URLSearchParams`). */
|
|
35
36
|
export function flattenToFormFields(body: unknown): Record<string, string> {
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
for (let i = 0; i < value.length; i++) walk(value[i], key ? `${key}[${i}]` : String(i));
|
|
41
|
-
} else if (typeof value === "object") {
|
|
42
|
-
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
43
|
-
walk(v, key ? `${key}[${k}]` : k);
|
|
44
|
-
}
|
|
45
|
-
} else if (key) {
|
|
46
|
-
out[key] = String(value);
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
if (body && typeof body === "object") walk(body, "");
|
|
50
|
-
return out;
|
|
37
|
+
if (!body || typeof body !== "object") return {};
|
|
38
|
+
const params = new URLSearchParams();
|
|
39
|
+
for (const [k, v] of Object.entries(body as Record<string, unknown>)) appendFormParam(params, k, v);
|
|
40
|
+
return Object.fromEntries(params.entries());
|
|
51
41
|
}
|
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
import type { HttpRequest, HttpResponse } from "./types.ts";
|
|
2
2
|
import { type RateLimiter, parseRetryAfter, parseRateLimitHeaders } from "./rate-limiter.ts";
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* ARV-265: module-level audit hook for commands that don't drive their
|
|
6
|
+
* HTTP through the suite runner. Each `executeRequest` call fires the
|
|
7
|
+
* registered recorder with the final outcome (or the swallowed network
|
|
8
|
+
* error if the call exhausted its retry budget). Recorders MUST NOT
|
|
9
|
+
* throw — the http client treats them as fire-and-forget telemetry.
|
|
10
|
+
*
|
|
11
|
+
* Convention: a command sets the recorder before the live work begins,
|
|
12
|
+
* unsets it in a `finally` block. There can be at most one recorder at
|
|
13
|
+
* a time; nesting is a bug (we surface it on stderr but keep the inner
|
|
14
|
+
* recorder so the live command's results take precedence).
|
|
15
|
+
*/
|
|
16
|
+
export interface AuditRecord {
|
|
17
|
+
request: HttpRequest;
|
|
18
|
+
response?: HttpResponse;
|
|
19
|
+
durationMs: number;
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
let _auditRecorder: ((rec: AuditRecord) => void) | null = null;
|
|
24
|
+
|
|
25
|
+
export function setHttpAuditRecorder(recorder: ((rec: AuditRecord) => void) | null): void {
|
|
26
|
+
if (recorder && _auditRecorder) {
|
|
27
|
+
process.stderr.write("zond: nested HTTP audit recorder — overriding the outer scope.\n");
|
|
28
|
+
}
|
|
29
|
+
_auditRecorder = recorder;
|
|
30
|
+
}
|
|
31
|
+
|
|
4
32
|
export interface FetchOptions {
|
|
5
33
|
timeout: number;
|
|
6
34
|
retries: number;
|
|
@@ -153,7 +181,7 @@ export async function executeRequest(
|
|
|
153
181
|
}
|
|
154
182
|
}
|
|
155
183
|
|
|
156
|
-
|
|
184
|
+
const httpResp: HttpResponse = {
|
|
157
185
|
status: response.status,
|
|
158
186
|
headers,
|
|
159
187
|
body: bodyText,
|
|
@@ -161,6 +189,11 @@ export async function executeRequest(
|
|
|
161
189
|
duration_ms,
|
|
162
190
|
network_retry_count: networkRetryCount,
|
|
163
191
|
};
|
|
192
|
+
if (_auditRecorder) {
|
|
193
|
+
try { _auditRecorder({ request, response: httpResp, durationMs: duration_ms }); }
|
|
194
|
+
catch { /* recorder is fire-and-forget */ }
|
|
195
|
+
}
|
|
196
|
+
return httpResp;
|
|
164
197
|
} catch (err) {
|
|
165
198
|
lastError = err instanceof Error ? err : new Error(String(err));
|
|
166
199
|
const isNet = isTransientNetworkError(lastError);
|
|
@@ -181,6 +214,10 @@ export async function executeRequest(
|
|
|
181
214
|
await Bun.sleep(opts.retry_delay);
|
|
182
215
|
continue;
|
|
183
216
|
}
|
|
217
|
+
if (_auditRecorder) {
|
|
218
|
+
try { _auditRecorder({ request, durationMs: 0, error: lastError.message }); }
|
|
219
|
+
catch { /* recorder is fire-and-forget */ }
|
|
220
|
+
}
|
|
184
221
|
throw lastError;
|
|
185
222
|
}
|
|
186
223
|
}
|
|
@@ -16,7 +16,7 @@ function scanRefs(value: unknown, out: Set<string>): void {
|
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
function collectCapturesAndSets(step: TestStep, out: Set<string>): void {
|
|
19
|
+
export function collectCapturesAndSets(step: TestStep, out: Set<string>): void {
|
|
20
20
|
if (step.set) {
|
|
21
21
|
for (const k of Object.keys(step.set)) out.add(k);
|
|
22
22
|
}
|
|
@@ -57,6 +57,23 @@ export interface MissingVarHit {
|
|
|
57
57
|
* forward references inside the suite are tolerated (correctness requires
|
|
58
58
|
* runtime ordering checks anyway).
|
|
59
59
|
*/
|
|
60
|
+
/** All non-generator {{var}} references a step reads (path, headers, body,
|
|
61
|
+
* query, skip_if, retry_until, set, for_each). Excludes `$generators`. */
|
|
62
|
+
export function collectStepRefs(step: TestStep): Set<string> {
|
|
63
|
+
const refs = new Set<string>();
|
|
64
|
+
scanRefs(step.path, refs);
|
|
65
|
+
scanRefs(step.headers, refs);
|
|
66
|
+
scanRefs(step.json, refs);
|
|
67
|
+
scanRefs(step.form, refs);
|
|
68
|
+
scanRefs(step.multipart, refs);
|
|
69
|
+
scanRefs(step.query, refs);
|
|
70
|
+
if (step.skip_if) scanRefs(step.skip_if, refs);
|
|
71
|
+
if (step.retry_until) scanRefs(step.retry_until.condition, refs);
|
|
72
|
+
if (step.set) scanRefs(step.set, refs);
|
|
73
|
+
if (step.for_each) scanRefs(step.for_each.in, refs);
|
|
74
|
+
return refs;
|
|
75
|
+
}
|
|
76
|
+
|
|
60
77
|
export function preflightCheckVars(
|
|
61
78
|
suites: TestSuite[],
|
|
62
79
|
env: Record<string, string>,
|
|
@@ -71,20 +88,7 @@ export function preflightCheckVars(
|
|
|
71
88
|
}
|
|
72
89
|
for (const step of suite.tests) collectCapturesAndSets(step, known);
|
|
73
90
|
|
|
74
|
-
const scanStepRefs =
|
|
75
|
-
const refs = new Set<string>();
|
|
76
|
-
scanRefs(step.path, refs);
|
|
77
|
-
scanRefs(step.headers, refs);
|
|
78
|
-
scanRefs(step.json, refs);
|
|
79
|
-
scanRefs(step.form, refs);
|
|
80
|
-
scanRefs(step.multipart, refs);
|
|
81
|
-
scanRefs(step.query, refs);
|
|
82
|
-
if (step.skip_if) scanRefs(step.skip_if, refs);
|
|
83
|
-
if (step.retry_until) scanRefs(step.retry_until.condition, refs);
|
|
84
|
-
if (step.set) scanRefs(step.set, refs);
|
|
85
|
-
if (step.for_each) scanRefs(step.for_each.in, refs);
|
|
86
|
-
return refs;
|
|
87
|
-
};
|
|
91
|
+
const scanStepRefs = collectStepRefs;
|
|
88
92
|
|
|
89
93
|
const suiteRefs = new Set<string>();
|
|
90
94
|
if (suite.base_url) scanRefs(suite.base_url, suiteRefs);
|
|
@@ -58,13 +58,19 @@ function applyResetPause(prevNextAvailable: number, meta: RateLimitMeta, now: nu
|
|
|
58
58
|
|
|
59
59
|
class IntervalRateLimiter implements RateLimiter {
|
|
60
60
|
private nextAvailable = 0;
|
|
61
|
+
/** 0 means adaptive: no proactive throttling until a policy is learned
|
|
62
|
+
* from a response (matches the original AdaptiveRateLimiter behaviour). */
|
|
61
63
|
private intervalMs: number;
|
|
62
64
|
|
|
63
|
-
constructor(reqPerSec
|
|
64
|
-
if (
|
|
65
|
-
|
|
65
|
+
constructor(reqPerSec?: number) {
|
|
66
|
+
if (reqPerSec === undefined) {
|
|
67
|
+
this.intervalMs = 0;
|
|
68
|
+
} else {
|
|
69
|
+
if (!Number.isFinite(reqPerSec) || reqPerSec <= 0) {
|
|
70
|
+
throw new Error(`Invalid rate limit: ${reqPerSec}`);
|
|
71
|
+
}
|
|
72
|
+
this.intervalMs = 1000 / reqPerSec;
|
|
66
73
|
}
|
|
67
|
-
this.intervalMs = 1000 / reqPerSec;
|
|
68
74
|
}
|
|
69
75
|
|
|
70
76
|
async acquire(): Promise<void> {
|
|
@@ -89,30 +95,6 @@ class IntervalRateLimiter implements RateLimiter {
|
|
|
89
95
|
}
|
|
90
96
|
}
|
|
91
97
|
|
|
92
|
-
class AdaptiveRateLimiter implements RateLimiter {
|
|
93
|
-
private nextAvailable = 0;
|
|
94
|
-
/** Learned from RateLimit-Policy. 0 until a policy is seen — until then,
|
|
95
|
-
* parallel acquires are not spaced (matches the original adaptive
|
|
96
|
-
* behaviour). Once known, every acquire reserves a slot of `intervalMs`. */
|
|
97
|
-
private intervalMs = 0;
|
|
98
|
-
|
|
99
|
-
async acquire(): Promise<void> {
|
|
100
|
-
const now = Date.now();
|
|
101
|
-
const slot = Math.max(now, this.nextAvailable);
|
|
102
|
-
const waitMs = slot - now;
|
|
103
|
-
this.nextAvailable = slot + this.intervalMs;
|
|
104
|
-
if (waitMs > 0) await Bun.sleep(waitMs);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
note(meta: RateLimitMeta, now: number = Date.now()): void {
|
|
108
|
-
if (meta.intervalMs !== undefined && meta.intervalMs > this.intervalMs) {
|
|
109
|
-
this.nextAvailable += meta.intervalMs - this.intervalMs;
|
|
110
|
-
this.intervalMs = meta.intervalMs;
|
|
111
|
-
}
|
|
112
|
-
this.nextAvailable = applyResetPause(this.nextAvailable, meta, now);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
98
|
export function createRateLimiter(reqPerSec: number | undefined): RateLimiter | undefined {
|
|
117
99
|
if (reqPerSec === undefined || reqPerSec === null) return undefined;
|
|
118
100
|
if (!Number.isFinite(reqPerSec) || reqPerSec <= 0) return undefined;
|
|
@@ -128,7 +110,7 @@ export function createRateLimiter(reqPerSec: number | undefined): RateLimiter |
|
|
|
128
110
|
* from blowing through small windows (e.g. 5-req/1-sec policies).
|
|
129
111
|
*/
|
|
130
112
|
export function createAdaptiveRateLimiter(): RateLimiter {
|
|
131
|
-
return new
|
|
113
|
+
return new IntervalRateLimiter();
|
|
132
114
|
}
|
|
133
115
|
|
|
134
116
|
/**
|
|
@@ -16,12 +16,18 @@
|
|
|
16
16
|
* - `check` — every suite path lives under `apis/<api>/checks/`. Mirrors
|
|
17
17
|
* the same logic: conformance checks don't reflect endpoint
|
|
18
18
|
* coverage breadth.
|
|
19
|
+
* - `request` — a single ad-hoc `zond request` invocation persisted into
|
|
20
|
+
* the session DB (ARV-265). Excluded from `pass-coverage`
|
|
21
|
+
* but counted toward `audit-coverage`.
|
|
22
|
+
* - `fixture` — `prepare-fixtures --cascade` discovery list-calls
|
|
23
|
+
* (ARV-265). Pure HTTP touches in service of fixture
|
|
24
|
+
* derivation; audit-coverage only.
|
|
19
25
|
* - `regular` — anything else, including mixed runs (probe + smoke). A
|
|
20
26
|
* mixed run is treated as regular because at least one
|
|
21
27
|
* suite contributed real coverage signal.
|
|
22
28
|
*/
|
|
23
29
|
|
|
24
|
-
export type RunKind = "regular" | "probe" | "check";
|
|
30
|
+
export type RunKind = "regular" | "probe" | "check" | "request" | "fixture";
|
|
25
31
|
|
|
26
32
|
const PROBE_SEGMENT_RE = /(^|\/)probes(\/|$)/;
|
|
27
33
|
const CHECK_SEGMENT_RE = /(^|\/)checks(\/|$)/;
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import Ajv2020 from "ajv/dist/2020.js";
|
|
2
1
|
import Ajv from "ajv";
|
|
3
|
-
import addFormats from "ajv-formats";
|
|
4
2
|
import type { ErrorObject, ValidateFunction, AnySchema } from "ajv";
|
|
5
3
|
import type { OpenAPIV3 } from "openapi-types";
|
|
4
|
+
import { makeAjv } from "../util/ajv.ts";
|
|
6
5
|
import { specPathToRegex } from "../generator/coverage-scanner.ts";
|
|
7
6
|
import type { AssertionResult } from "./types.ts";
|
|
8
7
|
|
|
@@ -32,10 +31,7 @@ export function createSchemaValidator(doc: OpenAPIV3.Document): SchemaValidator
|
|
|
32
31
|
// OpenAPI 3.1 → JSON Schema Draft 2020-12; 3.0 → Draft 4/7-ish.
|
|
33
32
|
// verbose:true exposes parentSchema on each error so humanize() can render
|
|
34
33
|
// the full required-set alongside the missing field name (TASK-277).
|
|
35
|
-
const ajv = isV31
|
|
36
|
-
? new (Ajv2020 as unknown as typeof Ajv)({ strict: false, allErrors: true, verbose: true })
|
|
37
|
-
: new Ajv({ strict: false, allErrors: true, verbose: true });
|
|
38
|
-
addFormats(ajv);
|
|
34
|
+
const ajv = makeAjv(isV31, { strict: false, allErrors: true, verbose: true });
|
|
39
35
|
applyStrictFormats(ajv);
|
|
40
36
|
|
|
41
37
|
const endpoints: EndpointEntry[] = [];
|
|
@@ -3,11 +3,7 @@ import { loadEnvironment, substituteString, substituteDeep } from "../parser/var
|
|
|
3
3
|
import { getDb } from "../../db/schema.ts";
|
|
4
4
|
import { findCollectionByNameOrId } from "../../db/queries.ts";
|
|
5
5
|
import { encodeFormBody } from "./form-encode.ts";
|
|
6
|
-
|
|
7
|
-
function hasHeaderCI(headers: Record<string, string>, name: string): boolean {
|
|
8
|
-
const lower = name.toLowerCase();
|
|
9
|
-
return Object.keys(headers).some((k) => k.toLowerCase() === lower);
|
|
10
|
-
}
|
|
6
|
+
import { hasHeaderCI } from "../util/headers.ts";
|
|
11
7
|
|
|
12
8
|
export function extractByPath(obj: unknown, path: string): unknown {
|
|
13
9
|
return extractByPathWithDiagnostic(obj, path).value;
|
|
@@ -31,9 +27,18 @@ export function extractByPathWithDiagnostic(
|
|
|
31
27
|
return { value: undefined, resolved, failedAt: seg, reason: "previous segment resolved to null/undefined" };
|
|
32
28
|
}
|
|
33
29
|
if (Array.isArray(current)) {
|
|
30
|
+
// ARV-207: `length` resolves to the array length (jq-style). This
|
|
31
|
+
// lets `--json-path data.items.length` answer "is the list non-empty?"
|
|
32
|
+
// in one call — the harvest flow needs it ('length>0 → has data;
|
|
33
|
+
// length==0 → seed') and previously had to shell out to jq.
|
|
34
|
+
if (seg === "length") {
|
|
35
|
+
current = current.length;
|
|
36
|
+
resolved.push(seg);
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
34
39
|
const idx = parseInt(seg, 10);
|
|
35
40
|
if (isNaN(idx)) {
|
|
36
|
-
return { value: undefined, resolved, failedAt: seg, reason: `expected an array index, got non-numeric segment "${seg}"` };
|
|
41
|
+
return { value: undefined, resolved, failedAt: seg, reason: `expected an array index, got non-numeric segment "${seg}" (hint: use 'length' for array size)` };
|
|
37
42
|
}
|
|
38
43
|
if (idx < 0 || idx >= current.length) {
|
|
39
44
|
return { value: undefined, resolved, failedAt: seg, reason: `array index ${idx} out of bounds (length ${current.length})` };
|
package/src/core/runner/types.ts
CHANGED
|
@@ -84,5 +84,11 @@ export interface TestRunResult {
|
|
|
84
84
|
passed: number;
|
|
85
85
|
failed: number;
|
|
86
86
|
skipped: number;
|
|
87
|
+
/** ARV-318: steps whose status is "error" (couldn't execute — env_issue,
|
|
88
|
+
* network, etc.). Previously counted in `total` but in none of
|
|
89
|
+
* passed/failed/skipped, so `total === passed+failed+skipped` silently
|
|
90
|
+
* broke and error steps reconciled into no bucket. Optional for back-compat
|
|
91
|
+
* with older result literals; the runner always sets it. */
|
|
92
|
+
errored?: number;
|
|
87
93
|
steps: StepResult[];
|
|
88
94
|
}
|
package/src/core/setup-api.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
serializeApiFixtureManifest,
|
|
15
15
|
} from "./generator/index.ts";
|
|
16
16
|
import { decycleSchema } from "./generator/schema-utils.ts";
|
|
17
|
+
import { mergeOpenApiDocs } from "./spec/merge-specs.ts";
|
|
17
18
|
import { schemeVarName } from "./generator/suite-generator.ts";
|
|
18
19
|
import type { SecuritySchemeInfo } from "./generator/types.ts";
|
|
19
20
|
import { hashSpec } from "./meta/meta-store.ts";
|
|
@@ -179,11 +180,17 @@ function toYaml(vars: Record<string, string>): string {
|
|
|
179
180
|
export interface SetupApiOptions {
|
|
180
181
|
name?: string;
|
|
181
182
|
spec?: string;
|
|
183
|
+
/** ARV-375: additional specs to union into one merged document. When
|
|
184
|
+
* set (length > 1), takes precedence over `spec`; the specs are
|
|
185
|
+
* dereferenced individually then merged deterministically (last-wins on
|
|
186
|
+
* path/component collision, collisions surfaced as warnings). */
|
|
187
|
+
specs?: string[];
|
|
182
188
|
dir?: string;
|
|
183
189
|
envVars?: Record<string, string>;
|
|
184
190
|
dbPath?: string;
|
|
185
191
|
force?: boolean;
|
|
186
192
|
insecure?: boolean;
|
|
193
|
+
caPath?: string;
|
|
187
194
|
}
|
|
188
195
|
|
|
189
196
|
export interface SetupApiResult {
|
|
@@ -221,7 +228,13 @@ function deriveAuthVarNames(schemes: SecuritySchemeInfo[]): string[] {
|
|
|
221
228
|
}
|
|
222
229
|
|
|
223
230
|
export async function setupApi(options: SetupApiOptions): Promise<SetupApiResult> {
|
|
224
|
-
const {
|
|
231
|
+
const { dbPath } = options;
|
|
232
|
+
// ARV-375: `--spec` may be repeated. `specs` (from the variadic CLI
|
|
233
|
+
// collector) wins; fall back to the single `spec` for back-compat.
|
|
234
|
+
const specList = options.specs && options.specs.length > 0
|
|
235
|
+
? options.specs
|
|
236
|
+
: (options.spec ? [options.spec] : []);
|
|
237
|
+
const spec = specList[0];
|
|
225
238
|
|
|
226
239
|
getDb(dbPath);
|
|
227
240
|
|
|
@@ -238,22 +251,47 @@ export async function setupApi(options: SetupApiOptions): Promise<SetupApiResult
|
|
|
238
251
|
// a self-contained file — no external $ref resolution at runtime.
|
|
239
252
|
let dereferencedDoc: unknown = null;
|
|
240
253
|
let authVarNames: string[] = [];
|
|
241
|
-
if (
|
|
242
|
-
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
+
if (specList.length > 0) {
|
|
255
|
+
// Read + validate each spec independently. dereference() happily
|
|
256
|
+
// round-trips arbitrary JSON (e.g. a marketing-site landing payload),
|
|
257
|
+
// so without this guard `zond add api foo --spec https://example.com`
|
|
258
|
+
// silently registers a 0-endpoint API.
|
|
259
|
+
const loaded: { source: string; doc: any }[] = [];
|
|
260
|
+
for (const s of specList) {
|
|
261
|
+
const d = await readOpenApiSpec(s, { insecure: options.insecure, caPath: options.caPath });
|
|
262
|
+
const dAny = d as any;
|
|
263
|
+
if (typeof dAny?.openapi !== "string" && typeof dAny?.swagger !== "string") {
|
|
264
|
+
throw new Error(
|
|
265
|
+
`Spec at ${s} is not an OpenAPI/Swagger document — missing top-level 'openapi' (3.x) or 'swagger' (2.x) field. Check the URL points to the JSON spec, not the API root.`,
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
loaded.push({ source: s, doc: d });
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
let doc: any;
|
|
272
|
+
if (loaded.length === 1) {
|
|
273
|
+
doc = loaded[0]!.doc;
|
|
274
|
+
openapiSpec = spec!;
|
|
275
|
+
} else {
|
|
276
|
+
// ARV-375: deterministic union of N specs into one audit target.
|
|
277
|
+
const { merged, summary } = mergeOpenApiDocs(loaded);
|
|
278
|
+
doc = merged;
|
|
279
|
+
openapiSpec = null; // no single source path — rely on the local snapshot
|
|
280
|
+
const per = summary.sources.map((s) => `${s.paths} from ${s.source}`).join(", ");
|
|
281
|
+
warnings.push(`Merged ${summary.sources.length} specs → ${summary.totalPaths} paths (${per}).`);
|
|
282
|
+
if (summary.pathCollisions.length > 0) {
|
|
283
|
+
warnings.push(
|
|
284
|
+
`${summary.pathCollisions.length} path collision(s) — later spec wins: ${summary.pathCollisions.join(", ")}.`,
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
if (summary.schemaConflicts.length > 0) {
|
|
288
|
+
warnings.push(
|
|
289
|
+
`${summary.schemaConflicts.length} component collision(s) with differing shapes (later wins, verify): ${summary.schemaConflicts.join(", ")}.`,
|
|
290
|
+
);
|
|
291
|
+
}
|
|
254
292
|
}
|
|
255
293
|
dereferencedDoc = doc;
|
|
256
|
-
|
|
294
|
+
const docAny = doc as any;
|
|
257
295
|
if ((doc as any).servers?.[0]?.url) {
|
|
258
296
|
baseUrl = (doc as any).servers[0].url;
|
|
259
297
|
// Resolve OpenAPI server variables (e.g. {region}) using their declared defaults.
|
|
@@ -13,14 +13,6 @@
|
|
|
13
13
|
|
|
14
14
|
export type Severity = "critical" | "high" | "medium" | "low" | "info";
|
|
15
15
|
|
|
16
|
-
export const SEVERITY_ORDER: readonly Severity[] = [
|
|
17
|
-
"critical",
|
|
18
|
-
"high",
|
|
19
|
-
"medium",
|
|
20
|
-
"low",
|
|
21
|
-
"info",
|
|
22
|
-
] as const;
|
|
23
|
-
|
|
24
16
|
const SEVERITY_RANK: Record<Severity, number> = {
|
|
25
17
|
critical: 0,
|
|
26
18
|
high: 1,
|
|
@@ -29,65 +21,10 @@ const SEVERITY_RANK: Record<Severity, number> = {
|
|
|
29
21
|
info: 4,
|
|
30
22
|
};
|
|
31
23
|
|
|
32
|
-
/**
|
|
33
|
-
* Strength of evidence backing a finding. Drives the severity cap
|
|
34
|
-
* applied at finding-emission time.
|
|
35
|
-
*
|
|
36
|
-
* - `end_to_end`: zond demonstrated the impact itself (read another
|
|
37
|
-
* user's data, executed action without auth, file read confirmed).
|
|
38
|
-
* Required for CRITICAL.
|
|
39
|
-
* - `evidence_chain`: ≥2 requests prove the finding (storage +
|
|
40
|
-
* reflection found, follow-up GET shows persistence, OOB callback
|
|
41
|
-
* received). Required for HIGH.
|
|
42
|
-
* - `single_signal`: one request/response indicates an anomaly but
|
|
43
|
-
* no follow-up confirms impact (server accepted CRLF / 169.254 /
|
|
44
|
-
* is_admin field — outcome unknown). Capped at LOW.
|
|
45
|
-
* - `static`: spec-lint, style, naming, missing additionalProperties.
|
|
46
|
-
* No runtime evidence. Capped at INFO.
|
|
47
|
-
*/
|
|
48
|
-
export type ProofKind = "end_to_end" | "evidence_chain" | "single_signal" | "static";
|
|
49
|
-
|
|
50
|
-
const PROOF_CAP: Record<ProofKind, Severity> = {
|
|
51
|
-
end_to_end: "critical",
|
|
52
|
-
evidence_chain: "high",
|
|
53
|
-
single_signal: "low",
|
|
54
|
-
static: "info",
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Caps a claimed severity by the strength of evidence behind it.
|
|
59
|
-
* Producers should pass their natural severity claim and the proof
|
|
60
|
-
* kind; the cap function downgrades if claim exceeds what evidence
|
|
61
|
-
* supports.
|
|
62
|
-
*
|
|
63
|
-
* Example: mass-assignment probe wants HIGH (dangerous field), but
|
|
64
|
-
* only has single-signal proof (server returned 200, didn't verify
|
|
65
|
-
* persistence). Cap returns LOW. To get HIGH, probe must escalate
|
|
66
|
-
* proof to evidence_chain by doing follow-up GET.
|
|
67
|
-
*/
|
|
68
|
-
export function capSeverityByProof(claim: Severity, proof: ProofKind): Severity {
|
|
69
|
-
const cap = PROOF_CAP[proof];
|
|
70
|
-
return rankSeverity(claim) < rankSeverity(cap) ? cap : claim;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
24
|
export function rankSeverity(s: Severity): number {
|
|
74
25
|
return SEVERITY_RANK[s];
|
|
75
26
|
}
|
|
76
27
|
|
|
77
|
-
/** True iff `a` is at least as severe as `b`. */
|
|
78
|
-
export function isAtLeast(a: Severity, b: Severity): boolean {
|
|
79
|
-
return rankSeverity(a) <= rankSeverity(b);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/** Highest severity among inputs; returns 'info' on empty list. */
|
|
83
|
-
export function maxSeverity(items: readonly Severity[]): Severity {
|
|
84
|
-
let best: Severity = "info";
|
|
85
|
-
for (const s of items) {
|
|
86
|
-
if (rankSeverity(s) < rankSeverity(best)) best = s;
|
|
87
|
-
}
|
|
88
|
-
return best;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
28
|
/**
|
|
92
29
|
* Empty severity-bucket map. Use as starting tally; downstream code
|
|
93
30
|
* increments per finding.
|