@kirrosh/zond 0.22.0 → 0.26.0

package/src/core/checks/checks/cursor_boundary_fuzzing.ts

@@ -0,0 +1,219 @@
+/**
+ * `cursor_boundary_fuzzing` (ARV-273) — fuzz cursor/page-token style
+ * query parameters on list endpoints with malformed values. The server
+ * is expected to reject with 400/422 (or 401/403 if the path is auth-
+ * gated). A 5xx response on *any* well-formed cursor value is a real
+ * bug (Stripe `/v1/billing/alerts` was found this way during the m-22
+ * Stripe scan).
+ *
+ * Detection rule: parameter `name` matches a small set of conventional
+ * cursor names (cursor / starting_after / ending_before / after /
+ * before / page_token / next_token / continuation), parameter is
+ * `in: "query"`, and schema type is string (or untyped — many SDKs
+ * leave the cursor schema open).
+ *
+ * Mutation vectors (7): empty string, numeric, "null", very-long
+ * string (200 chars), valid-shape-wrong-resource (Stripe-id), SQL-
+ * shaped (`' OR 1=1--`), JSON-shaped (`{"foo":"bar"}`).
+ *
+ * Severity ladder:
+ *   - any 5xx → HIGH (server should never crash on bad cursor)
+ *   - any 2xx/204 → LOW (server silently tolerates a malformed cursor)
+ *   - 4xx other than 4xx-expected → no finding (server rejected — good)
+ *   - 401/403 only → skip (auth-gated, not the check's concern)
+ *
+ * Why not done by `negative_data_rejection`: that check mutates one
+ * value per case based on the declared schema (string → integer,
+ * integer overflow, etc.). Cursor fuzzing is a *cross-API convention*
+ * fuzz family that does not derive from schema type; we know these
+ * are cursor-shaped strings and we know the standard malformations
+ * that crash naive cursor parsers.
+ */
+import type { OpenAPIV3 } from "openapi-types";
+import type { CrudStatefulCheck } from "../stateful.ts";
+import type { CheckOutcome, Severity } from "../types.ts";
+import { fillPathParams } from "./_crud-helpers.ts";
+import { buildUrl as buildUrlBase } from "../../util/url.ts";
+import { MAX_REQUESTS_SKIP_REASON } from "../../runner/executor.ts";
+
+const CURSOR_PARAM_NAME_RE = /^(cursor|starting_after|ending_before|after|before|page_token|next_token|continuation)$/i;
+
+interface MutationVector {
+  label: string;
+  value: string;
+}
+
+const MUTATION_VECTORS: readonly MutationVector[] = [
+  { label: "empty", value: "" },
+  { label: "numeric", value: "12345" },
+  { label: "null-literal", value: "null" },
+  { label: "very-long", value: "a".repeat(200) },
+  { label: "wrong-resource-id", value: "cus_invalid_zzz" },
+  { label: "sql-shaped", value: "' OR 1=1--" },
+  { label: "json-shaped", value: '{"foo":"bar"}' },
+];
+
+function isCursorParam(p: OpenAPIV3.ParameterObject): boolean {
+  if (p.in !== "query") return false;
+  if (!CURSOR_PARAM_NAME_RE.test(p.name)) return false;
+  const schema = p.schema as OpenAPIV3.SchemaObject | undefined;
+  if (!schema) return true;
+  // Accept untyped schemas (many specs leave cursor open) and string.
+  if (!schema.type) return true;
+  return schema.type === "string";
+}
+
+function buildUrl(
+  base: string,
+  path: string,
+  pathVars: Record<string, string> | undefined,
+  cursorParam: string,
+  cursorValue: string,
+): string {
+  return buildUrlBase(base, fillPathParams(path, pathVars), { [cursorParam]: cursorValue });
+}
+
+interface VectorOutcome {
+  param: string;
+  vector: string;
+  status: number;
+  body_excerpt?: string;
+}
+
+function excerpt(s: string | undefined, n: number): string | undefined {
+  if (!s) return undefined;
+  const trimmed = s.length > n ? s.slice(0, n) + "…" : s;
+  return trimmed;
+}
+
+export const cursorBoundaryFuzzing: CrudStatefulCheck = {
+  id: "cursor_boundary_fuzzing",
+  // Per-finding severity overrides via outcome.severity: 5xx → high, 2xx → low.
+  // Declared baseline 'low' so it doesn't dominate per-check gating tables.
+  severity: "low",
+  defaultExpected:
+    "Cursor-style query params (cursor / starting_after / page_token / …) must reject malformed values with 4xx, never 5xx",
+  references: [{ id: "ARV-273" }],
+  phase: "crud",
+  applies(g) {
+    if (!g.list) return false;
+    // ARV-310: cursor fuzzing is only meaningful on a paginating GET list.
+    // A create/mutation endpoint that happens to carry a cursor-shaped query
+    // param (or a mis-grouped POST bound as `list`) must not trip this check.
+    if (g.list.method.toUpperCase() !== "GET") return false;
+    return g.list.parameters.some(isCursorParam);
+  },
+  async run(g, h): Promise<CheckOutcome> {
+    if (h.bootstrapCleanupFailed) {
+      return { kind: "skip", reason: "bootstrap-cleanup failed — stateful checks paused" };
+    }
+    const list = g.list!;
+    const cursorParams = list.parameters.filter(isCursorParam);
+    if (cursorParams.length === 0) {
+      return { kind: "skip", reason: "no cursor-style query params on list endpoint" };
+    }
+
+    const baseHeaders = { Accept: "application/json", ...h.authHeaders };
+    const serverErrors: VectorOutcome[] = [];
+    const silentAccepts: VectorOutcome[] = [];
+    let totalAttempted = 0;
+    let allAuthGated = true;
+
+    let budgetExhausted = false;
+    outer: for (const param of cursorParams) {
+      for (const vec of MUTATION_VECTORS) {
+        const url = buildUrl(h.baseUrl, list.path, h.pathVars, param.name, vec.value);
+        let resp;
+        try {
+          resp = await h.send({ method: "GET", url, headers: baseHeaders });
+        } catch (err) {
+          // ARV-308: distinguish budget exhaustion (every subsequent
+          // mutation will throw the same MAX_REQUESTS_SKIP_REASON and
+          // wasn't dispatched on the wire) from real per-request
+          // network errors (genuine connection problems, where we
+          // should keep fuzzing the remaining vectors). The earlier
+          // `catch {}` swallowed both as "network errors", which
+          // misreported the run on Stripe — budget had simply run out
+          // before this check got its turn, but the summary blamed
+          // the network and the 500 finding never surfaced.
+          if (err instanceof Error && err.message === MAX_REQUESTS_SKIP_REASON) {
+            budgetExhausted = true;
+            break outer;
+          }
+          continue;
+        }
+        totalAttempted += 1;
+        const status = resp.status;
+        if (status !== 401 && status !== 403) allAuthGated = false;
+        if (status >= 500 && status < 600) {
+          serverErrors.push({
+            param: param.name,
+            vector: vec.label,
+            status,
+            body_excerpt: excerpt(resp.body, 240),
+          });
+        } else if (status >= 200 && status < 300) {
+          silentAccepts.push({
+            param: param.name,
+            vector: vec.label,
+            status,
+          });
+        }
+      }
+    }
+
+    if (totalAttempted === 0) {
+      // ARV-308: dedicated reason for budget exhaustion so the run
+      // summary shows the real cause (and an actionable fix — raise
+      // --max-requests / --budget) instead of blaming the network.
+      if (budgetExhausted) {
+        return { kind: "skip", reason: MAX_REQUESTS_SKIP_REASON };
+      }
+      return { kind: "skip", reason: "no mutations dispatched (network errors on every probe)" };
+    }
+    if (allAuthGated) {
+      return { kind: "skip", reason: "endpoint auth-gated for all probes (401/403 only)" };
+    }
+    if (serverErrors.length > 0) {
+      const sev: Severity = "high";
+      const first = serverErrors[0]!;
+      return {
+        kind: "fail",
+        severity: sev,
+        // ARV-310: attribute to the GET list actually probed, not the group's
+        // POST create.
+        operation: { path: list.path, method: "GET", operationId: list.operationId },
+        message:
+          `Server returned ${first.status} on ${first.vector} cursor (${first.param}) — ${serverErrors.length}/${totalAttempted} mutation(s) hit 5xx`,
+        evidence: {
+          resource: g.resource,
+          list_path: list.path,
+          kind: "server_error_on_bad_cursor",
+          cursor_params: cursorParams.map((p) => p.name),
+          attempted: totalAttempted,
+          server_errors: serverErrors,
+        },
+      };
+    }
+    if (silentAccepts.length > 0) {
+      return {
+        kind: "fail",
+        severity: "low",
+        // ARV-310: attribute to the GET list actually probed, not the group's
+        // POST create.
+        operation: { path: list.path, method: "GET", operationId: list.operationId },
+        message:
+          `Server returned 2xx on ${silentAccepts.length}/${totalAttempted} malformed cursor mutation(s) — likely silent tolerance of bad input`,
+        evidence: {
+          resource: g.resource,
+          list_path: list.path,
+          kind: "silent_accept_on_bad_cursor",
+          cursor_params: cursorParams.map((p) => p.name),
+          attempted: totalAttempted,
+          accepts: silentAccepts,
+        },
+      };
+    }
+    return { kind: "pass" };
+  },
+};

package/src/core/checks/checks/ensure_resource_availability.ts

@@ -0,0 +1,62 @@
+/**
+ * `ensure_resource_availability` (m-15 ARV-3) — create a resource via
+ * POST, then GET by id; the read must succeed (2xx). Catches lost-
+ * write bugs where the create returns 201 but the resource never
+ * actually appears in storage.
+ */
+import type { CrudStatefulCheck } from "../stateful.ts";
+import { extractIdFromCreateResponse, fillPathWithId, fillPathParams, serializeCheckBody, resolveCreateBody } from "./_crud-helpers.ts";
+
+export const ensureResourceAvailability: CrudStatefulCheck = {
+  id: "ensure_resource_availability",
+  severity: "medium",
+  defaultExpected: "GET on a freshly-created resource must succeed (2xx)",
+  references: [{ id: "CWE-924" }],
+  phase: "crud",
+  applies(g) {
+    return Boolean(g.create && g.read);
+  },
+  async run(g, h) {
+    if (h.bootstrapCleanupFailed) {
+      return { kind: "skip", reason: "bootstrap-cleanup failed — security checks paused (ARV-3 AC #6)" };
+    }
+    const create = g.create!;
+    const read = g.read!;
+    const baseHeaders = { Accept: "application/json", ...h.authHeaders };
+    // ARV-191: form-urlencoded vs JSON dispatch — Stripe-style APIs
+    // declare x-www-form-urlencoded; JSON.stringify would yield "400
+    // missing param" the broken-baseline guard then silently swallows.
+    // ARV-187: prefer LLM-authored seed_body over generator.
+    const seedBody = h.resourceConfigs?.get(g.resource)?.seedBody;
+    const generated = resolveCreateBody(create, seedBody) ?? {};
+    const { body, contentType } = serializeCheckBody(
+      create,
+      generated,
+      h.pathVars,
+      seedBody?.contentType,
+    );
+    const createResp = await h.send({
+      method: "POST",
+      url: `${h.baseUrl.replace(/\/+$/, "")}${fillPathParams(create.path, h.pathVars)}`,
+      headers: { ...baseHeaders, "Content-Type": contentType },
+      body,
+    });
+    if (createResp.status < 200 || createResp.status >= 300) {
+      return { kind: "skip", reason: `create returned ${createResp.status} — broken-baseline guard` };
+    }
+    const id = extractIdFromCreateResponse(createResp.body_parsed ?? createResp.body, g.idParam);
+    if (id == null) return { kind: "skip", reason: "could not extract id from create response" };
+
+    const readResp = await h.send({
+      method: "GET",
+      url: `${h.baseUrl.replace(/\/+$/, "")}${fillPathWithId(fillPathParams(read.path, h.pathVars), g.idParam, id)}`,
+      headers: baseHeaders,
+    });
+    if (readResp.status >= 200 && readResp.status < 300) return { kind: "pass" };
+    return {
+      kind: "fail",
+      message: `GET on freshly-created resource ${id} returned ${readResp.status}`,
+      evidence: { resource: g.resource, id, create_status: createResp.status, read_status: readResp.status },
+    };
+  },
+};

package/src/core/checks/checks/idempotency_replay.ts

@@ -0,0 +1,242 @@
+/**
+ * `idempotency_replay` (m-20 ARV-170) — Idempotency-Key honor probe.
+ *
+ * For each CRUD group with create+delete where idempotency is opted-in
+ * (either via `.api-resources.yaml` `idempotency:` block or by the
+ * create endpoint declaring an `Idempotency-Key` header parameter),
+ * POST the same body twice with the same key. The server must:
+ *
+ *   1. return the *same* resource id on both calls (id1 == id2 — no
+ *      duplicate created), AND
+ *   2. return bit-identical responses modulo a small allow-list of
+ *      timestamp / request-id fields (R1 == R2).
+ *
+ * Severity policy: HIGH. The two failure classes share one finding —
+ * the runner doesn't support per-finding severity downgrade and
+ * `duplicate_resource` is the dominant signal anyway. `non_bit_identical`
+ * piggybacks via finding.evidence.kind so consumers can split the
+ * digest if they care.
+ *
+ * Anti-FP guards:
+ *   • Skip when the second POST gets a 429 / 409 / 5xx — replay rate
+ *     limiting and locking races confuse the verdict.
+ *   • Skip when either POST fails the broken-baseline check (non-2xx).
+ *   • Cleanup tolerates missing DELETE wiring — emits a warning via
+ *     `cleanup_warning` in evidence.
+ *
+ * Auto-detect fallback: if no yaml block exists but the create endpoint
+ * declares an `Idempotency-Key` header in its `parameters[]`, the check
+ * runs with default settings. Explicit yaml wins — it lets the user
+ * override the header name and customize the ignore list.
+ */
+import type { OpenAPIV3 } from "openapi-types";
+import type { CrudStatefulCheck } from "../stateful.ts";
+import type { IdempotencyConfig } from "../../generator/resources-builder.ts";
+import { extractIdFromCreateResponse, fillPathWithId, fillPathParams, serializeCheckBody, resolveCreateBody } from "./_crud-helpers.ts";
+
+/** Default header name used when yaml omits it and we're running on
+ *  spec-detected idempotency support. Matches the Stripe / Resend
+ *  convention; specs that use a different casing should declare it
+ *  explicitly. */
+const DEFAULT_HEADER = "Idempotency-Key";
+
+/** Baseline response fields stripped before bit-identical compare.
+ *  Mirrors the readback-diff baseline minus a few read-shape-specific
+ *  ones (livemode, _links). Timestamps + request-id + etag cover the
+ *  common "every replay has a new request_id" surface. */
+const DEFAULT_IGNORE_RESPONSE: ReadonlySet<string> = new Set([
+  "created",
+  "created_at",
+  "createdAt",
+  "updated",
+  "updated_at",
+  "updatedAt",
+  "request_id",
+  "requestId",
+  "x_request_id",
+  "etag",
+  "_etag",
+]);
+
+function safeParse(v: unknown): unknown {
+  if (typeof v !== "string") return v;
+  try {
+    return JSON.parse(v);
+  } catch {
+    return v;
+  }
+}
+
+function detectSpecHeader(create: { parameters: OpenAPIV3.ParameterObject[] }): string | null {
+  for (const p of create.parameters) {
+    if (p.in !== "header") continue;
+    if (p.name.toLowerCase() === "idempotency-key") return p.name;
+  }
+  return null;
+}
+
+function resolveConfig(
+  cfg: IdempotencyConfig | undefined,
+  create: { parameters: OpenAPIV3.ParameterObject[] },
+): { header: string; ignore: ReadonlySet<string> } | null {
+  if (cfg) {
+    const header = cfg.header ?? DEFAULT_HEADER;
+    const ignore = cfg.ignoreResponseFields
+      ? new Set<string>([...DEFAULT_IGNORE_RESPONSE, ...cfg.ignoreResponseFields])
+      : DEFAULT_IGNORE_RESPONSE;
+    return { header, ignore };
+  }
+  const detected = detectSpecHeader(create);
+  if (detected) return { header: detected, ignore: DEFAULT_IGNORE_RESPONSE };
+  return null;
+}
+
+/** Shallow object diff with field-level ignore. Returns the list of
+ *  keys whose values differ (or whose presence differs) between a and
+ *  b, excluding ignored fields. Both inputs treated as `{}` when not
+ *  object-shaped. */
+function diffFields(a: unknown, b: unknown, ignore: ReadonlySet<string>): string[] {
+  const av = (a && typeof a === "object" && !Array.isArray(a)) ? (a as Record<string, unknown>) : {};
+  const bv = (b && typeof b === "object" && !Array.isArray(b)) ? (b as Record<string, unknown>) : {};
+  const keys = new Set<string>([...Object.keys(av), ...Object.keys(bv)]);
+  const diffs: string[] = [];
+  for (const k of keys) {
+    if (ignore.has(k)) continue;
+    const sa = JSON.stringify(av[k] ?? null);
+    const sb = JSON.stringify(bv[k] ?? null);
+    if (sa !== sb) diffs.push(k);
+  }
+  return diffs;
+}
+
+function generateKey(): string {
+  return crypto.randomUUID();
+}
+
+export const idempotencyReplay: CrudStatefulCheck = {
+  id: "idempotency_replay",
+  severity: "high",
+  defaultExpected: "Two POSTs with the same Idempotency-Key must return the same resource id and bit-identical responses",
+  references: [{ id: "ARV-170" }, { id: "stripe-idempotent-requests" }],
+  phase: "crud",
+  applies(g) {
+    return Boolean(g.create);
+  },
+  async run(g, h) {
+    if (h.bootstrapCleanupFailed) {
+      return { kind: "skip", reason: "bootstrap-cleanup failed — stateful checks paused" };
+    }
+    const create = g.create!;
+
+    const cfg = h.resourceConfigs?.get(g.resource)?.idempotency;
+    const resolved = resolveConfig(cfg, create);
+    if (!resolved) {
+      return { kind: "skip", reason: "no idempotency config and no Idempotency-Key parameter in spec" };
+    }
+
+    const seedBody = h.resourceConfigs?.get(g.resource)?.seedBody;
+    if (!create.requestBodySchema && !seedBody) {
+      return { kind: "skip", reason: "create has no requestBody schema and no seed_body — nothing to replay" };
+    }
+    const writeBody = resolveCreateBody(create, seedBody);
+    if (writeBody == null) {
+      return { kind: "skip", reason: "could not produce a create body (no seed_body, generator returned non-object)" };
+    }
+
+    const key = generateKey();
+    // ARV-191: form-urlencoded vs JSON dispatch — Stripe-style APIs
+    // honor Idempotency-Key but expect x-www-form-urlencoded payloads;
+    // JSON.stringify would yield broken-baseline 400s on every replay.
+    const { body: bodyStr, contentType } = serializeCheckBody(create, writeBody, h.pathVars, seedBody?.contentType);
+    const baseHeaders = {
+      Accept: "application/json",
+      "Content-Type": contentType,
+      [resolved.header]: key,
+      ...h.authHeaders,
+    };
+    const url = `${h.baseUrl.replace(/\/+$/, "")}${fillPathParams(create.path, h.pathVars)}`;
+
+    // 1st POST
+    const r1 = await h.send({ method: "POST", url, headers: baseHeaders, body: bodyStr });
+    if (r1.status < 200 || r1.status >= 300) {
+      return { kind: "skip", reason: `1st create returned ${r1.status} — broken-baseline guard` };
+    }
+    const body1 = r1.body_parsed ?? safeParse(r1.body);
+    const id1 = extractIdFromCreateResponse(body1, g.idParam);
+    if (id1 == null) return { kind: "skip", reason: "could not extract id from 1st create response" };
+
+    // 2nd POST — same body, same key
+    const r2 = await h.send({ method: "POST", url, headers: baseHeaders, body: bodyStr });
+    if (r2.status === 429 || r2.status === 409) {
+      // Cleanup r1 before bailing — we did create something.
+      await tryCleanup(g, h, id1);
+      return { kind: "skip", reason: `2nd create returned ${r2.status} — rate-limit/conflict, replay verdict ambiguous` };
+    }
+    if (r2.status < 200 || r2.status >= 300) {
+      await tryCleanup(g, h, id1);
+      return { kind: "skip", reason: `2nd create returned ${r2.status} — broken-baseline guard` };
+    }
+    const body2 = r2.body_parsed ?? safeParse(r2.body);
+    const id2 = extractIdFromCreateResponse(body2, g.idParam);
+
+    // Verdict
+    const duplicate = id2 != null && String(id1) !== String(id2);
+    const diffs = duplicate ? [] : diffFields(body1, body2, resolved.ignore);
+    const nonBitIdentical = !duplicate && diffs.length > 0;
+
+    // Cleanup. If duplicate, both ids need to go.
+    const cleanupWarn: string[] = [];
+    const okCleanup1 = await tryCleanup(g, h, id1);
+    if (!okCleanup1) cleanupWarn.push(`failed to DELETE id=${id1}`);
+    if (duplicate && id2 != null) {
+      const okCleanup2 = await tryCleanup(g, h, id2);
+      if (!okCleanup2) cleanupWarn.push(`failed to DELETE id=${id2}`);
+    }
+
+    if (!duplicate && !nonBitIdentical) {
+      return { kind: "pass" };
+    }
+
+    const kind = duplicate && nonBitIdentical
+      ? "both"
+      : duplicate ? "duplicate_resource" : "non_bit_identical";
+    const message = duplicate
+      ? `Idempotency-Key not honored on ${g.resource}: replay produced a new resource (id1=${id1}, id2=${id2})`
+      : `Idempotency-Key replay on ${g.resource} is not bit-identical (${diffs.length} field(s) differ): ${diffs.slice(0, 5).join(", ")}`;
+
+    return {
+      kind: "fail",
+      message,
+      evidence: {
+        resource: g.resource,
+        kind,
+        header: resolved.header,
+        key,
+        id1,
+        id2,
+        diff_fields: diffs,
+        ...(cleanupWarn.length > 0 ? { cleanup_warning: cleanupWarn } : {}),
+      },
+    };
+  },
+};
+
+async function tryCleanup(
+  g: { delete?: { path: string }; idParam: string },
+  h: import("../stateful.ts").StatefulHarness,
+  id: string | number,
+): Promise<boolean> {
+  if (!g.delete) return false;
+  const url = `${h.baseUrl.replace(/\/+$/, "")}${fillPathWithId(fillPathParams(g.delete.path, h.pathVars), g.idParam, id)}`;
+  try {
+    const resp = await h.send({
+      method: "DELETE",
+      url,
+      headers: { Accept: "application/json", ...h.authHeaders },
+    });
+    // 404 = already gone (good); 2xx = deleted; anything else = failure.
+    return resp.status === 404 || (resp.status >= 200 && resp.status < 300);
+  } catch {
+    return false;
+  }
+}