@kirrosh/zond 0.22.0 → 0.23.0

package/src/core/checks/sarif.ts

@@ -0,0 +1,230 @@
+/**
+ * SARIF v2.1.0 reporter for `zond checks` (m-15 ARV-5).
+ *
+ * Maps `CheckFinding[]` into a SARIF log that GitHub Code Scanning can
+ * ingest via `github/codeql-action/upload-sarif@v3`. Two invariants the
+ * format relies on:
+ *
+ *   - `tool.driver.rules` — descriptors for every registered check, so
+ *     even a finding-less run carries the catalog. ruleId follows the
+ *     `<category>-<check_id>` form that oasdiff uses.
+ *   - `partialFingerprints.primary` — sha1(ruleId + jsonPointer +
+ *     spec_hash). Stable across re-runs of the same spec, so GitHub
+ *     dedupes rather than re-opening alerts every push (42Crunch-style).
+ *
+ * The reporter is deliberately schema-only — it builds the JSON
+ * document, the CLI handles writing it to disk.
+ */
+import { createHash } from "node:crypto";
+
+import { listChecks } from "./registry.ts";
+import { listStatefulChecks } from "./stateful.ts";
+import type { CheckFinding, Severity } from "./types.ts";
+import { categoryFor, type Category } from "../severity/category.ts";
+import { severityToSarifLevel } from "../severity/index.ts";
+
+export { categoryFor };
+
+export function ruleIdFor(checkId: string): string {
+  return `${categoryFor(checkId)}-${checkId}`;
+}
+
+const severityToLevel = severityToSarifLevel;
+
+/** RFC 6901 JSON Pointer for the operation: `/paths/<escaped>/<method>`.
+ *  Escapes `~` → `~0` and `/` → `~1` so paths like `/users/{id}` survive
+ *  serialization. */
+export function jsonPointerForOperation(path: string, method: string): string {
+  const escaped = path.replace(/~/g, "~0").replace(/\//g, "~1");
+  return `/paths/${escaped}/${method.toLowerCase()}`;
+}
+
+function sha1(s: string): string {
+  return createHash("sha1").update(s).digest("hex");
+}
+
+export function specHashOf(specContent: string): string {
+  return sha1(specContent);
+}
+
+export function partialFingerprintFor(ruleId: string, jsonPointer: string, specHash: string): string {
+  return sha1(`${ruleId}\n${jsonPointer}\n${specHash}`);
+}
+
+interface SarifReportingDescriptor {
+  id: string;
+  name: string;
+  shortDescription: { text: string };
+  defaultConfiguration: { level: "error" | "warning" | "note" };
+  helpUri?: string;
+  properties: {
+    category: Category;
+    severity: Severity;
+    references: string[];
+    tags: string[];
+  };
+}
+
+interface SarifResult {
+  ruleId: string;
+  ruleIndex: number;
+  level: "error" | "warning" | "note";
+  message: { text: string };
+  locations: Array<{
+    physicalLocation: {
+      artifactLocation: { uri: string; uriBaseId?: string };
+      // SARIF requires region to specify at least one of startLine/charOffset/
+      // byteOffset. We don't parse the spec into lines — startLine: 1 keeps
+      // GitHub Code Scanning happy and the JSON Pointer travels in the
+      // logicalLocations + properties below.
+      region: { startLine: number; snippet: { text: string } };
+    };
+    logicalLocations: Array<{ fullyQualifiedName: string; kind: string }>;
+  }>;
+  partialFingerprints: { primary: string };
+  properties: Record<string, unknown>;
+}
+
+export interface SarifLog {
+  $schema: string;
+  version: "2.1.0";
+  runs: Array<{
+    tool: {
+      driver: {
+        name: string;
+        version: string;
+        informationUri: string;
+        rules: SarifReportingDescriptor[];
+      };
+    };
+    results: SarifResult[];
+  }>;
+}
+
+export interface SarifReportOptions {
+  findings: CheckFinding[];
+  /** Raw spec text — fed into `sha1` for the partial-fingerprint salt
+   *  so two runs against the same spec produce identical fingerprints. */
+  specContent: string;
+  /** SARIF artifactLocation.uri. Defaults to "spec.json"; CLI sets the
+   *  spec's relative path so GitHub Code Scanning links findings to the
+   *  spec source file. */
+  specUri?: string;
+  toolVersion: string;
+  toolInformationUri?: string;
+}
+
+function buildRules(): SarifReportingDescriptor[] {
+  const all = [
+    ...listChecks().map((c) => ({
+      id: c.id,
+      severity: c.severity,
+      defaultExpected: c.defaultExpected,
+      references: c.references,
+    })),
+    ...listStatefulChecks().map((c) => ({
+      id: c.id,
+      severity: c.severity,
+      defaultExpected: c.defaultExpected,
+      references: c.references,
+    })),
+  ];
+  const seen = new Set<string>();
+  const rules: SarifReportingDescriptor[] = [];
+  for (const c of all) {
+    const ruleId = ruleIdFor(c.id);
+    if (seen.has(ruleId)) continue;
+    seen.add(ruleId);
+    const cat = categoryFor(c.id);
+    const helpUri = c.references.find((r) => r.url)?.url;
+    const descriptor: SarifReportingDescriptor = {
+      id: ruleId,
+      name: c.id,
+      shortDescription: { text: c.defaultExpected },
+      defaultConfiguration: { level: severityToLevel(c.severity) },
+      properties: {
+        category: cat,
+        severity: c.severity,
+        references: c.references.map((r) => r.id),
+        tags: [cat, "openapi", "zond"],
+      },
+    };
+    if (helpUri) descriptor.helpUri = helpUri;
+    rules.push(descriptor);
+  }
+  return rules.sort((a, b) => a.id.localeCompare(b.id));
+}
+
+export function generateSarifReport(opts: SarifReportOptions): SarifLog {
+  const specHash = specHashOf(opts.specContent);
+  const specUri = opts.specUri ?? "spec.json";
+  const rules = buildRules();
+  const ruleIndex = new Map(rules.map((r, i) => [r.id, i] as const));
+
+  // Sort findings deterministically so two runs over the same spec emit
+  // byte-identical SARIF — GitHub diffs the file across pushes and any
+  // reordering churn would re-open and re-close alerts spuriously.
+  const sorted = [...opts.findings].sort((a, b) => {
+    const aId = ruleIdFor(a.check);
+    const bId = ruleIdFor(b.check);
+    if (aId !== bId) return aId.localeCompare(bId);
+    if (a.operation.path !== b.operation.path) return a.operation.path.localeCompare(b.operation.path);
+    if (a.operation.method !== b.operation.method) return a.operation.method.localeCompare(b.operation.method);
+    return a.message.localeCompare(b.message);
+  });
+
+  const results: SarifResult[] = sorted.map((f) => {
+    const ruleId = ruleIdFor(f.check);
+    const ptr = jsonPointerForOperation(f.operation.path, f.operation.method);
+    const idx = ruleIndex.get(ruleId);
+    if (idx === undefined) {
+      throw new Error(`SARIF: no rule descriptor registered for check "${f.check}" (ruleId "${ruleId}")`);
+    }
+    const properties: Record<string, unknown> = {
+      severity: f.severity,
+      method: f.operation.method,
+      path: f.operation.path,
+      request_signature: f.request_signature,
+      response_status: f.response_summary.status,
+    };
+    if (f.operation.operationId) properties.operationId = f.operation.operationId;
+    if (f.response_summary.content_type) properties.response_content_type = f.response_summary.content_type;
+    if (f.evidence) properties.evidence = f.evidence;
+    if (f.recommended_action) properties.recommendedAction = f.recommended_action;
+    return {
+      ruleId,
+      ruleIndex: idx,
+      level: severityToLevel(f.severity),
+      message: { text: f.message },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: { uri: specUri },
+            region: { startLine: 1, snippet: { text: ptr } },
+          },
+          logicalLocations: [{ fullyQualifiedName: ptr, kind: "object" }],
+        },
+      ],
+      partialFingerprints: { primary: partialFingerprintFor(ruleId, ptr, specHash) },
+      properties,
+    };
+  });
+
+  return {
+    $schema: "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+    version: "2.1.0",
+    runs: [
+      {
+        tool: {
+          driver: {
+            name: "zond",
+            version: opts.toolVersion,
+            informationUri: opts.toolInformationUri ?? "https://github.com/kirrosh/zond",
+            rules,
+          },
+        },
+        results,
+      },
+    ],
+  };
+}

package/src/core/checks/stateful.ts

@@ -0,0 +1,121 @@
+/**
+ * Stateful checks (m-15 ARV-3) — security-flavored checks that need
+ * to orchestrate multiple HTTP requests against a single operation
+ * (auth probes) or a CRUD chain (use-after-free / availability).
+ *
+ * Kept in a parallel registry from the per-response `Check`s so the
+ * single-response runner stays simple. `runChecks` calls
+ * `runStateful(...)` after the per-op response phase.
+ */
+import type { OpenAPIV3 } from "openapi-types";
+import type { CrudGroup, EndpointInfo } from "../generator/types.ts";
+import type { HttpRequest, HttpResponse } from "../runner/types.ts";
+import { executeRequest } from "../runner/http-client.ts";
+import type { CheckOutcome, CheckReference, CheckRuntimeOptions, Severity } from "./types.ts";
+
+export interface StatefulHarness {
+  baseUrl: string;
+  doc: OpenAPIV3.Document;
+  /** Headers that constitute "real auth" for the run. Empty when the
+   *  caller didn't pass --auth-header / no env vars. */
+  authHeaders: Record<string, string>;
+  /** When true, security checks should skip with a warning (ARV-3 AC #6). */
+  bootstrapCleanupFailed: boolean;
+  /** ARV-181: real path-param fixtures from `.env.yaml`. Mirrors what
+   *  the per-response runner already does via ARV-141 — without this
+   *  the stateful harness rebuilds URLs with literal `{event_id}`
+   *  placeholders, gets routed to 403/404, and the broken-baseline
+   *  guard silently skips real auth checks. Optional so unit tests
+   *  can stub without it; production callers always pass them. */
+  pathVars?: Record<string, string>;
+  /** ARV-181: per-run knobs (e.g. strict401). Mirrors CheckContext.options
+   *  for stateful checks so they can read the same flags as per-response
+   *  ones. */
+  options?: CheckRuntimeOptions;
+  /** ARV-169 (m-20): per-resource overrides for cross-call probes,
+   *  keyed by `resource` name from `.api-resources.yaml`. Today only
+   *  `cross_call_references` reads `readbackDiff`; future m-20 probes
+   *  (idempotency, pagination, lifecycle) will append their own keys
+   *  to the per-resource entry. Optional — when absent each probe
+   *  falls back to its built-in defaults. */
+  resourceConfigs?: Map<string, {
+    readbackDiff?: import("../generator/resources-builder.ts").ReadbackDiffConfig;
+    idempotency?: import("../generator/resources-builder.ts").IdempotencyConfig;
+    pagination?: import("../generator/resources-builder.ts").PaginationConfig;
+    lifecycle?: import("../generator/resources-builder.ts").LifecycleConfig;
+    /** ARV-187: LLM-authored example POST body — preferred over
+     *  generateFromSchema(create) by stateful CRUD checks. */
+    seedBody?: import("../generator/resources-builder.ts").SeedBodyConfig;
+  }>;
+  send(req: HttpRequest, opts?: { timeoutMs?: number }): Promise<HttpResponse>;
+}
+
+export interface BaseStatefulCheck {
+  id: string;
+  severity: Severity;
+  defaultExpected: string;
+  references: CheckReference[];
+}
+
+export interface AuthStatefulCheck extends BaseStatefulCheck {
+  phase: "auth";
+  applies(op: EndpointInfo): boolean;
+  run(op: EndpointInfo, h: StatefulHarness): Promise<CheckOutcome>;
+}
+
+export interface CrudStatefulCheck extends BaseStatefulCheck {
+  phase: "crud";
+  applies(group: CrudGroup): boolean;
+  run(group: CrudGroup, h: StatefulHarness): Promise<CheckOutcome>;
+}
+
+export type StatefulCheck = AuthStatefulCheck | CrudStatefulCheck;
+
+const STATEFUL_REGISTRY = new Map<string, StatefulCheck>();
+
+export function registerStatefulCheck(c: StatefulCheck): void {
+  if (STATEFUL_REGISTRY.has(c.id)) throw new Error(`Stateful check "${c.id}" already registered`);
+  STATEFUL_REGISTRY.set(c.id, c);
+}
+
+export function listStatefulChecks(): StatefulCheck[] {
+  return [...STATEFUL_REGISTRY.values()].sort((a, b) => a.id.localeCompare(b.id));
+}
+
+export function makeHarness(
+  baseUrl: string,
+  doc: OpenAPIV3.Document,
+  opts: {
+    authHeaders?: Record<string, string>;
+    bootstrapCleanupFailed?: boolean;
+    timeoutMs?: number;
+    pathVars?: Record<string, string>;
+    options?: CheckRuntimeOptions;
+    resourceConfigs?: StatefulHarness["resourceConfigs"];
+    /** ARV-227: shared with the per-response phase so a single
+     *  `--max-requests` cap bounds the whole run. Throws (caught by
+     *  the runner's per-check try/catch and converted to skip) once
+     *  the budget is exhausted, so a stateful probe mid-chain doesn't
+     *  silently spin past the cap. */
+    requestBudget?: import("../runner/executor.ts").RequestBudget;
+  } = {},
+): StatefulHarness {
+  return {
+    baseUrl,
+    doc,
+    authHeaders: opts.authHeaders ?? {},
+    bootstrapCleanupFailed: opts.bootstrapCleanupFailed ?? false,
+    pathVars: opts.pathVars,
+    options: opts.options,
+    resourceConfigs: opts.resourceConfigs,
+    send: async (req, sendOpts) => {
+      if (opts.requestBudget) {
+        const { reserveRequest, MAX_REQUESTS_SKIP_REASON } = await import("../runner/executor.ts");
+        if (!reserveRequest(opts.requestBudget)) {
+          throw new Error(MAX_REQUESTS_SKIP_REASON);
+        }
+      }
+      return executeRequest(req, { timeout: sendOpts?.timeoutMs ?? opts.timeoutMs ?? 30000 });
+    },
+  };
+}

package/src/core/checks/types.ts

@@ -0,0 +1,189 @@
+/**
+ * Type contracts for the `zond checks` framework (m-15 ARV-1).
+ *
+ * A `Check` is a single named conformance/security probe applied to one
+ * operation × one HTTP response. The `runner` resolves the spec into
+ * operations, generates a request via `core/generator/data-factory`,
+ * sends it via `core/runner/send-request`, and feeds each (case,
+ * response) pair to every active check.
+ *
+ * Names mirror schemathesis V4 1-to-1 so benchmarks and findings carry
+ * across (see `backlog/milestones/m-15`). New checks register themselves
+ * in `checks/index.ts` via `registerCheck`.
+ */
+import type { OpenAPIV3 } from "openapi-types";
+import type { EndpointInfo } from "../generator/types.ts";
+import type { SchemaValidator } from "../runner/schema-validator.ts";
+import type { RecommendedAction } from "../diagnostics/failure-hints.ts";
+
+// Severity unified in src/core/severity (ARV-250). Re-exported here for
+// backwards-compatible imports across the checks subsystem; the canonical
+// definition is in core/severity. Adds 'info' tier (previously absent in
+// checks, now needed for hygiene-class findings per m-21 pivot).
+import type { Severity } from "../severity/index.ts";
+import { emptySeverityBuckets } from "../severity/index.ts";
+import type { Category } from "../severity/category.ts";
+import { emptyCategoryBuckets } from "../severity/category.ts";
+export type { Severity, Category };
+
+export type Phase = "examples" | "coverage" | "all";
+
+/** Probe shapes a check may need. ARV-1 shipped only `positive`; ARV-2
+ *  adds two more for header/method-rejection checks; ARV-4 will add
+ *  `negative_data` for the data-rejection pair. The runner generates a
+ *  case for each kind that at least one active check declares. */
+export type CaseKind =
+  | "positive"
+  | "missing_required_header"
+  | "unsupported_method"
+  | "negative_data";
+
+export interface CheckReference {
+  /** CWE / OWASP / RFC identifier — free form, agent-readable. */
+  id: string;
+  /** Optional URL for the human reader. */
+  url?: string;
+}
+
+export interface CheckResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: unknown;
+  duration_ms: number;
+}
+
+export interface CheckCase {
+  /** The operation under test (path + method + parameters). */
+  operation: EndpointInfo;
+  /** Resolved request that produced the response below. */
+  request: {
+    method: string;
+    url: string;
+    headers: Record<string, string>;
+    body?: string;
+  };
+  /** Mode the case was generated under — positive (valid) or negative
+   *  (intentionally invalid). Drives the ARV-7 `--mode` filter. */
+  mode: "positive" | "negative";
+  /** Probe shape — what the case is *built* to exercise. A check only
+   *  runs against a case whose `kind` it declared (or all if it
+   *  declared none — defaults to `positive`). */
+  kind: CaseKind;
+  /** For probe-style cases, an opaque hint the check itself produced
+   *  (e.g. the header name that was dropped, the method that was
+   *  swapped in). Lets a check emit a precise finding without parsing
+   *  the request back out. */
+  meta?: Record<string, unknown>;
+}
+
+/** ARV-179: per-run knobs surfaced from `RunCheckOptions` into the
+ *  check itself. Add new fields here when a check needs an opt-in
+ *  behaviour that doesn't deserve its own check id. Keep this lean —
+ *  most knobs belong upstream in the case-generator, not in the
+ *  check's response-side logic. */
+export interface CheckRuntimeOptions {
+  /** ARV-179: require strict 405 for `unsupported_method` (matches
+   *  schemathesis V4 default). Off by default — zond's pragmatic policy
+   *  accepts 401/403/404 as legitimate rejections of an undeclared
+   *  method (common nginx/gateway behaviour). */
+  strict405?: boolean;
+  /** ARV-181: require strict 401 for `ignored_auth` no-auth / bogus-auth
+   *  variants (matches schemathesis V4 default). Off by default — zond's
+   *  pragmatic policy accepts any 4xx as a legitimate auth-reject (403,
+   *  404, 422 are common). With this on, only 401 passes — a 403 on
+   *  no_auth becomes a finding "expected 401, got 403". */
+  strict401?: boolean;
+}
+
+export interface CheckContext {
+  case: CheckCase;
+  response: CheckResponse;
+  /** Pre-built once per run so checks don't pay AJV compile cost
+   *  per-case. Optional so unit tests can stub a context without one. */
+  schemaValidator?: SchemaValidator;
+  /** Original spec doc — checks that need declared headers /
+   *  content-types / status codes look them up here. */
+  doc?: OpenAPIV3.Document;
+  /** ARV-179: per-run knobs (see CheckRuntimeOptions). */
+  options?: CheckRuntimeOptions;
+}
+
+export type CheckOutcome =
+  | { kind: "pass" }
+  | { kind: "skip"; reason: string }
+  | { kind: "fail"; message: string; evidence?: Record<string, unknown> };
+
+export interface Check {
+  /** Stable identifier — must match schemathesis name where possible. */
+  id: string;
+  severity: Severity;
+  /** Default expected outcome ("server should NOT 5xx", etc) — surfaced
+   *  by `zond checks list` so an agent can read the contract. */
+  defaultExpected: string;
+  references: CheckReference[];
+  /** Probe shapes this check consumes. Default `["positive"]` — most
+   *  conformance checks just inspect the standard response. ARV-2's
+   *  `missing_required_header` / `unsupported_method` declare their
+   *  own kinds so the runner generates the matching probe case. */
+  caseKinds?: CaseKind[];
+  /** Whether this check is meaningful for the given operation. Used by
+   *  the runner to skip checks that don't apply (e.g. auth-related
+   *  checks on operations with no security requirement). */
+  applies(operation: EndpointInfo): boolean;
+  run(ctx: CheckContext): CheckOutcome;
+}
+
+export interface CheckFinding {
+  check: string;
+  severity: Severity;
+  /** ARV-251: finding category drives per-section roll-up in reports.
+   *  Optional for backwards compat — when absent, downstream code derives
+   *  it via `categoryFor(check)`. Storing it on the finding keeps probe
+   *  emitters and check emitters using the same shape. */
+  category?: Category;
+  operation: { path: string; method: string; operationId?: string };
+  request_signature: string;
+  response_summary: { status: number; content_type?: string };
+  message: string;
+  evidence?: Record<string, unknown>;
+  /** ARV-11 — closed enum so agents can route on it without parsing
+   *  the message. Resolved by `recommendForCheck()` keyed on the
+   *  check id (and response status for `network_error`). Optional
+   *  because synthetic findings (e.g. unit-test fakes) may skip it. */
+  recommended_action?: RecommendedAction;
+}
+
+export interface CheckRunSummary {
+  operations: number;
+  cases: number;
+  checks_run: number;
+  findings: number;
+  by_severity: Record<Severity, number>;
+  /** ARV-251: per-category roll-up — small teams use this to triage
+   *  "0 security, 12 reliability, 40 contract, 200 hygiene" instead of
+   *  reading one flat severity pile. */
+  by_category: Record<Category, number>;
+  /** ARV-26: count of `kind: "skip"` outcomes returned by checks, keyed by
+   *  `"<check_id>: <reason>"`. Surfaces the gap between probe and runtime
+   *  validators — e.g. `response_schema_conformance: no JSON Schema on this
+   *  response branch ×2` tells the user why "0 findings" doesn't mean "all
+   *  green" (probe got 4xx, response schema only declared on 2xx). */
+  skipped_outcomes: Record<string, number>;
+}
+
+export interface CheckRunData {
+  findings: CheckFinding[];
+  summary: CheckRunSummary;
+}
+
+export function emptySummary(): CheckRunSummary {
+  return {
+    operations: 0,
+    cases: 0,
+    checks_run: 0,
+    findings: 0,
+    by_severity: emptySeverityBuckets(),
+    by_category: emptyCategoryBuckets(),
+    skipped_outcomes: {},
+  };
+}