@lovrozagar/oat 0.10.0 → 0.12.0

package/dist/runtime/run.js

@@ -5,7 +5,9 @@
  * everything downstream is reported BLOCKED against that single cause, rather than re-failing
  * per case — which is what made prior attempts produce dozens of failures from one broken fixture.
  */
-import { buildModel } from "../spec/graph.js";
+import { randomBytes } from "node:crypto";
+import { isSecretHeaderName, registerSecret, registerSecretHeader } from "./redact.js";
+import { buildModel, operationResolver } from "../spec/graph.js";
 import { dereference, loadSpec } from "../spec/load.js";
 import { createPrincipal } from "./auth.js";
 import { createExchangeJournal } from "./exchanges.js";
@@ -13,9 +15,8 @@ import { DEFAULT_NETWORK_RETRIES, DEFAULT_NETWORK_WAIT_MS, NetworkError, createN
 import { resolveBackoff } from "./poll.js";
 import { persistedToPrincipal, snapshotPrincipal } from "./principals.js";
 import { CHECKS } from "./checks.js";
-import { Client } from "./client.js";
-import { FindingCollector } from "./finding.js";
-import { reportFeatureGateSchemaDrift } from "./feature-gate.js";
+import { AuthRefreshError, Client, } from "./client.js";
+import { FindingCollector, isRootCause } from "./finding.js";
 import { formatUniqueSets } from "../spec/extensions.js";
 import { excludedByProfile, resolveProfile } from "./profile.js";
 import { authStepOperationIds, buildScopeReport, createIsAuthProvisioned, GradeLedger, resolveTargetScope, staticSubjects, } from "./scope.js";
@@ -25,18 +26,27 @@ import { Ledger } from "./teardown.js";
 import { SchemaValidator } from "./validate.js";
 import { isOverflowError, overflowFrom } from "./fixture.js";
 import { SeedError, fillPath, listExisting, probeCreateFixtures, resolvePathScope, seedCohort, } from "./world.js";
-function readPointer(body, pointer) {
-    const path = pointer
-        .replace(/^\$\.?/, "")
-        .split(".")
-        .filter(Boolean);
-    let node = body;
-    for (const segment of path) {
-        if (node === null || typeof node !== "object")
-            return undefined;
-        node = node[segment];
-    }
-    return node;
+/**
+ * Static principals have no auth runtime. Copy the configured headers so adopting a credential
+ * does not mutate the config object, and keep a `Bearer ` prefix when that is how the header
+ * was already written.
+ */
+function holdHeaders(initial) {
+    let current = initial === undefined ? {} : { ...initial };
+    return {
+        adoptCredential: (token) => {
+            if (token === "")
+                return;
+            const key = Object.keys(current).find((name) => name.toLowerCase() === "authorization") ?? "authorization";
+            const previous = current[key];
+            const value = previous === undefined || /^bearer\s+/i.test(previous) ? `Bearer ${token}` : token;
+            current = { ...current, [key]: value };
+            registerSecretHeader(key);
+            registerSecret(token);
+            registerSecret(value);
+        },
+        headers: () => current,
+    };
 }
 function sameTenant(a, b) {
     const keys = [...new Set([...Object.keys(a), ...Object.keys(b)])];
@@ -81,37 +91,6 @@ async function teardownPrincipals(principals, hooks, findings) {
         }
     }
 }
-/**
- * A 429 is a finding only when the request that drew it was demonstrably under a rate the
- * *document* declared — `rateLimitSource === "tag"` and the bucket had a free token, meaning oat
- * did not have to wait for one. A 429 against a config-supplied rate is the operator's own guess
- * about the environment, not a claim the API made, so it is paced around and never reported; a
- * 429 that only arrived after oat's own bucket made the request wait means oat's rate model was
- * too generous, which is oat's fault, not the backend's.
- *
- * Grouped by category rather than reported per request: the point is "the declared rate for X is
- * wrong", once, not a flood of identical findings for every request that category serves.
- */
-function reportRateLimitViolations(client, findings) {
-    const byCategory = new Map();
-    for (const exchange of client.transcript) {
-        if (exchange.status !== 429)
-            continue;
-        if (exchange.rateLimitSource !== "tag" || exchange.rateLimitHadRoom !== true)
-            continue;
-        const category = exchange.rateLimitCategory ?? "unknown";
-        const list = byCategory.get(category) ?? [];
-        list.push(exchange);
-        byCategory.set(category, list);
-    }
-    for (const [category, exchanges] of byCategory) {
-        findings.spec("spec.declared-rate-limit-is-honoured", category, `the declared "${category}" rate limit is not honoured`, `${exchanges.length} request(s) to the "${category}" category returned 429 despite oat ` +
-            "pacing them within the rate x-rate-limit declares — each had a free token in its own " +
-            "bucket at the moment it was sent, so this is not oat outrunning its own model. Either " +
-            `the declared rate is wrong, or the backend enforces a stricter one than "${category}" ` +
-            "documents.", exchanges.slice(0, 3));
-    }
-}
 /**
  * Resolves a principal to something that can produce headers on demand.
  *
@@ -124,8 +103,17 @@ async function resolvePrincipal(principal, model, client, hooks, outOfBand, orig
     /* A principal without a flow authenticates by static header — a long-lived API key needs no
      * acquisition at all, and forcing one would be ceremony. */
     if (principal.auth === undefined) {
+        /* A static key is a credential like any other: never written down in clear. */
+        for (const [name, value] of Object.entries(principal.headers ?? {})) {
+            if (!isSecretHeaderName(name) && !/key|token|secret|auth/i.test(name))
+                continue;
+            registerSecretHeader(name);
+            registerSecret(value);
+        }
+        const held = holdHeaders(principal.headers);
         return {
-            headers: () => principal.headers ?? {},
+            adoptCredential: held.adoptCredential,
+            headers: held.headers,
             id: principal.id,
             inviteAs: principal.inviteAs,
             rank: principal.rank ?? 0,
@@ -156,6 +144,7 @@ async function resolvePrincipal(principal, model, client, hooks, outOfBand, orig
         refreshIfStale: runtime.refreshIfStale,
     });
     return {
+        adoptCredential: (token) => runtime.adoptCredential(token),
         headers,
         id: principal.id,
         inviteAs: principal.inviteAs,
@@ -166,7 +155,7 @@ async function resolvePrincipal(principal, model, client, hooks, outOfBand, orig
     };
 }
 /** Reads whatever the list endpoint already returns, for degraded read-only coverage. */
-async function readExisting(listOp, client, headers, roots) {
+async function readExisting(listOp, model, client, headers, roots) {
     /* Seed with every known root, not only the list route's own parameters. Sibling routes for
      * the same entity are often scoped differently — a global list beside a tenant-scoped item
      * route — and a scope built from the list alone leaves those unresolvable. */
@@ -175,7 +164,7 @@ async function readExisting(listOp, client, headers, roots) {
         if (scope[param] === undefined)
             return { records: [], scope };
     }
-    const records = await listExisting(listOp, client, headers, scope);
+    const records = await listExisting(listOp, model, client, headers, scope);
     return { records, scope };
 }
 export async function run(options) {
@@ -190,7 +179,7 @@ export async function run(options) {
     let currentCheck;
     let currentEntityIndex;
     let entityTotal;
-    const defectCount = () => findings.findings.filter((f) => f.verdict !== "BLOCKED" && f.verdict !== "COVERAGE_GAP").length;
+    const defectCount = () => findings.findings.filter(isRootCause).length;
     const oldestInflight = () => {
         let oldest;
         for (const probe of inflight) {
@@ -294,9 +283,11 @@ export async function run(options) {
         remember(exchange);
         if (journal !== null) {
             try {
+                /* The exchange names its own check and entity: a shared "current check" was wrong
+                 * whenever checks ran concurrently, and journaled a whole batch under the last one. */
                 await journal.record(exchange, {
-                    ...(currentCheck === undefined ? {} : { check: currentCheck }),
-                    ...(currentEntity === undefined ? {} : { entity: currentEntity }),
+                    ...(exchange.check === undefined ? {} : { check: exchange.check }),
+                    ...(exchange.subject === undefined ? {} : { entity: exchange.subject }),
                     phase: currentPhase,
                 });
             }
@@ -354,10 +345,15 @@ export async function run(options) {
     client = new Client(options.baseUrl, options.globalHeaders ?? {}, options.maxInFlight ?? 4, onExchange, rateLimiter, httpHooks, networkClient);
     if (hooks.resolveHeaders !== undefined)
         client.setResolveHeaders(hooks.resolveHeaders);
-    const validator = new SchemaValidator();
+    client.setOperationResolver(operationResolver(model));
+    const validator = new SchemaValidator(model.defs);
     const ledger = new Ledger();
     const seed = options.seed ?? 1;
+    /* Fresh per run: the seed reproduces a run, the nonce keeps two runs' records apart. */
+    const nonce = options.nonce ?? randomBytes(4).toString("hex");
     const outOfBand = resolveBackoff(options.outOfBand);
+    /* Write paths that have had the whole payload catalog, shared by every entity in the run. */
+    const payloadPaths = new Map();
     if (options.principals[0] === undefined)
         throw new Error("oat: at least one principal is required");
     const originClients = await loadOriginClients(options.origins ?? [], hooks, options.maxInFlight ?? 4, options.globalHeaders ?? {}, onExchange, httpHooks, networkClient);
@@ -398,18 +394,21 @@ export async function run(options) {
     const resolved = [];
     try {
         for (const principal of options.principals) {
-            resolved.push(await resolvePrincipal(principal, model, client, hooks, outOfBand, originClients.size === 0 ? undefined : originClients));
+            resolved.push(await resolvePrincipal(principal, model, client.view({ purpose: "auth" }), hooks, outOfBand, originClients.size === 0 ? undefined : originClients));
         }
     }
     catch (error) {
         if (!isNetworkError(error))
             throw error;
         noteNetwork(error, "auth");
+        await journal?.flush().catch(() => undefined);
         return {
             checksOutOfScope: [],
             checksRun: [],
             checksSkipped: [],
             checksSuppressed: [],
+            checkNotes: [],
+            checkTimings: [],
             client,
             created: 0,
             entitiesTested: [],
@@ -430,14 +429,51 @@ export async function run(options) {
         phase: "auth",
         requests: client.transcript.length,
     });
+    /* Every exchange names the principal whose credential it carried. */
+    client.setPrincipalResolver((sent) => resolved.find((principal) => {
+        const own = principal.headers();
+        return Object.keys(own).length > 0 && Object.entries(own).every(([key, value]) => sent[key] === value);
+    })?.id);
     const alpha = resolved[0];
-    /* Isolation peer: first principal whose roots are a different tenant — not "whoever is
-     * second in the array". A same-tenant viewer sitting at index 1 must not steal that slot. */
-    const peer = resolved.slice(1).find((candidate) => !sameTenant(alpha.roots, candidate.roots));
+    const ownerOf = (principal) => ({ headers: principal.headers, id: principal.id });
+    const alphaOwner = ownerOf(alpha);
+    /* A record a principal created and could not remove itself — a member cannot delete — may be
+     * removed by a stronger principal of the same tenant, and by nobody else. */
+    const fallbackDeleters = (owner) => {
+        const creator = resolved.find((principal) => principal.id === owner.id);
+        if (creator === undefined)
+            return [];
+        return resolved
+            .filter((principal) => principal.id !== creator.id && sameTenant(principal.roots, creator.roots))
+            .sort((a, b) => b.rank - a.rank)
+            .map(ownerOf);
+    };
+    const recorder = (owner) => (entity, id, values) => {
+        ledger.record(entity, id, values, owner);
+    };
+    /**
+     * Proactive refreshes, before each entity and before the credentials are persisted. A failure
+     * is reported — the backend would not renew a session — and the run carries on to teardown,
+     * where an exception here used to end it with every record still in place.
+     */
+    const refreshAll = async (where) => {
+        for (const principal of resolved) {
+            try {
+                await principal.runtime?.refreshIfStale();
+            }
+            catch (error) {
+                findings.blocked("auth.refresh", where, `refreshing the credential of "${principal.id}" failed`, error instanceof Error ? error.message : String(error));
+                return false;
+            }
+        }
+        return true;
+    };
     const entitiesTested = [];
     const checksRun = new Set();
     const checksSkipped = [];
     const checksSuppressed = [];
+    const checkTimings = [];
+    const checkNotes = [];
     const checksOutOfScope = [];
     const profileExcludes = (operationId) => {
         const op = model.byOperationId.get(operationId);
@@ -465,6 +501,17 @@ export async function run(options) {
         const createOp = model.byOperationId.get(entity.create ?? "");
         if (listOp === undefined)
             return;
+        /* Cohort variants the backend refused. A check that stands down for want of cohort data
+         * says which variants never arrived, so the gap points at the seed rather than the API. */
+        let lostVariants = [];
+        /* This entity's own scratch parents, shared by its records and its principals' scopes but by
+         * no other entity: entities whose parent chains are disjoint run side by side, and one that
+         * shares a parent record with another could see that entity's writes through it. */
+        const ancestors = new Map();
+        const withLostVariants = (needs) => lostVariants.length === 0 || !/cohort|record/.test(needs)
+            ? needs
+            : `${needs} (the cohort is missing its ${lostVariants.map((v) => `"${v}"`).join(", ")} ` +
+                `variant${lostVariants.length === 1 ? "" : "s"}, which failed to seed)`;
         if (networkGate.exhausted)
             return;
         /* Targets this entity's checks can grade. Seeding findings land on them: under --ops a seed
@@ -487,6 +534,8 @@ export async function run(options) {
         currentEntity = entity.name;
         currentCheck = undefined;
         currentPhase = "seed";
+        /* Everything this entity's world-building sends is seeding, attributed to the entity. */
+        const seeding = client.view({ purpose: "seed", subject: entity.name });
         tick({
             entity: entity.name,
             entityIndex: currentEntityIndex,
@@ -495,8 +544,8 @@ export async function run(options) {
             phase: "seed",
             requests: client.transcript.length,
         });
-        for (const principal of resolved)
-            await principal.runtime?.refreshIfStale();
+        if (!(await refreshAll(entity.name)))
+            return;
         const rootValues = { ...options.roots, ...alpha.roots };
         let scope;
         let records;
@@ -506,7 +555,7 @@ export async function run(options) {
             /* Same fallback a failed create takes below: read-only coverage against whatever
              * already exists beats no coverage, and it is exactly the state most likely to hide a
              * read-path bug. The gap finding excludeOp already reported names the reason. */
-            const existing = await readExisting(listOp, client, alpha.headers(), {
+            const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
                 ...options.roots,
                 ...alpha.roots,
             });
@@ -522,7 +571,7 @@ export async function run(options) {
         else if (createOp === undefined || authProvisioned) {
             /* Invite is not a fixture create; register / x-fresh-principal already ran in auth.
              * The invite check is the only thing that POSTs a grant, and it uses inviteAs. */
-            const existing = await readExisting(listOp, client, alpha.headers(), {
+            const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
                 ...options.roots,
                 ...alpha.roots,
             });
@@ -532,8 +581,12 @@ export async function run(options) {
         }
         else {
             try {
-                scope = await resolvePathScope(createOp, model, client, {
+                scope = await resolvePathScope(createOp, model, seeding, {
                     authHeaders: alpha.headers,
+                    ancestors,
+                    onCreate: recorder(alphaOwner),
+                    principal: alpha.id,
+                    nonce,
                     roots: rootValues,
                     seed,
                     uploads,
@@ -543,9 +596,13 @@ export async function run(options) {
                  * a tenant-scoped item route — and a scope built from create alone leaves those
                  * unresolvable, which shows up as a wall of "could not complete" gaps. */
                 scope.values = { ...rootValues, ...scope.values };
-                const cohort = await seedCohort(createOp, model, client, {
+                const cohort = await seedCohort(createOp, model, seeding, {
                     authHeaders: alpha.headers,
                     ...(options.cohortSize === undefined ? {} : { cohortSize: options.cohortSize }),
+                    ancestors,
+                    onCreate: recorder(alphaOwner),
+                    principal: alpha.id,
+                    nonce,
                     roots: rootValues,
                     seed,
                     uploads,
@@ -553,6 +610,19 @@ export async function run(options) {
                 if (cohort.uniqueGap !== undefined) {
                     entityFindings.gap("world.seed", entity.name, cohort.uniqueGap, cohort.uniqueGap);
                 }
+                lostVariants = (cohort.failedVariants ?? []).map((failed) => failed.variant);
+                for (const failed of cohort.failedVariants ?? []) {
+                    const said = JSON.stringify(failed.exchange.responseBody).slice(0, 300);
+                    if (failed.exchange.status >= 500) {
+                        entityFindings.backend("create.does-not-error", entity.name, `creating the "${failed.variant}" variant of a "${entity.name}" fails with a server error`, `${createOp.operationId} returned ${failed.exchange.status} for a body generated from the ` +
+                            `documented schema (${said}). The other variants were created, so this input is what ` +
+                            "breaks the handler. Checks that need this variant's data run without it.", [failed.exchange]);
+                    }
+                    else {
+                        entityFindings.gap("world.seed", entity.name, `the "${failed.variant}" variant of "${entity.name}" was refused with ${failed.exchange.status}`, `${createOp.operationId} refused a body the document permits (${said}). Checks that ` +
+                            "rely on this variant's values run without them.");
+                    }
+                }
                 if (cohort.adopted === true) {
                     /* Plan limit after an effect already created the row: keep the id so children
                      * (row after extract→table) can seed, but do not assert write-path oracles
@@ -560,17 +630,10 @@ export async function run(options) {
                     entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" hit a plan limit; using an existing same-tenant record`, `${createOp.operationId} returned a plan-limit refusal and the list already ` +
                         "had a record — likely an earlier x-effects create. Write-path checks stand " +
                         "down rather than treat payment_required as a backend defect.");
+                    /* Adopted, not created: what oat did create was recorded as it happened, and
+                     * this record is somebody else's to keep. */
                     records = cohort.records;
                     degraded = true;
-                    for (const ancestor of scope.created) {
-                        ledger.record(ancestor.entity, ancestor.id, scope.values);
-                    }
-                    for (const record of records) {
-                        const id = record[entity.identity ?? "id"];
-                        if (typeof id === "string" || typeof id === "number") {
-                            ledger.record(entity.name, String(id), scope.values);
-                        }
-                    }
                 }
                 else if (cohort.uniqueAdopted === true) {
                     /* Unique 409 on the first variant with a same-tenant list: adopt like 402,
@@ -584,24 +647,14 @@ export async function run(options) {
                     records = cohort.records;
                     degraded = true;
                     uniqueAdopted = true;
-                    for (const ancestor of scope.created) {
-                        ledger.record(ancestor.entity, ancestor.id, scope.values);
-                    }
-                    for (const record of records) {
-                        const id = record[entity.identity ?? "id"];
-                        if (typeof id === "string" || typeof id === "number") {
-                            ledger.record(entity.name, String(id), scope.values);
-                        }
-                    }
                 }
                 else if (cohort.featureGate !== null) {
                     /* Same degradation a profile-excluded create takes: the tag said this
                      * principal cannot create the row, so a correct 403 is coverage, not a
-                     * seed defect. The 403 body still has to match the documented schema. */
-                    reportFeatureGateSchemaDrift(entityFindings, validator, createOp, model.rawOperations.get(createOp.operationId), cohort.featureGate.exchange, entity.name);
+                     * seed defect. schema.error-response-matches-document still judges its body. */
                     entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" is gated by ${cohort.featureGate.detail}`, `${cohort.featureGate.detail}. Checks that need a row oat created stand down ` +
                         "rather than treat the documented 403 as a defect.");
-                    const existing = await readExisting(listOp, client, alpha.headers(), {
+                    const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
                         ...options.roots,
                         ...alpha.roots,
                     });
@@ -614,15 +667,9 @@ export async function run(options) {
                     degraded = true;
                 }
                 else {
+                    /* Every ancestor and cohort record was ledgered as it was created, in creation
+                     * order — the unwind reverses it, so children go before their parents. */
                     records = cohort.records;
-                    /* Ancestors first, then the cohort — the unwind reverses this, so children are
-                     * always removed before the parents they hang from. */
-                    for (const ancestor of scope.created) {
-                        ledger.record(ancestor.entity, ancestor.id, scope.values);
-                    }
-                    for (const record of records) {
-                        ledger.record(entity.name, String(record[entity.identity ?? "id"]), scope.values);
-                    }
                 }
             }
             catch (error) {
@@ -633,7 +680,7 @@ export async function run(options) {
                 if (isOverflowError(error)) {
                     const overflow = overflowFrom(error, createOp.operationId);
                     entityFindings.gap("world.seed", entity.name, overflow.message, overflow.message);
-                    const existing = await readExisting(listOp, client, alpha.headers(), {
+                    const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
                         ...options.roots,
                         ...alpha.roots,
                     });
@@ -652,7 +699,10 @@ export async function run(options) {
                     const failedOp = (error instanceof SeedError ? error.operationId : undefined) ?? createOp.operationId;
                     const evidence = error instanceof SeedError && error.exchange !== undefined
                         ? [error.exchange]
-                        : client.transcript.filter((e) => e.status >= 500).slice(-1);
+                        : await Promise.all(client.transcript
+                            .filter((e) => e.status >= 500)
+                            .slice(-1)
+                            .map((e) => client.hydrate(e)));
                     /* Under --ops a create nobody targeted is support. Its failure leaves the targets
                      * ungraded — blocked, with the evidence — but it is not what this run was asked to
                      * judge, so it is not reported as a defect of its own. */
@@ -670,7 +720,7 @@ export async function run(options) {
                     /* Fall back to whatever already exists. A backend whose create is broken can still
                      * have a working list, and read-only coverage beats no coverage — this is exactly
                      * the state in which a read-path bug is most likely to be sitting undiscovered. */
-                    const existing = await readExisting(listOp, client, alpha.headers(), {
+                    const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
                         ...options.roots,
                         ...alpha.roots,
                     });
@@ -688,13 +738,19 @@ export async function run(options) {
         const actorOf = async (principal, seedOffset) => {
             const roots = { ...options.roots, ...principal.roots };
             try {
-                const next = await resolvePathScope(listOp, model, client, {
+                const next = await resolvePathScope(listOp, model, seeding, {
                     authHeaders: principal.headers,
+                    /* Created as this principal, so removed as this principal. */
+                    ancestors,
+                    onCreate: recorder(ownerOf(principal)),
+                    principal: principal.id,
+                    nonce,
                     roots,
                     seed: seed + seedOffset,
                     uploads: worldUploads(seedOffset),
                 });
                 return {
+                    adoptCredential: principal.adoptCredential,
                     headers: principal.headers,
                     id: principal.id,
                     inviteAs: principal.inviteAs,
@@ -706,6 +762,7 @@ export async function run(options) {
             }
             catch {
                 return {
+                    adoptCredential: principal.adoptCredential,
                     headers: principal.headers,
                     id: principal.id,
                     inviteAs: principal.inviteAs,
@@ -718,6 +775,7 @@ export async function run(options) {
         };
         const actors = [
             {
+                adoptCredential: alpha.adoptCredential,
                 headers: alpha.headers,
                 id: alpha.id,
                 inviteAs: alpha.inviteAs,
@@ -745,9 +803,22 @@ export async function run(options) {
          * reporting that it had cleaned up everything it made. Recording centrally rather than at
          * each call site means a new check cannot forget: the wrapper sees every request.
          */
-        const trackingClient = createOp === undefined || degraded
-            ? client
-            : new Proxy(client, {
+        let createPath;
+        try {
+            createPath = createOp === undefined ? undefined : fillPath(createOp.path, scope.values);
+        }
+        catch {
+            createPath = undefined;
+        }
+        /* Whoever sent the request owns what it created: matched on the credential it carried. */
+        const ownerByHeaders = (sent) => {
+            const rendered = JSON.stringify(typeof sent === "function" ? sent() : (sent ?? {}));
+            const match = resolved.find((principal) => JSON.stringify(principal.headers()) === rendered);
+            return match === undefined ? alphaOwner : ownerOf(match);
+        };
+        const tracked = (base) => createOp === undefined || degraded
+            ? base
+            : new Proxy(base, {
                 get(target, property, receiver) {
                     if (property !== "request")
                         return Reflect.get(target, property, receiver);
@@ -755,15 +826,18 @@ export async function run(options) {
                         const exchange = await target.request(method, path, options ?? {});
                         if (method.toUpperCase() !== "POST" || exchange.status >= 300)
                             return exchange;
+                        /* Only this entity's own create makes one of its records. Any other 2xx
+                         * POST carrying an `id` — an invite, an action — names something else, and
+                         * on an integer-id API deleting by that number removes an unrelated row. */
+                        if (path !== createPath)
+                            return exchange;
                         const body = exchange.responseBody;
                         if (body === null || typeof body !== "object")
                             return exchange;
                         const id = body[entity.identity ?? "id"];
                         if (typeof id !== "string" && typeof id !== "number")
                             return exchange;
-                        /* Recording the same id twice is harmless — the unwind tolerates a 404
-                         * on an already-removed record — and missing one is not. */
-                        ledger.record(entity.name, String(id), scope.values);
+                        ledger.record(entity.name, String(id), scope.values, ownerByHeaders(options?.headers));
                         return exchange;
                     };
                 },
@@ -814,9 +888,11 @@ export async function run(options) {
             inScope: (op) => runScope.inScope(op.operationId),
             hooks,
             outOfBand,
+            payloads: { policy: options.payloads ?? "per-write-path", ran: payloadPaths },
             auth: alpha.headers,
+            recordCreated: recorder(alphaOwner),
             ...(alpha.runtime === undefined ? {} : { refreshIfStale: alpha.runtime.refreshIfStale }),
-            client: trackingClient,
+            client: tracked(client.view({ purpose: "assertion", subject: entity.name })),
             collectionKey: listOp.collection?.key ?? null,
             /* In degraded mode oat did not write these records, so it has no oracle for them —
              * every write-path check must sit out rather than assert against data it did not
@@ -840,6 +916,7 @@ export async function run(options) {
             softDelete: model.operations.find((op) => op.entity === entity.name && op.softDelete !== null)?.softDelete ??
                 listOp.softDelete,
             seed,
+            nonce,
             updateOp: degraded || updateExcluded ? undefined : updateOpModel,
             uploads,
             validator,
@@ -858,12 +935,20 @@ export async function run(options) {
         entitiesTested.push(entity.name);
         /* Subjects each check grades on this entity: in scope, and not excluded by the profile. */
         const gradedBy = new Map();
+        /* What each check's plan gathered, and how each check ended, on this entity. */
+        const plans = new Map();
+        const outcomes = new Map();
         const runOne = async (check) => {
             const graded = gradedBy.get(check.id) ?? [];
             let judged;
+            /* Its own client: every exchange names this check, even while a batch runs concurrently.
+             * A check that does not mutate gets one that refuses to write. */
+            const stamp = { check: check.id, purpose: "assertion", subject: entity.name };
+            const attributed = graded.length === 0 ? findings : findings.attributed(graded);
             const view = {
                 ...ctx,
-                ...(graded.length === 0 ? {} : { findings: findings.attributed(graded) }),
+                client: tracked(check.mutates === true ? client.view(stamp) : client.readOnlyView(stamp)),
+                findings: attributed.ownedBy(check.id),
                 judged: (operationIds) => {
                     judged ??= new Set();
                     for (const id of operationIds)
@@ -881,15 +966,45 @@ export async function run(options) {
                 phase: "test",
                 requests: client.transcript.length,
             });
+            const start = findings.findings.length;
+            const began = performance.now();
+            let outcome;
             try {
-                await check.run(view);
+                outcome = await check.run(view, plans.get(check.id));
             }
             catch (error) {
                 if (isNetworkError(error)) {
                     noteNetwork(error, entity.name);
+                    outcomes.set(check.id, { kind: "unresolved", reason: "the network failed" });
+                    return;
+                }
+                if (error instanceof AuthRefreshError) {
+                    outcomes.set(check.id, { kind: "unresolved", reason: "a credential could not be refreshed" });
+                    attributed.blocked("auth.refresh", entity.name, `refreshing the credential${error.principal === undefined ? "" : ` of "${error.principal}"`} failed`, `${error.message}. ${check.id} could not complete.`);
                     return;
                 }
-                view.findings.gap(check.id, entity.name, `check "${check.id}" could not complete`, error instanceof Error ? error.message : String(error));
+                outcome = view.findings.gap(check.id, entity.name, `check "${check.id}" could not complete`, error instanceof Error ? error.message : String(error));
+            }
+            checkTimings.push({ check: check.id, entity: entity.name, ms: Math.round(performance.now() - began) });
+            /* A check that reported along the way and then fell through to "asserted" did not
+             * assert anything: what it reported is the outcome. */
+            if (outcome.kind === "asserted") {
+                const reported = findings.findings
+                    .slice(start)
+                    .find((f) => f.check === check.id && f.entity === entity.name && isRootCause(f));
+                if (reported !== undefined)
+                    outcome = { kind: "finding", verdict: reported.verdict };
+            }
+            outcomes.set(check.id, outcome);
+            if (outcome.kind === "asserted" && outcome.note !== undefined) {
+                checkNotes.push({ check: check.id, entity: entity.name, note: outcome.note });
+            }
+            if (outcome.kind === "stood-down") {
+                /* Not a pass: the property was never tested here, and the report says why. */
+                const needs = withLostVariants(outcome.reason);
+                checksSkipped.push({ check: check.id, entity: entity.name, needs });
+                grades.skipped(graded, needs);
+                return;
             }
             grades.graded(judged === undefined ? graded : graded.filter((id) => judged?.has(id) === true), check.id);
         };
@@ -911,7 +1026,13 @@ export async function run(options) {
         const suppressedBy = new Map();
         const suppressed = (check) => {
             for (const dependency of check.dependsOn ?? []) {
-                const failed = findings.findings.some((f) => f.check === dependency && f.entity === entity.name);
+                /* Only a failed premise suppresses. A dependency that stood down, could not reach a
+                 * verdict, or was blocked says nothing about whether its premise holds. */
+                const outcome = outcomes.get(dependency);
+                const failed = outcome === undefined
+                    ? /* Not run as a check here, but reported against — during seeding, say. */
+                        findings.findings.some((f) => f.check === dependency && f.entity === entity.name && isRootCause(f))
+                    : outcome.kind === "finding";
                 const inherited = suppressedBy.get(dependency);
                 if (!failed && inherited === undefined)
                     continue;
@@ -940,6 +1061,8 @@ export async function run(options) {
             await Promise.all(pending.map(runOne));
         };
         for (const check of CHECKS) {
+            if (options.signal?.aborted === true)
+                break;
             const graded = check.subjects(entity, model).filter((id) => runScope.inScope(id) && !profileExcludes(id));
             /* Out of scope is not a skip: the check could have run, this run was not asked to. */
             if (runScope.mode === "targeted" && graded.length === 0) {
@@ -947,15 +1070,19 @@ export async function run(options) {
                 continue;
             }
             gradedBy.set(check.id, graded);
-            if (!check.applicable(ctx)) {
+            const planned = check.plan(ctx);
+            if (!planned.ok) {
                 /* Recorded, not dropped: on an API shaped unlike the fixture this is most of the
                  * suite, and a silent skip reads exactly like a clean result. */
-                const needs = check.needs ?? "an unstated precondition";
+                const needs = withLostVariants(planned.needs ?? check.needs ?? "an unstated precondition");
                 checksSkipped.push({ check: check.id, entity: entity.name, needs });
                 grades.skipped(graded, needs);
                 continue;
             }
-            if (check.mutates === true) {
+            plans.set(check.id, planned.value);
+            /* A writer runs alone; so does a check that judges the transcript, once everything
+             * queued before it has finished sending. */
+            if (check.mutates === true || check.judgesTranscript === true) {
                 await flush();
                 if (suppressed(check))
                     continue;
@@ -972,22 +1099,55 @@ export async function run(options) {
         await flush();
     };
     /*
-     * Entities run in series. Checks inside an entity stay sequential because cascade
-     * suppression consults findings already reported for it — concurrent checks would let a
-     * root cause and its consequences race and both be reported. Nested graphs also couple
-     * entities: a child create in flight while a parent page-walk runs invents pagination
-     * findings, so there are no entity lanes.
+     * Checks inside an entity stay ordered because cascade suppression consults findings already
+     * reported for it — concurrent checks would let a root cause and its consequences race and
+     * both be reported. Nested graphs couple entities too: a child create in flight while a
+     * parent page-walk runs invents pagination findings. So entities that can observe each
+     * other share a lane and run in series, and only lanes run side by side.
      */
     const queue = runScope.entities;
     entityTotal = queue.length;
-    for (const [index, entity] of queue.entries()) {
-        if (networkGate.exhausted) {
-            const left = queue.length - index;
-            findings.blocked("net.unreachable", "run", `skipped ${left} remaining entit${left === 1 ? "y" : "ies"}`, "network down");
-            break;
+    let interrupted = false;
+    let started = 0;
+    let stopNoted = false;
+    /* Entities that share nothing run side by side; see `entityLanes`. */
+    const runLane = async (lane) => {
+        for (const entity of lane) {
+            /* Stopped from outside — Ctrl-C, a deadline. Nothing more is tested, and everything
+             * made so far is still removed below. */
+            if (options.signal?.aborted === true || networkGate.exhausted) {
+                if (stopNoted)
+                    return;
+                stopNoted = true;
+                const left = queue.length - started;
+                if (options.signal?.aborted === true) {
+                    interrupted = true;
+                    findings.gap("run.interrupted", "run", `stopped before ${left} remaining entit${left === 1 ? "y" : "ies"}`, "the run was interrupted; what was tested is reported, and what oat created is removed");
+                }
+                else {
+                    findings.blocked("net.unreachable", "run", `skipped ${left} remaining entit${left === 1 ? "y" : "ies"}`, "network down");
+                }
+                return;
+            }
+            started += 1;
+            currentEntityIndex = started;
+            try {
+                await testEntity(entity);
+            }
+            catch (error) {
+                /* An error oat did not expect must not skip teardown: it is reported, and the run
+                 * carries on to remove what it made and write its report. */
+                findings.blocked("run.error", entity.name, `testing "${entity.name}" stopped on an unexpected error`, errorText(error));
+            }
         }
-        currentEntityIndex = index + 1;
-        await testEntity(entity);
+    };
+    try {
+        await Promise.all(entityLanes(model, queue).map(runLane));
+    }
+    catch (error) {
+        /* Each entity's errors are caught where it runs; this is the scheduling around them. What
+         * was created is still removed below, and the report still written. */
+        findings.blocked("run.error", "run", "testing stopped on an unexpected error", errorText(error));
     }
     /* Unwind after every check has run, never per case: a check may legitimately depend on records
      * another one created, and tearing down early turns that into a phantom defect. */
@@ -998,25 +1158,32 @@ export async function run(options) {
         phase: "teardown",
         requests: client.transcript.length,
     });
-    const teardown = options.keepFixtures === true || ledger.size === 0
-        ? null
-        : await ledger.unwind(model, client, alpha.headers, (done, total, item) => {
-            if (done % 25 !== 0 && done !== total)
-                return;
-            tick({
-                entity: item.entity,
-                message: `${done}/${total} ${item.entity} ${item.id}`,
-                phase: "teardown",
-                requests: client.transcript.length,
+    let teardown = null;
+    if (options.keepFixtures !== true && ledger.size > 0) {
+        try {
+            teardown = await ledger.unwind(model, client.view({ purpose: "teardown" }), fallbackDeleters, (done, total, item) => {
+                if (done % 25 !== 0 && done !== total)
+                    return;
+                tick({
+                    entity: item.entity,
+                    message: `${done}/${total} ${item.entity} ${item.id}`,
+                    phase: "teardown",
+                    requests: client.transcript.length,
+                });
             });
-        });
+        }
+        catch (error) {
+            findings.blocked("world.teardown", "run", `teardown stopped on an unexpected error with ${ledger.size} record(s) ledgered`, errorText(error));
+        }
+    }
     if (teardown !== null && teardown.unsupported.length > 0) {
         findings.gap("world.teardown", teardown.unsupported.join(", "), "records created during the run could not be removed", `no delete operation is reachable for ${teardown.unsupported.join(", ")}, so this run left ` +
             "fixtures behind. Declare x-cleanup to name the route that removes them.");
     }
-    reportRateLimitViolations(client, findings);
-    for (const principal of resolved)
-        await principal.runtime?.refreshIfStale();
+    await refreshAll("run");
+    /* Every check has judged the transcript; bodies moved out of memory are no longer needed. */
+    await client.dispose();
+    await journal?.flush().catch(() => undefined);
     const persisted = resolved.map((principal) => snapshotPrincipal({
         headers: principal.headers,
         id: principal.id,
@@ -1027,7 +1194,12 @@ export async function run(options) {
     }));
     const coverage = scopeReport();
     if ((options.origins ?? []).length > 0 && options.skipPrincipalTeardown !== true) {
-        await runSecondaryOrigins(options, persisted, findings, checksRun, checksSkipped, checksSuppressed, checksOutOfScope, entitiesTested, runScope, coverage);
+        try {
+            await runSecondaryOrigins(options, persisted, findings, checksRun, checksSkipped, checksSuppressed, checksOutOfScope, entitiesTested, runScope, coverage);
+        }
+        catch (error) {
+            findings.blocked("run.error", "origins", "the secondary origins stopped on an unexpected error", errorText(error));
+        }
     }
     if (options.skipPrincipalTeardown !== true) {
         await teardownPrincipals(resolved, hooks, findings);
@@ -1038,6 +1210,8 @@ export async function run(options) {
         requests: client.transcript.length,
     });
     return {
+        checkNotes,
+        checkTimings,
         checksOutOfScope,
         checksRun: [...checksRun].sort(),
         checksSkipped,
@@ -1053,6 +1227,7 @@ export async function run(options) {
         profileExclusions,
         scope: coverage,
         teardown,
+        ...(interrupted ? { interrupted } : {}),
         ...(journal === null ? {} : { exchanges: { count: journal.count } }),
         ...(networkOutcome === undefined ? {} : { network: networkOutcome }),
     };
@@ -1074,6 +1249,9 @@ async function loadOriginClients(origins, hooks, maxInFlight, globalHeaders, onE
     }
     return map;
 }
+function errorText(error) {
+    return error instanceof Error ? (error.stack ?? error.message) : String(error);
+}
 async function runSecondaryOrigins(options, persisted, findings, checksRun, checksSkipped, checksSuppressed, checksOutOfScope, entitiesTested, runScope, coverage) {
     for (const origin of options.origins ?? []) {
         /* Targets were resolved against every origin up front; an origin none of them name has
@@ -1138,4 +1316,62 @@ async function runSecondaryOrigins(options, persisted, findings, checksRun, chec
         }
     }
 }
+/**
+ * Entities grouped into lanes that can be tested side by side.
+ *
+ * Two entities share a lane when one can observe the other's writes: one's collection lives
+ * under the other's (a child, created inside a parent oat also makes), an operation of one
+ * invalidates a route of the other, or one's effects, waits or async polls land on the other.
+ * Inside a lane entities keep their order and run one after another; lanes share nothing, so
+ * nothing one does can turn up as a finding in another.
+ */
+export function entityLanes(model, entities) {
+    const names = entities.map((entity) => entity.name);
+    const parent = new Map(names.map((name) => [name, name]));
+    const find = (name) => {
+        let root = name;
+        while (parent.get(root) !== root)
+            root = parent.get(root) ?? root;
+        return root;
+    };
+    const join = (a, b) => {
+        if (a === null || a === undefined || b === null || b === undefined)
+            return;
+        if (!parent.has(a) || !parent.has(b))
+            return;
+        parent.set(find(a), find(b));
+    };
+    const collection = (entity) => {
+        const op = model.byOperationId.get(entity.list ?? entity.create ?? "");
+        return op?.path.replace(/\/+$/, "");
+    };
+    for (const a of entities) {
+        const pathA = collection(a);
+        for (const b of entities) {
+            const pathB = collection(b);
+            if (a === b || pathA === undefined || pathB === undefined)
+                continue;
+            if (pathB.startsWith(`${pathA}/`))
+                join(a.name, b.name);
+        }
+    }
+    for (const op of model.operations) {
+        for (const route of op.invalidates)
+            join(op.entity, model.byRoute.get(route)?.entity);
+        for (const effect of op.effects)
+            join(op.entity, effect.entity);
+        if (op.wait !== null)
+            join(op.entity, model.byOperationId.get(op.wait.operationId)?.entity);
+        if (op.async !== null) {
+            const poll = model.byOperationId.get(op.async.poll) ?? model.byRoute.get(op.async.poll);
+            join(op.entity, poll?.entity);
+        }
+    }
+    const lanes = new Map();
+    for (const entity of entities) {
+        const root = find(entity.name);
+        lanes.set(root, [...(lanes.get(root) ?? []), entity]);
+    }
+    return [...lanes.values()];
+}
 //# sourceMappingURL=run.js.map