@workerstack/testing 0.0.0-stage → 0.31.0

package/dist/api/runtime/checks/families/consistency/staleness-within-bound.js

@@ -0,0 +1,109 @@
+import { defineCheck } from "../../core/define.js";
+import { readCacheSignals } from "../../../../client/cache-signals.js";
+import { asserted, standDown } from "../../../finding.js";
+import { cannot, ready } from "../../core/plan.js";
+import { subjectsOf } from "../../planning/subjects.js";
+import { primedWrite, readAfter, writeKinds } from "./freshness-write.js";
+import { stalenessSchedule, watchCapMs } from "./staleness-schedule.js";
+import { SYSTEM_CLOCK, watchUntilFresh } from "./staleness-watch.js";
+const ID = "consistency.staleness-within-bound";
+/** The clock the watch keeps; a test swaps it for one that only moves when the watch sleeps. */
+export const STALENESS_TIMING = { clock: SYSTEM_CLOCK };
+/**
+ * After a write, how long a read answers the old value: the read as a client sends it, past no
+ * bypass, on `stalenessSchedule`. Past a declared `x-consistency.staleness` it is a finding; with
+ * none declared the window is only reported, so an API that promises nothing is never failed on it.
+ */
+export const stalenessWithinBound = defineCheck({
+    plan: (ctx) => {
+        const { createOp, readOp } = ctx;
+        if (createOp === undefined || readOp === undefined)
+            return cannot();
+        const kinds = writeKinds(ctx);
+        return kinds.length === 0
+            ? cannot("an update or a delete whose effect the read can show")
+            : ready({ kinds, readOp });
+    },
+    mutates: true,
+    id: ID,
+    needs: "a create, an item read, and an update or a delete",
+    subjects: subjectsOf.writeAndRead,
+    async run(ctx, { kinds, readOp }) {
+        const boundMs = readOp.consistency?.stalenessMs ?? null;
+        const measured = [];
+        let refused;
+        for (const kind of kinds) {
+            const step = await measure(ctx, { boundMs, kind, readOp });
+            if ("outcome" in step)
+                refused ??= step.outcome;
+            else
+                measured.push(step);
+        }
+        if (measured.length === 0)
+            return refused ?? standDown("no write could be measured");
+        return judge(ctx, measured, boundMs);
+    },
+});
+async function measure(ctx, ask) {
+    const written = await primedWrite(ctx, { check: ID, kind: ask.kind, readOp: ask.readOp });
+    if ("outcome" in written)
+        return written;
+    const capMs = watchCapMs(ask.boundMs);
+    let pastStatus = 0;
+    const bracket = await watchUntilFresh({
+        capMs,
+        clock: STALENESS_TIMING.clock,
+        fresh: written.shows,
+        read: () => ctx.client.get(written.path, { ...readAfter(written), as: { principal: ctx.actor.id }, cache: "use" }),
+        schedule: stalenessSchedule(capMs),
+        /* A stale read is a cache's doing only if a read past every cache shows the write. */
+        showsPastCache: async () => {
+            const past = await ctx.client.get(written.path, { ...readAfter(written), as: { principal: ctx.actor.id } });
+            pastStatus = past.status;
+            return written.shows(past);
+        },
+    });
+    if (bracket.neverTook) {
+        return {
+            outcome: standDown(`a read past the cache answers ${pastStatus} too, so the ${ask.kind} never showed and no cache is to blame`),
+        };
+    }
+    return { bracket, kind: ask.kind, path: written.path, write: written.write };
+}
+/** What a stale answer said about the cache that gave it. */
+function cacheEvidence(stale) {
+    if (stale === null)
+        return "";
+    const { ageS, status } = readCacheSignals(stale.responseHeaders);
+    const said = [
+        ...status.map((entry) => `Cache-Status ${entry.cache} ${entry.hit ? "hit" : `fwd=${entry.fwd ?? "?"}`}`),
+        ...(ageS === null ? [] : [`Age ${ageS}`]),
+    ];
+    return said.length === 0 ? "" : `; the stale answer said ${said.join(", ")}`;
+}
+function windowOf({ bracket, kind }) {
+    if (bracket.staleAtMs === null)
+        return `after the ${kind} the read showed it at once`;
+    if (bracket.freshAtMs === null)
+        return `after the ${kind} the read was still stale at ${bracket.staleAtMs}ms`;
+    return `after the ${kind} the read showed it at ${bracket.freshAtMs}ms (still stale at ${bracket.staleAtMs}ms)`;
+}
+function judge(ctx, measured, boundMs) {
+    const windows = measured.map(windowOf).join("; ");
+    if (boundMs === null)
+        return asserted(`${windows}; no staleness is declared, so the window is reported, not judged`);
+    const over = measured.find((m) => m.bracket.freshAtMs === null);
+    if (over === undefined)
+        return asserted(`${windows}, within the declared ${boundMs}ms`);
+    /* Stale at the longest watch, still short of a longer bound: noted, never judged past it. */
+    if ((over.bracket.staleAtMs ?? 0) < boundMs) {
+        return asserted(`${windows}, the longest the API tester watches; the declared ${boundMs}ms is not judged past it`);
+    }
+    return ctx.findings.backend(ID, ctx.entityName, {
+        detail: `${windowOf(over)}: GET ${over.path} answered the old ${over.kind === "delete" ? "record" : "value"} ` +
+            `past the ${boundMs}ms its x-consistency.staleness declares, no fresh read within it${cacheEvidence(over.bracket.stale)}`,
+        evidence: [over.write, ...(over.bracket.stale === null ? [] : [over.bracket.stale])],
+        summary: "a cached read stayed stale past the staleness it declares",
+    });
+}
+//# sourceMappingURL=staleness-within-bound.js.map

package/dist/api/runtime/checks/families/effects-async/async-reaches-terminal-state.d.ts

@@ -0,0 +1,3 @@
+import type { Check } from "../../core/check.ts";
+export declare const asyncReachesTerminalState: Check;
+//# sourceMappingURL=async-reaches-terminal-state.d.ts.map

package/dist/api/runtime/checks/families/effects-async/async-reaches-terminal-state.js

@@ -0,0 +1,161 @@
+import { defineCheck } from "../../core/define.js";
+import { driveAsync, inspectStreamAsync, matchesPredicate } from "../../../async.js";
+import { overBudget } from "../../../wait.js";
+import { fillPath } from "../../../fill-path.js";
+import { ASSERTED, asserted } from "../../../finding.js";
+import { receiptId } from "../../../proven-id.js";
+import { forEachInvocation } from "../../../upload-each.js";
+import { when } from "../../core/plan.js";
+import { encodeOpBody } from "../../payloads/body.js";
+import { standDownForFeatureGate } from "../../planning/stand-down.js";
+import { isEventStream, subjectsOf } from "../../planning/subjects.js";
+import { bodyForOp, forOperation, subject } from "./requests.js";
+export const asyncReachesTerminalState = defineCheck({
+    plan: (ctx) => when(ctx.asyncOps.length > 0),
+    mutates: true,
+    id: "async.reaches-terminal-state",
+    needs: "an operation declaring x-async",
+    subjects: subjectsOf.async,
+    async run(all) {
+        const notes = [];
+        for (const op of all.asyncOps) {
+            const ctx = forOperation(all, op);
+            const spec = op.async;
+            if (spec === null)
+                continue;
+            await forEachInvocation(op.operationId, ctx.uploads, (uploads, slot) => watchJob({ check: this.id, ctx, fixture: slot?.filename, notes, op, spec }, uploads));
+        }
+        return notes.length === 0 ? ASSERTED : asserted([...new Set(notes)].join("; "));
+    },
+});
+function about(run) {
+    return subject(run.ctx.entityName, run.op.operationId, run.fixture);
+}
+/** Starts one job and follows it — through its stream, or by polling — to a terminal state. */
+async function watchJob(run, uploads) {
+    const { ctx, op, spec } = run;
+    const start = await startJob(run, uploads);
+    if (start === undefined)
+        return;
+    const streamed = isEventStream(start, op);
+    const fromStream = streamed ? inspectStreamAsync(start.responseBody, spec) : null;
+    if (fromStream?.terminal !== null && fromStream?.terminal !== undefined) {
+        judgeStreamedTerminal(run, fromStream.terminal);
+        return;
+    }
+    const receipt = fromStream?.idRecord !== null && fromStream?.idRecord !== undefined ? fromStream.idRecord : start.responseBody;
+    if (streamed && fromStream !== null && (fromStream.id === undefined || fromStream.id === null)) {
+        ctx.findings.backend(run.check, about(run), {
+            summary: `${op.operationId} job disappeared before completing`,
+            detail: "the stream ended without a terminal frame and without a job id, so the poll route cannot be named.",
+            evidence: [start],
+            fixture: run.fixture,
+        });
+        return;
+    }
+    const outcome = await driveAsync({
+        client: ctx.client,
+        operation: (operationId) => ctx.model.byOperationId.get(operationId),
+        principal: ctx.actor.id,
+        receipt,
+        scope: ctx.scope,
+        spec,
+        start,
+    });
+    recordJob(run, receipt, start);
+    judgePolled(run, start, outcome);
+}
+/** The start request, or `undefined` once a start that began no job has been reported. */
+async function startJob(run, uploads) {
+    const { ctx, op } = run;
+    const body = op.hasRequestBody ? bodyForOp(ctx, op) : undefined;
+    const start = await ctx.client.request(op.method, fillPath(op.path, ctx.scope), {
+        as: { principal: ctx.actor.id },
+        ...(body === undefined ? {} : await encodeOpBody(ctx, op, body, { variant: "baseline", index: 0, uploads })),
+    });
+    if (standDownForFeatureGate(ctx, op, start, run.check))
+        return undefined;
+    /* A start that crashed started nothing: the lifecycle cannot be judged, and the crash
+     * itself is failed by the server-error rule. Say so rather than stand down quietly. */
+    if (start.status === 0 || start.status >= 500) {
+        ctx.findings.unresolved(run.check, about(run), `${op.operationId} answered ${start.status === 0 ? "nothing" : start.status}, so no job was started to watch`);
+        return undefined;
+    }
+    if (start.status >= 400) {
+        ctx.findings.gap(run.check, about(run), {
+            summary: `${op.operationId} could not be started`,
+            detail: `returned ${start.status}; the async lifecycle after it is untested`,
+            fixture: run.fixture,
+        });
+        return undefined;
+    }
+    return start;
+}
+function judgeStreamedTerminal(run, terminal) {
+    const { successWhen } = run.spec;
+    if (successWhen === undefined || matchesPredicate(terminal, successWhen))
+        return;
+    run.ctx.findings.gap(run.check, about(run), {
+        summary: `${run.op.operationId} reached a non-success terminal state`,
+        detail: `terminal state did not satisfy "${successWhen}"; downstream effects of this operation are untested`,
+        fixture: run.fixture,
+    });
+}
+/** The job this start created is a record like any other, and the API tester's to remove. */
+function recordJob(run, receipt, start) {
+    const { ctx, op, spec } = run;
+    const jobId = spec.idFrom === undefined ? undefined : receiptId(receipt, spec.idFrom);
+    const polled = (ctx.model.byRoute.get(spec.poll) ?? ctx.model.byOperationId.get(spec.poll))?.entity;
+    if (polled === null || polled === undefined)
+        return;
+    /* A receipt that names no id still made a job, and nothing can remove it. */
+    if (jobId === undefined)
+        ctx.recordUnidentified?.(polled, op.operationId, start);
+    else
+        ctx.recordCreated?.({ entity: polled, id: jobId, scope: { ...ctx.scope } }, start.principal);
+}
+function judgePolled(run, start, outcome) {
+    const { ctx, op, spec } = run;
+    if (outcome.state === "timed-out") {
+        ctx.findings.backend(run.check, about(run), {
+            summary: `${op.operationId} never reached a terminal state`,
+            detail: `polled ${spec.poll} ${outcome.polls} time(s) over ${Math.round(outcome.elapsedMs)}ms ` +
+                `without satisfying "${spec.until ?? "any terminal state"}". A job that neither ` +
+                "completes nor fails leaves callers polling forever.",
+            evidence: [start, ...outcome.exchanges.slice(-2)],
+            fixture: run.fixture,
+        });
+        return;
+    }
+    if (outcome.state === "unfillable") {
+        /* The API tester could not name the job to ask about it: nothing the backend did. */
+        ctx.findings.unresolved(run.check, about(run), `the poll route ${spec.poll} could not be filled from the receipt: ${outcome.unfillable}`);
+        return;
+    }
+    if (outcome.state === "vanished") {
+        ctx.findings.backend(run.check, about(run), {
+            summary: `${op.operationId} job disappeared before completing`,
+            detail: `the poll route stopped serving the job after ${outcome.polls} poll(s). A job that ` +
+                "vanishes is indistinguishable from one that never existed.",
+            evidence: [start, ...outcome.exchanges.slice(-2)],
+            fixture: run.fixture,
+        });
+        return;
+    }
+    if (outcome.succeeded) {
+        const late = overBudget(run.op.operationId, {
+            elapsedMs: outcome.elapsedMs,
+            kind: "x-async",
+            timeoutMs: run.spec.timeoutMs,
+        });
+        if (late !== undefined)
+            run.notes.push(late);
+        return;
+    }
+    ctx.findings.gap(run.check, about(run), {
+        summary: `${op.operationId} reached a non-success terminal state`,
+        detail: `terminal state did not satisfy "${outcome.unmet}"; downstream effects of this operation are untested`,
+        fixture: run.fixture,
+    });
+}
+//# sourceMappingURL=async-reaches-terminal-state.js.map

package/dist/api/runtime/checks/families/effects-async/async-receipt-identifies-the-job.d.ts

@@ -0,0 +1,3 @@
+import type { Check } from "../../core/check.ts";
+export declare const asyncReceiptIsResolvable: Check;
+//# sourceMappingURL=async-receipt-identifies-the-job.d.ts.map

package/dist/api/runtime/checks/families/effects-async/async-receipt-identifies-the-job.js

@@ -0,0 +1,54 @@
+import { defineCheck } from "../../core/define.js";
+import { resolveAsyncId } from "../../../async.js";
+import { fillPath } from "../../../fill-path.js";
+import { ASSERTED } from "../../../finding.js";
+import { when } from "../../core/plan.js";
+import { subjectsOf } from "../../planning/subjects.js";
+import { forOperation } from "./requests.js";
+export const asyncReceiptIsResolvable = defineCheck({
+    plan: (ctx) => when(ctx.asyncOps.some((op) => op.async?.idFrom !== undefined)),
+    id: "async.receipt-identifies-the-job",
+    judgesTranscript: true,
+    needs: "x-async with an idFrom pointer",
+    subjects: subjectsOf.asyncReceipt,
+    async run(all) {
+        for (const op of all.asyncOps) {
+            const ctx = forOperation(all, op);
+            const spec = op.async;
+            if (spec?.idFrom === undefined)
+                continue;
+            /* Match the async operation's own exchange by resolved path — any POST would do here
+             * otherwise, and a sibling create's response would be inspected instead. */
+            let resolved;
+            try {
+                resolved = fillPath(op.path, ctx.scope);
+            }
+            catch {
+                continue;
+            }
+            const attempts = ctx.client
+                .exchangesFor(op.operationId)
+                .filter((e) => ctx.client.relativePath(e.url) === resolved);
+            const started = attempts.find((e) => e.status > 0 && e.status < 300);
+            if (started === undefined) {
+                /* The action never answered with a receipt: say so rather than pass in silence. */
+                const refused = attempts.at(-1);
+                if (refused !== undefined) {
+                    ctx.findings.unresolved(this.id, ctx.entityName, `${op.operationId} answered ${refused.status === 0 ? "nothing" : refused.status}, so there was no receipt to read`);
+                }
+                continue;
+            }
+            const node = resolveAsyncId((await ctx.client.hydrate(started)).responseBody, spec.idFrom);
+            if (node !== undefined && node !== null)
+                continue;
+            ctx.findings.spec(this.id, ctx.entityName, {
+                summary: `${op.operationId} response does not carry the job identifier it declares`,
+                detail: `x-async names "${spec.idFrom}" as the job id, but the response has no such value. ` +
+                    "Callers cannot poll for a job they cannot name.",
+                evidence: [started],
+            });
+        }
+        return ASSERTED;
+    },
+});
+//# sourceMappingURL=async-receipt-identifies-the-job.js.map

package/dist/api/runtime/checks/families/effects-async/effect-verdicts.d.ts

@@ -0,0 +1,27 @@
+import type { EffectSpec } from "../../../../spec/extensions/effects.ts";
+import type { OperationModel } from "../../../../spec/graph/types.ts";
+import type { Exchange } from "../../../exchange.ts";
+import type { WriteContext } from "../../core/context.ts";
+import { type Observation } from "./observe.ts";
+/** Where findings about one invocation of one action land. */
+export interface Reporter {
+    check: string;
+    ctx: WriteContext;
+    op: OperationModel;
+    fixture: string | undefined;
+}
+/** One action that succeeded, with the baselines its effects are judged against. */
+export interface ActionRun extends Reporter {
+    effects: readonly EffectSpec[];
+    befores: ReadonlyMap<string, Observation>;
+    invoked: Exchange;
+    /** Path parameters bound so far; a created parent's id is added as it is learned. */
+    scope: Record<string, string>;
+}
+/** A gap on the invocation's subject. */
+export declare function reportGap(run: Reporter, title: string, detail: string): void;
+/** `missing {a}, {b}`: the parameters `scope` cannot fill. */
+export declare function missingParams(params: readonly string[], scope: Record<string, string>): string;
+/** Judges every declared effect of a succeeded action. */
+export declare function judgeEffects(action: ActionRun): Promise<void>;
+//# sourceMappingURL=effect-verdicts.d.ts.map

package/dist/api/runtime/checks/families/effects-async/effect-verdicts.js

@@ -0,0 +1,133 @@
+import { describeEffectHold, effectHolds } from "../../../effects/cardinality.js";
+import { bindAfterCreateEffects, bindCreatedScope, identityPathParam } from "../../../effects/scope.js";
+import { listedIds } from "../../../proven-id.js";
+import { listOpFor, observe } from "./observe.js";
+import { subject } from "./requests.js";
+/** A gap on the invocation's subject. */
+export function reportGap(run, title, detail) {
+    run.ctx.findings.gap(run.check, subject(run.ctx.entityName, run.op.operationId, run.fixture), {
+        summary: title,
+        detail,
+        fixture: run.fixture,
+    });
+}
+/** `missing {a}, {b}`: the parameters `scope` cannot fill. */
+export function missingParams(params, scope) {
+    const missing = params.filter((name) => scope[name] === undefined);
+    return `missing ${missing.map((name) => `{${name}}`).join(", ")}`;
+}
+/** Judges every declared effect of a succeeded action. */
+export async function judgeEffects(action) {
+    const deltas = new Map();
+    Object.assign(action.scope, bindAfterCreateEffects(action.ctx.model, action.effects, { body: action.invoked.responseBody, deltas }));
+    await bindCreatedParents(action, deltas);
+    for (const effect of action.effects)
+        await judgeEffect(action, effect);
+}
+/**
+ * A created parent the response does not name is found in its collection's delta, so a later
+ * effect's list path — the parent's children — can be filled.
+ */
+async function bindCreatedParents(action, deltas) {
+    const { ctx, scope } = action;
+    for (const effect of action.effects) {
+        if (effect.op !== "create")
+            continue;
+        const param = identityPathParam(ctx.model, effect.entity);
+        if (scope[param] !== undefined)
+            continue;
+        const listOp = listOpFor(ctx, effect.entity);
+        if (listOp === undefined)
+            continue;
+        const afterParent = await observe(ctx, listOp, scope);
+        if (afterParent.status !== "ok")
+            continue;
+        const before = action.befores.get(effect.entity);
+        const prior = before?.status === "ok" ? before.ids : [];
+        const added = afterParent.ids.filter((id) => !prior.includes(id));
+        deltas.set(effect.entity, added);
+        Object.assign(scope, bindCreatedScope(ctx.model, effect.entity, { added, body: action.invoked.responseBody }));
+    }
+}
+async function judgeEffect(action, effect) {
+    const { ctx, op, scope, invoked } = action;
+    const after = await observeAfter(action, effect);
+    if (after === undefined)
+        return;
+    const before = action.befores.get(effect.entity);
+    if (before?.status === "error") {
+        reportGap(action, `${op.operationId} could not observe "${effect.entity}" before the write`, `${before.reason}; no baseline, so the effect is unverified`);
+        return;
+    }
+    const prior = before?.status === "ok" ? before.ids : [];
+    const added = after.ids.filter((id) => !prior.includes(id));
+    const removed = prior.filter((id) => !after.ids.includes(id));
+    /* Records the operation made on the API tester's behalf are the API tester's to remove. */
+    if (effect.op === "create" || effect.op === "append") {
+        for (const id of listedIds(added))
+            ctx.recordCreated?.({ entity: effect.entity, id, scope: { ...scope } }, invoked.principal);
+    }
+    if (effectHolds(effect, { added: added.length, delta: after.ids.length - prior.length, removed: removed.length })) {
+        if (effect.op === "create") {
+            Object.assign(scope, bindCreatedScope(ctx.model, effect.entity, { added, body: invoked.responseBody }));
+        }
+        return;
+    }
+    reportUnheld(action, effect, { added: added.length, after, prior: prior.length, removed: removed.length });
+}
+/** The effect's collection after the write, or `undefined` once a collection that could not be read is reported. */
+async function observeAfter(action, effect) {
+    const { ctx, op, scope } = action;
+    const listOp = listOpFor(ctx, effect.entity);
+    if (listOp === undefined) {
+        reportGap(action, `${op.operationId} declares an effect on "${effect.entity}", which has no list route`, "the effect cannot be observed, so it is not verified");
+        return undefined;
+    }
+    const after = await observe(ctx, listOp, scope);
+    if (after.status === "unfillable") {
+        reportGap(action, `${op.operationId} declares an effect on "${effect.entity}", but the list path cannot be filled`, `${missingParams(listOp.pathParams, scope)} after the write; the effect is unverified`);
+        return undefined;
+    }
+    if (after.status === "error") {
+        reportGap(action, `${op.operationId} could not observe "${effect.entity}" after the write`, `${after.reason}; the declared effect is unverified`);
+        return undefined;
+    }
+    return after;
+}
+/** A declared effect that did not hold: the collection changed by other than what was declared. */
+function reportUnheld(action, effect, sizes) {
+    const { ctx, op, invoked, fixture } = action;
+    const where = subject(ctx.entityName, op.operationId, fixture);
+    const evidence = [invoked, sizes.after.exchange];
+    const now = sizes.after.ids.length;
+    if (effect.op === "create" || effect.op === "append") {
+        ctx.findings.backend(action.check, where, {
+            summary: `${op.operationId} did not produce the "${effect.entity}" records it declares`,
+            detail: `x-effects declares ${describeEffectHold(effect)} on "${effect.entity}", but the ` +
+                `collection went from ${sizes.prior} to ${now} ` +
+                `(${sizes.added} added, ${sizes.removed} removed). A declared effect that does ` +
+                "not occur means callers cannot rely on the operation having done anything.",
+            evidence,
+            fixture,
+        });
+        return;
+    }
+    if (effect.op === "delete") {
+        ctx.findings.backend(action.check, where, {
+            summary: `${op.operationId} did not remove the "${effect.entity}" records it declares`,
+            detail: `x-effects declares ${describeEffectHold(effect)} on "${effect.entity}", but the collection ` +
+                `went from ${sizes.prior} to ${now}`,
+            evidence,
+            fixture,
+        });
+        return;
+    }
+    ctx.findings.backend(action.check, where, {
+        summary: `${op.operationId} changed the size of "${effect.entity}" while declaring ${effect.op}`,
+        detail: `x-effects declares ${effect.op}, which must not add or remove records, but the ` +
+            `collection went from ${sizes.prior} to ${now}`,
+        evidence,
+        fixture,
+    });
+}
+//# sourceMappingURL=effect-verdicts.js.map

package/dist/api/runtime/checks/families/effects-async/effects-declared-effect-occurs.d.ts

@@ -0,0 +1,3 @@
+import type { Check } from "../../core/check.ts";
+export declare const declaredEffectsOccur: Check;
+//# sourceMappingURL=effects-declared-effect-occurs.d.ts.map