cloving 1.3.0 → 1.5.0

package/dist/src/broker/actor/projection-memory.d.ts

@@ -0,0 +1,43 @@
+import type { Database, Logger } from "solid-objects";
+import { type EffectIntent } from "../../authority/reducer.ts";
+import { type ProjectionSnapshot } from "../../storage/decision-snapshots.ts";
+import type { DomainChange } from "../../storage/domain-changes.ts";
+import { type StoredState } from "../authority-restore.ts";
+import type { Sequences } from "./authority-queries.ts";
+export interface OperationRecord {
+    readonly operationId: string;
+    readonly kind: EffectIntent["kind"];
+    readonly runId: string;
+    readonly createdSequence: number;
+    readonly releasedSequence: number | null;
+    readonly deliveries: readonly {
+        readonly effectId: string;
+        readonly outcome: string | null;
+        readonly receipt: string | null;
+        readonly purpose: string;
+        readonly epoch: number | null;
+    }[];
+}
+export interface Projection {
+    readonly snapshot: ProjectionSnapshot;
+    readonly state: () => StoredState;
+    readonly eligible: () => readonly EffectIntent[];
+    readonly operations: () => readonly OperationRecord[];
+    readonly sequences: () => Sequences;
+    readonly workflowIds: (runIds?: readonly string[]) => Readonly<Record<string, string>>;
+}
+type Source = Pick<Database, "connection">;
+export interface ProjectionMemory {
+    readonly record: (change: RecordedChange) => void;
+    readonly load: (database: Source) => Promise<void>;
+    readonly read: (database: Source) => Promise<Projection>;
+}
+export interface RecordedChange {
+    readonly messageId: string;
+    readonly change: DomainChange;
+}
+export declare function projectionMemory(options: {
+    readonly logger: Logger;
+}): ProjectionMemory;
+export declare function operationRecords(snapshot: Pick<ProjectionSnapshot, "outbox" | "deliveries">): readonly OperationRecord[];
+export {};

package/dist/src/broker/actor/projection-memory.js

@@ -0,0 +1,207 @@
+import { eligibleIntents } from "../../authority/reducer.js";
+import { loadProjections, } from "../../storage/decision-snapshots.js";
+import { messageOutcomes, recordedTimes, runTimes, storedWorkflows, submittedWorkflows, } from "../../storage/projection-lookups.js";
+import { changedProjection, followingSequence, projectedChange, } from "../../storage/projection-changes.js";
+import { currentState, intentKind, } from "../authority-restore.js";
+const pendingLimit = 1_000;
+export function projectionMemory(options) {
+    const memory = {
+        held: undefined,
+        stale: null,
+        recorded: new Map(),
+        logger: options.logger,
+    };
+    return {
+        record: ({ messageId, change }) => {
+            memory.recorded.delete(messageId);
+            memory.recorded.set(messageId, projectedChange(change));
+            if (memory.recorded.size <= pendingLimit)
+                return;
+            memory.recorded.clear();
+            memory.held = undefined;
+        },
+        load: (database) => database.connection(async (connection) => {
+            memory.recorded.clear();
+            memory.held = await rebuilt(connection);
+        }),
+        read: (database) => database.connection(async (connection) => {
+            const held = await advanced(memory, connection);
+            return held.projection;
+        }),
+    };
+}
+async function advanced(memory, connection) {
+    const recorded = [...memory.recorded];
+    memory.recorded.clear();
+    try {
+        memory.held = await updated(memory, { recorded, connection });
+    }
+    catch (error) {
+        memory.held = undefined;
+        memory.stale = `update failed: ${String(Reflect.get(Object(error), "message"))}`;
+        throw error;
+    }
+    return memory.held;
+}
+async function updated(memory, update) {
+    const { recorded, connection } = update;
+    if (memory.held === undefined)
+        return rebuiltAfter(memory, connection);
+    if (recorded.length === 0)
+        return memory.held;
+    const outcomes = await messageOutcomes(connection, recorded.map(([messageId]) => messageId));
+    const committed = [];
+    let sequence = memory.held.snapshot.sequence;
+    for (const [messageId, change] of recorded) {
+        const outcome = outcomes.get(messageId) ?? "missing";
+        if (outcome === "rolled-back")
+            continue;
+        const reason = mismatch({ messageId, change, outcome, sequence });
+        if (reason !== null) {
+            memory.stale = reason;
+            return rebuiltAfter(memory, connection);
+        }
+        committed.push(change);
+        sequence = followingSequence(sequence, change);
+    }
+    return applied(memory.held, { changes: committed, connection });
+}
+async function rebuiltAfter(memory, connection) {
+    const held = await rebuilt(connection);
+    if (memory.stale !== null)
+        memory.logger.warn({
+            event: "authority-projection-rebuilt",
+            reason: memory.stale,
+        });
+    memory.stale = null;
+    return held;
+}
+function mismatch(recorded) {
+    const { messageId, change, outcome, sequence } = recorded;
+    if (outcome === "missing")
+        return `message ${messageId} is not stored`;
+    if (change.previous === sequence)
+        return null;
+    return `sequence ${String(change.previous)} does not follow ${String(sequence)}`;
+}
+async function rebuilt(connection) {
+    const snapshot = await loadProjections(connection);
+    return withProjection({
+        snapshot,
+        times: await runTimes(connection),
+        workflows: new Map([
+            ...snapshot.details.flatMap((row) => row.submission === null ? [] : [[row.submission, null]]),
+            ...(await submittedWorkflows(connection)),
+        ]),
+    });
+}
+async function applied(previous, input) {
+    const { changes } = input;
+    if (changes.length === 0)
+        return previous;
+    return withProjection({
+        snapshot: changedProjection(previous.snapshot, changes),
+        times: await changedTimes(previous.times, input),
+        workflows: await changedWorkflows(previous.workflows, input),
+    });
+}
+async function changedTimes(previous, input) {
+    const runs = new Map(input.changes.flatMap((change) => change.runs.map((row) => [row.runId, row.sequence])));
+    const carried = new Map(input.changes.flatMap((change) => change.events.map((event) => [event.sequence, event.recordedAt])));
+    const stored = await recordedTimes(input.connection, [
+        ...new Set([...runs.values()].filter((sequence) => !carried.has(sequence))),
+    ]);
+    const times = new Map(previous);
+    for (const [runId, sequence] of runs) {
+        const time = carried.get(sequence) ?? stored.get(sequence);
+        if (time === undefined)
+            times.delete(runId);
+        else
+            times.set(runId, time);
+    }
+    return times;
+}
+async function changedWorkflows(previous, input) {
+    const submissions = [
+        ...new Set(input.changes.flatMap((change) => change.details.flatMap((row) => row.submission === null || previous.has(row.submission)
+            ? []
+            : [row.submission]))),
+    ];
+    const stored = await storedWorkflows(input.connection, submissions);
+    return new Map([
+        ...previous,
+        ...submissions.map((submission) => [submission, stored.get(submission) ?? null]),
+    ]);
+}
+function withProjection(parts) {
+    return { ...parts, projection: projection(parts) };
+}
+function remembered(compute) {
+    let value;
+    return () => {
+        value ??= { current: compute() };
+        return value.current;
+    };
+}
+function projection(parts) {
+    const { snapshot, times, workflows } = parts;
+    const state = remembered(() => currentState(snapshot));
+    return {
+        snapshot,
+        state,
+        eligible: remembered(() => eligible({ snapshot, state: state() })),
+        operations: remembered(() => operationRecords(snapshot)),
+        sequences: remembered(() => sequences({ snapshot, times })),
+        workflowIds: (runIds) => workflowIds({ snapshot, workflows, runIds }),
+    };
+}
+function sequences(parts) {
+    const timed = parts.snapshot.runs.flatMap((row) => {
+        const time = parts.times.get(row.runId);
+        return time === undefined ? [] : [{ row, time }];
+    });
+    return {
+        latest: parts.snapshot.sequence,
+        runs: Object.fromEntries(timed.map(({ row }) => [row.runId, row.sequence])),
+        times: Object.fromEntries(timed.map(({ row, time }) => [row.runId, time])),
+    };
+}
+function workflowIds(parts) {
+    const wanted = parts.runIds === undefined ? null : new Set(parts.runIds);
+    return Object.fromEntries(parts.snapshot.details.flatMap((row) => {
+        const workflowId = row.submission === null ? null : parts.workflows.get(row.submission);
+        if (typeof workflowId !== "string")
+            return [];
+        return wanted === null || wanted.has(row.runId)
+            ? [[row.runId, workflowId]]
+            : [];
+    }));
+}
+export function operationRecords(snapshot) {
+    const deliveries = Map.groupBy(snapshot.deliveries, (row) => row.operationId);
+    return snapshot.outbox.map((row) => ({
+        operationId: row.operationId,
+        kind: intentKind(row.kind),
+        runId: row.runId,
+        createdSequence: row.createdSequence,
+        releasedSequence: row.releasedSequence,
+        deliveries: (deliveries.get(row.operationId) ?? []).map(({ effectId, outcome, receipt, purpose, epoch }) => ({
+            effectId,
+            outcome,
+            receipt,
+            purpose,
+            epoch,
+        })),
+    }));
+}
+function eligible(parts) {
+    return eligibleIntents({
+        ...parts.state,
+        effects: {
+            operations: Object.fromEntries(parts.snapshot.pending.map((operationId) => [
+                operationId,
+                { status: "dispatch-pending" },
+            ])),
+        },
+    });
+}

package/dist/src/broker/actor/runtime-parts.d.ts

@@ -1,4 +1,5 @@
 import { type ActorClass, type Database, type InstrumentationEvent, type SolidObjectsConfiguration, type SolidObjectsRuntime } from "solid-objects";
+import { type DomainChange } from "../../storage/domain-changes.ts";
 import { type AuthorityTiming } from "./authority-settings.ts";
 import type { StrandingWatchdog } from "./authority-watchdog.ts";
 export interface FailedDelivery {
@@ -20,4 +21,8 @@ export declare function registeredRuntime(registration: {
     readonly database: Database;
     readonly actor: ActorClass;
     readonly settings: Partial<SolidObjectsConfiguration>;
+    readonly record: (recorded: {
+        readonly messageId: string;
+        readonly change: DomainChange;
+    }) => void;
 }): SolidObjectsRuntime;

package/dist/src/broker/actor/runtime-parts.js

@@ -1,5 +1,5 @@
 import { createRuntime, } from "solid-objects";
-import { persistChange } from "../../storage/domain-changes.js";
+import { persistChange, } from "../../storage/domain-changes.js";
 import { persistAction } from "./authority-actor.js";
 import { brokerAuthorization } from "./authority-policy.js";
 import { librarySettings } from "./authority-settings.js";
@@ -26,9 +26,14 @@ export function runtimeSettings(parts) {
     };
 }
 export function registeredRuntime(registration) {
-    const { database, actor, settings } = registration;
+    const { database, actor, settings, record } = registration;
     const runtime = createRuntime({ database, ...settings });
     runtime.register(actor);
-    runtime.registerCommitAction(persistAction, (change, context) => persistChange(change, context.connection));
+    runtime.registerCommitAction(persistAction, async (change, context) => {
+        record({
+            messageId: context.messageId,
+            change: await persistChange(change, context.connection),
+        });
+    });
     return runtime;
 }

package/dist/src/broker/actor/stop-plan.js

@@ -1,3 +1,4 @@
+import { endedCommandWaitCheck } from "../../authority/command-waits.js";
 import { stopAttempts, } from "../../authority/stop-sequence.js";
 import { observingIntent, protectedIntent, } from "./delivery-intents.js";
 function present(state, planning) {
@@ -33,9 +34,14 @@ export function presentSubjects(state, planning) {
 }
 export function stopIntents(state, planning) {
     const { runId, stopping } = planning;
-    if (state.runs[runId]?.status !== "cancelled")
-        return [];
-    const subjects = presentSubjects(state, planning);
+    const cancelled = state.runs[runId]?.status === "cancelled";
+    const subjects = presentSubjects(state, planning).flatMap((subject) => {
+        if (endedCommandWaitCheck(state.commandWaits ?? {}, subject.operationId))
+            return [
+                { ...subject, stopSequence: subject.stopSequence ?? state.sequence },
+            ];
+        return cancelled ? [subject] : [];
+    });
     return stopAttempts(subjects, {
         cancelledSequence: planning.cancelledNow[0] ?? stopping.cancelledSequence ?? 0,
         stopLimit: planning.stopLimit,

package/dist/src/broker/actor/stop-turn.d.ts

@@ -1,4 +1,5 @@
 import type { StopSubject } from "../../authority/stop-sequence.ts";
+import type { CommandWaitsState } from "../../authority/command-waits.ts";
 import { type SqlConnection } from "../../storage/domain-rows.ts";
 export interface Stopping {
     readonly subjects: readonly StopSubject[];
@@ -10,6 +11,7 @@ export declare function turnStopping(connection: SqlConnection, scope: {
     readonly runId: string;
     readonly status: string | undefined;
     readonly kind: string;
+    readonly commandWaits?: CommandWaitsState;
 }): Promise<Stopping>;
 export declare function droppedRuns(connection: SqlConnection, rule: {
     readonly limit: number;

package/dist/src/broker/actor/stop-turn.js

@@ -11,23 +11,36 @@ function attemptOf(row) {
 const closed = "NOT EXISTS (SELECT 1 FROM deliveries x WHERE x.operation_id = o.operation_id AND x.outcome IS NULL)";
 const reported = "EXISTS (SELECT 1 FROM stopped_operations s WHERE s.operation_id = o.operation_id)";
 const finalStatuses = "'refused', 'completed', 'failed', 'stopped', 'cancelled'";
+const waitStopSequence = "(SELECT min(e.sequence) FROM command_waits w JOIN events e ON e.run_id = w.run_id WHERE w.check_operation = o.operation_id AND w.state != 'waiting' AND (e.type IN ('run.completed', 'run.failed', 'run.cancelled') OR (json_extract(e.payload, '$.waitName') = w.name AND (e.type = 'command-wait.expired' OR (e.type = 'command-wait.opened' AND round(unixepoch(e.recorded_at, 'subsec') * 1000) >= w.deadline)))))";
 export async function stopping(connection, runId) {
-    const rows = await connection.all(`SELECT o.operation_id, o.kind, d.effect_id, coalesce(a.purpose, 'dispatch') AS purpose, d.outcome, d.staged_sequence, p.status, p.receipt, ${reported} AS stopped FROM outbox o JOIN deliveries d ON d.operation_id = o.operation_id LEFT JOIN dispatch_attempts a ON a.effect_id = d.effect_id LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = ? AND ((coalesce(p.status, '') NOT IN (${finalStatuses}) AND NOT ${reported}) OR NOT ${closed}) ORDER BY d.staged_sequence, d.rowid`, [runId]);
+    const rows = await connection.all(`SELECT o.operation_id, o.kind, d.effect_id, coalesce(a.purpose, 'dispatch') AS purpose, d.outcome, d.staged_sequence, p.status, p.receipt, ${reported} AS stopped, ${waitStopSequence} AS stop_sequence FROM outbox o JOIN deliveries d ON d.operation_id = o.operation_id LEFT JOIN dispatch_attempts a ON a.effect_id = d.effect_id LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = ? AND ((coalesce(p.status, '') NOT IN (${finalStatuses}) AND NOT ${reported}) OR NOT ${closed}) ORDER BY d.staged_sequence, d.rowid`, [runId]);
     const grouped = Map.groupBy(rows, (row) => text(row, "operation_id"));
     return {
-        subjects: [...grouped].map(([operationId, attempts]) => ({
-            operationId,
-            kind: text(attempts[0], "kind"),
-            attempts: attempts.map(attemptOf),
-            status: optionalText(attempts[0], "status"),
-            receipt: optionalText(attempts[0], "receipt"),
-            stopped: integer(attempts[0], "stopped") === 1,
-        })),
+        subjects: [...grouped].map(([operationId, attempts]) => {
+            const stopSequence = optionalInteger(attempts[0], "stop_sequence");
+            return {
+                operationId,
+                kind: text(attempts[0], "kind"),
+                attempts: attempts.map(attemptOf),
+                status: optionalText(attempts[0], "status"),
+                receipt: optionalText(attempts[0], "receipt"),
+                stopped: integer(attempts[0], "stopped") === 1,
+                ...(stopSequence === null ? {} : { stopSequence }),
+            };
+        }),
         cancelledSequence: optionalInteger(await connection.get("SELECT max(sequence) AS sequence FROM events WHERE run_id = ? AND type = 'run.cancelled'", [runId]), "sequence"),
     };
 }
 export function turnStopping(connection, scope) {
-    const dropping = scope.status === "cancelled" || scope.kind === "cancel-run";
+    const endingWait = Object.values(scope.commandWaits ?? {}).some((wait) => wait.checkOperation !== null &&
+        (wait.state !== "waiting" ||
+            [
+                "expire-command-wait",
+                "open-command-wait",
+                "complete-run",
+                "fail-run",
+            ].includes(scope.kind)));
+    const dropping = scope.status === "cancelled" || scope.kind === "cancel-run" || endingWait;
     return dropping
         ? stopping(connection, scope.runId)
         : Promise.resolve(noStopping);
@@ -37,6 +50,6 @@ const last = "(SELECT d.outcome FROM deliveries d WHERE d.operation_id = o.opera
 const effectDue = `o.kind = 'protected-effect' AND p.status NOT IN (${finalStatuses}) AND (p.receipt IS NULL OR ?1 = 1 OR p.status IN ('recovering', 'blocked'))`;
 const runDue = `o.kind != 'protected-effect' AND NOT ${reported} AND ((${sent} AND ?1 = 1) OR (NOT ${sent} AND ${last} NOT IN ('withheld', 'blocked', 'absent')))`;
 export async function droppedRuns(connection, rule) {
-    const rows = await connection.all(`SELECT r.run_id FROM runs r WHERE r.status = 'cancelled' AND EXISTS (SELECT 1 FROM outbox o LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = r.run_id AND ${closed} AND ((${effectDue}) OR (${runDue}))) ORDER BY r.run_id LIMIT ?2`, [rule.stopSupported ? 1 : 0, rule.limit]);
+    const rows = await connection.all(`SELECT r.run_id FROM runs r WHERE EXISTS (SELECT 1 FROM outbox o LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = r.run_id AND (r.status = 'cancelled' OR EXISTS (SELECT 1 FROM command_waits w WHERE w.run_id = r.run_id AND w.check_operation = o.operation_id AND w.state != 'waiting')) AND ${closed} AND ((${effectDue}) OR (${runDue}))) ORDER BY r.run_id LIMIT ?2`, [rule.stopSupported ? 1 : 0, rule.limit]);
     return rows.map((row) => text(row, "run_id"));
 }

package/dist/src/broker/actor/turn-draft.d.ts

@@ -11,6 +11,7 @@ export interface ChangeDraft {
     readonly released: readonly Row[];
     readonly closed: readonly Row[];
     readonly gates: readonly Row[];
+    readonly waits: readonly Row[];
     readonly details: readonly Row[];
     readonly documents: readonly Row[];
     readonly budgets: readonly Row[];

package/dist/src/broker/actor/turn-draft.js

@@ -1,3 +1,4 @@
+import { commandWaitRows } from "../command-wait-restore.js";
 import { evidenceOf } from "../../authority/operations.js";
 import { detailRecord, gateRecord } from "../decision-restore.js";
 import { effectRows } from "../effect-restore.js";
@@ -23,6 +24,7 @@ export function unchangedAt(sequence, closed) {
         released: [],
         closed,
         gates: [],
+        waits: [],
         details: [],
         documents: [],
         ...unchangedEffects,
@@ -78,6 +80,7 @@ export function draft(result, extra) {
         closed,
         documents,
         ...effectDraft(result, before),
+        waits: commandWaitRows(changedEntries(state.commandWaits ?? {}, before.commandWaits ?? {})).map(columns),
         gates: changed(state.gates, before.gates).map((gate) => columns(gateRecord(gate))),
         details: changed(state.details, before.details).map((details) => columns(detailRecord(details))),
         previous: before.sequence,
@@ -127,6 +130,7 @@ export function merged(first, later) {
         released: joined("released"),
         closed: joined("closed"),
         gates: joined("gates"),
+        waits: joined("waits"),
         details: joined("details"),
         documents: joined("documents"),
         budgets: joined("budgets"),

package/dist/src/broker/actor/turn-state.js

@@ -1,3 +1,4 @@
+import { readCommandWaits } from "../command-wait-restore.js";
 import { parseCommand } from "../../authority/commands.js";
 import { readExecutionManifest } from "../../execution-pin.js";
 import { operationScope } from "../../operation-identity.js";
@@ -92,7 +93,7 @@ export async function recoverableIntents(connection) {
     return [...intents, ...(await uncertainEffects(connection))];
 }
 async function uncertainEffects(connection) {
-    const rows = await connection.all("SELECT p.operation_id, p.run_id, r.workflow FROM operations p JOIN runs r ON r.run_id = p.run_id WHERE p.status IN ('recovering', 'blocked') AND r.status != 'cancelled' AND NOT EXISTS (SELECT 1 FROM deliveries d WHERE d.operation_id = p.operation_id AND d.outcome IS NULL) ORDER BY p.requested_sequence LIMIT ?", [recoveryBatch]);
+    const rows = await connection.all("SELECT p.operation_id, p.run_id, r.workflow FROM operations p JOIN runs r ON r.run_id = p.run_id WHERE p.status IN ('recovering', 'blocked') AND r.status != 'cancelled' AND NOT EXISTS (SELECT 1 FROM command_waits w WHERE w.check_operation = p.operation_id AND w.state != 'waiting') AND NOT EXISTS (SELECT 1 FROM deliveries d WHERE d.operation_id = p.operation_id AND d.outcome IS NULL) ORDER BY p.requested_sequence LIMIT ?", [recoveryBatch]);
     const intents = [];
     for (const row of rows)
         intents.push(await pendingEffect(connection, {
@@ -161,8 +162,10 @@ async function turnState(connection, scope) {
         sql: "SELECT operation_id, kind, run_id, created_sequence, released_sequence FROM outbox WHERE run_id = ? AND released_sequence IS NULL ORDER BY created_sequence, rowid",
         parameters: [runId],
     });
+    const waits = await readCommandWaits(connection, runId);
     return {
         state: {
+            ...waits,
             sequence,
             runs: { ...near.runs, ...(stored ? { [runId]: stored } : {}) },
             commands: recorded ? { [commandId]: recorded } : {},
@@ -178,6 +181,7 @@ async function turnState(connection, scope) {
             runId,
             status: stored?.status,
             kind: scope.kind,
+            ...waits,
         }),
         runId,
         created: recorded

package/dist/src/broker/authority-admission.js

@@ -1,3 +1,5 @@
+import { commandWaitKey } from "../authority/command-waits.js";
+import { documentDigest } from "./decision-restore.js";
 import { parseCommand } from "../authority/commands.js";
 import { executionCapabilities, readExecutionManifest, resumePinned, } from "../execution-pin.js";
 import { effectInputDigest } from "./effect-restore.js";
@@ -41,6 +43,61 @@ const commandEffect = (command, context) => {
         return { kind: "rejected", reason: "command-not-declared" };
     return unpinned(command, context);
 };
+function waitMatches(command, wait) {
+    return (command.target === wait.command &&
+        command.maximumWaitSeconds === wait.maximumWaitSeconds &&
+        command.pollEverySeconds === wait.pollEverySeconds);
+}
+const commandWait = (command, context) => {
+    const pin = context.state.runs[command.runId]?.workflow ?? "";
+    const declaration = context.definitions.get(pin);
+    const name = String(command.waitName);
+    const waits = declaration?.commandWaits ?? {};
+    const wait = Object.hasOwn(waits, name) ? waits[name] : undefined;
+    if (wait === undefined || !waitMatches(command, wait))
+        return { kind: "rejected", reason: "command-wait-not-declared" };
+    return unpinned(command, context);
+};
+function declaredWaitCheck(command, context) {
+    const pin = context.state.runs[command.runId]?.workflow ?? "";
+    const declarations = context.definitions.get(pin)?.commandWaits ?? {};
+    const name = String(command.waitName);
+    const declaration = Object.hasOwn(declarations, name)
+        ? declarations[name]
+        : undefined;
+    const wait = context.state.commandWaits?.[commandWaitKey(command.runId, name)];
+    if (declaration === undefined || wait === undefined)
+        return undefined;
+    return waitMatches({
+        ...command,
+        target: wait.command,
+        maximumWaitSeconds: wait.maximumWaitSeconds,
+        pollEverySeconds: wait.pollEverySeconds,
+    }, declaration)
+        ? wait
+        : undefined;
+}
+const commandWaitCheck = (command, context) => {
+    const wait = declaredWaitCheck(command, context);
+    if (wait === undefined)
+        return { kind: "rejected", reason: "command-wait-not-declared" };
+    const computed = effectInputDigest({
+        capability: "command",
+        method: `run:${wait.command}`,
+        target: wait.command,
+        payload: wait.inputDigest,
+    });
+    return command.inputDigest === computed
+        ? unpinned(command, context)
+        : { kind: "rejected", reason: "input-digest-mismatch" };
+};
+const commandReport = (command, context) => {
+    const output = command.commandOutput;
+    if (output !== undefined &&
+        (command.worker !== "command" || documentDigest(output) !== command.result))
+        return { kind: "rejected", reason: "command-output-mismatch" };
+    return unpinned(command, context);
+};
 const handoffEffect = (command, context) => {
     if (command.capability !== "handoff")
         return unpinned(command, context);
@@ -67,6 +124,9 @@ const effectRequest = (command, context) => {
 };
 const rules = {
     "request-effect": effectRequest,
+    "check-command-wait": commandWaitCheck,
+    "report-effect": commandReport,
+    "open-command-wait": commandWait,
     "admit-run": admission,
     "decide-gate": resumption,
     "resume-run": resumption,

package/dist/src/broker/authority-restore.d.ts

@@ -1,3 +1,4 @@
+import type { CommandWaitRow } from "../storage/command-wait-rows.ts";
 import type { AuthorityEvent, AuthorityState, EffectIntent, RunState } from "../authority/reducer.ts";
 import type { DecisionRows, DocumentRow } from "../storage/decision-rows.ts";
 import type { EffectRows } from "../storage/effect-rows.ts";
@@ -7,6 +8,7 @@ import type { ExecutionManifest } from "../workflow-admission.ts";
 export type RestoredSnapshot = AuthoritySnapshot & Partial<DecisionRows> & {
     readonly documents?: readonly DocumentRow[];
     readonly effects?: EffectRows;
+    readonly waits?: readonly CommandWaitRow[];
 };
 export type Restoration = {
     readonly kind: "restored";

package/dist/src/broker/authority-restore.js

@@ -1,3 +1,4 @@
+import { commandWaitState, commandWaitRows, commandOutputsIntact, } from "./command-wait-restore.js";
 import { isDeepStrictEqual } from "node:util";
 import { parseEvent } from "../authority/events.js";
 import { rebuildHistory } from "../authority/history.js";
@@ -51,6 +52,7 @@ export function restore(snapshot) {
             isDeepStrictEqual(Object.fromEntries(snapshot.runs.map((row) => [row.runId, row.sequence])), history.runSequences),
         ],
         ["definitions", definitionsIntact(definitions, state)],
+        ...waitChecks(snapshot, { state, definitions, events }),
         ...decisionChecks(snapshot, state),
         ...(snapshot.effects === undefined
             ? []
@@ -68,6 +70,21 @@ export function restore(snapshot) {
         };
     return { kind: "restored", state, events, definitions };
 }
+function waitChecks(snapshot, context) {
+    const { state, definitions } = context;
+    return [
+        ["command-outputs", commandOutputsIntact(context.events)],
+        ["command-waits", commandWaitsIntact(definitions, state)],
+        ...(snapshot.waits === undefined
+            ? []
+            : [
+                [
+                    "waits",
+                    isDeepStrictEqual(snapshot.waits, commandWaitRows(state.commandWaits ?? {})),
+                ],
+            ]),
+    ];
+}
 function decisionChecks(snapshot, state) {
     const { gates, details, documents } = snapshot;
     if (gates === undefined || details === undefined || documents === undefined)
@@ -95,12 +112,29 @@ function definitionsIntact(definitions, state) {
     const used = new Set(Object.values(state.runs).map((run) => run.workflow));
     return ([...definitions].every(([pin, manifest]) => workflowPin(manifest) === pin && used.has(pin)) && [...used].every((pin) => definitions.has(pin)));
 }
+function commandWaitsIntact(definitions, state) {
+    return Object.values(state.commandWaits ?? {}).every((wait) => {
+        const pin = state.runs[wait.runId]?.workflow ?? "";
+        const waits = definitions.get(pin)?.commandWaits ?? {};
+        const permission = Object.hasOwn(waits, wait.name)
+            ? waits[wait.name]
+            : undefined;
+        return (permission !== undefined &&
+            permission.command === wait.command &&
+            permission.maximumWaitSeconds === wait.maximumWaitSeconds &&
+            permission.pollEverySeconds === wait.pollEverySeconds);
+    });
+}
 const projectionParts = ["sequence", "runs", "commands", "intents"];
 function text(error) {
     return String(Reflect.get(Object(error), "message"));
 }
 export function currentState(snapshot) {
-    return { ...projectedState(snapshot), ...decisionState(snapshot) };
+    return {
+        ...projectedState(snapshot),
+        ...decisionState(snapshot),
+        ...commandWaitState(snapshot.waits),
+    };
 }
 export function projectedState(snapshot) {
     return {

package/dist/src/broker/command-wait-restore.d.ts

@@ -0,0 +1,31 @@
+import type { Database } from "solid-objects";
+import { type CommandWaitsState } from "../authority/command-waits.ts";
+import type { AuthorityEvent } from "../authority/transitions.ts";
+import type { DispatchRead } from "../authority/dispatch-decision.ts";
+import type { AuthorityState } from "../authority/transitions.ts";
+import { type CommandWaitRow } from "../storage/command-wait-rows.ts";
+import type { SqlConnection } from "../storage/domain-rows.ts";
+export declare function commandWaitRows(waits: CommandWaitsState): readonly CommandWaitRow[];
+export declare function commandWaitState(rows: readonly CommandWaitRow[]): {
+    readonly commandWaits?: CommandWaitsState;
+};
+export declare function readCommandWaits(connection: SqlConnection, runId: string): Promise<{
+    readonly commandWaits?: CommandWaitsState;
+}>;
+export declare function upgradeCommandWaits(database: Database, state: AuthorityState): Promise<void>;
+export declare function upgradeCommandWaitChecks(database: Database): Promise<void>;
+export declare function commandWaitDeadline(waits: CommandWaitsState, operationId: string): {
+    readonly commandDeadline?: string;
+};
+export declare function dispatchCommandWait(connection: SqlConnection, check: {
+    readonly runId: string;
+    readonly workflow: string;
+    readonly operationId: string;
+}): Promise<Pick<DispatchRead, "commandWait">>;
+export declare function commandOutputsIntact(events: readonly AuthorityEvent[]): boolean;
+export declare function readCommandWaitDeadline(connection: SqlConnection, scope: {
+    readonly runId: string;
+    readonly operationId: string;
+}): Promise<{
+    readonly commandDeadline?: string;
+}>;