cloving 1.3.0 → 1.5.0

package/dist/src/broker/actor/authority-session.js

@@ -1,23 +1,22 @@
-import { eligibleIntents, } from "../../authority/reducer.js";
-import { loadOperationSnapshot, loadProjections, } from "../../storage/decision-snapshots.js";
 import { loadEvents } from "../../storage/domain-rows.js";
 import { pinnedResumption } from "../authority-admission.js";
-import { currentState, intentKind, storedDefinitions, storedEvents, } from "../authority-restore.js";
+import { storedDefinitions, storedEvents, } from "../authority-restore.js";
 import { checkedRequest } from "./authority-actor.js";
 import { subscriptions, } from "./authority-changes.js";
 import { authorityQueries, guardedRead, } from "./authority-queries.js";
 import { effectQueries } from "./effect-queries.js";
 import { backupInto } from "../../storage/domain-schema.js";
 import { observeSettings, } from "./authority-connection.js";
+import { authorityHealth } from "./authority-health.js";
 import { failed, serializable, storeReceipt, invalidCommand, oversized, tooLarge, unserializable, } from "./authority-receipts.js";
+export { operationRecords } from "./projection-memory.js";
 export function authoritySession(parts) {
     const session = {
         parts,
         stopping: new AbortController(),
         delivering: false,
         failure: null,
-        condition: null,
-        describing: null,
+        health: authorityHealth(parts),
         closed: false,
         running: Promise.resolve(),
         invocations: new Set(),
@@ -33,18 +32,18 @@ export function authoritySession(parts) {
     });
     return {
         execute: (command, context) => executeCommand(session, { command, context }),
-        state: async () => currentState(await projected(session)),
+        state: async () => (await projected(session)).state(),
         events: async () => {
             await refreshed(session);
             available(session);
             return storedEvents(await parts.reader.connection(loadEvents));
         },
-        eligibleIntents: async () => eligible(await projected(session)),
-        operations: async () => operationRecords(await read(loadOperationSnapshot)),
+        eligibleIntents: async () => (await projected(session)).eligible(),
+        operations: async () => (await projected(session)).operations(),
         resumePinned: async (runId, manifest) => {
-            const snapshot = await projected(session);
-            return pinnedResumption(currentState(snapshot), {
-                definitions: storedDefinitions(snapshot),
+            const current = await projected(session);
+            return pinnedResumption(current.state(), {
+                definitions: storedDefinitions(current.snapshot),
                 runId,
                 available: manifest,
             });
@@ -56,7 +55,11 @@ export function authoritySession(parts) {
         },
         epoch: parts.epoch,
         recoveryLimit: parts.recoveryLimit,
-        queries: authorityQueries(read),
+        queries: {
+            ...authorityQueries(read),
+            sequences: async () => (await projected(session)).sequences(),
+            workflowIds: async (runIds) => (await projected(session)).workflowIds(runIds),
+        },
         effects: effectQueries(read, { stopLimit: parts.stopLimit }),
         deliver: (signal) => deliver(session, signal),
         changes: { subscribe: session.changes.add },
@@ -82,29 +85,16 @@ async function stopped(session) {
     }
 }
 async function refreshed(session) {
-    if (session.parts.unhealthy() === undefined)
+    if (session.failure !== null || session.closed)
         return;
-    await failClosed(session, "authority failure");
+    await session.health.refresh();
 }
-async function failClosed(session, failure) {
+function failClosed(session, failure) {
     session.failure ??= failure;
-    await conditionRead(session);
-    close(session).catch(() => undefined);
-}
-async function conditionRead(session) {
-    if (session.condition !== null && session.describing === null)
-        return;
-    session.describing ??= described(session).finally(() => {
-        session.describing = null;
-    });
-    await session.describing;
+    close(session).catch(ignored);
 }
-async function described(session) {
-    const detected = session.parts.unhealthy();
-    if (detected === undefined)
-        return;
-    session.condition = detected;
-    session.condition = (await session.parts.condition()) ?? detected;
+function ignored() {
+    return undefined;
 }
 async function backup(session, target) {
     await refreshed(session);
@@ -116,36 +106,43 @@ async function backup(session, target) {
         throw new Error(`backup-failed: ${String(Reflect.get(Object(error), "message"))}`, { cause: error });
     }
 }
-function available(session) {
-    const failure = session.condition ?? session.failure;
-    if (failure !== null)
-        throw new Error(`authority-unavailable: ${failure}`);
+function opened(session) {
+    if (session.failure !== null)
+        throw new Error(`authority-unavailable: ${session.failure}`);
     if (session.closed)
         throw new Error("authority-unavailable: closed");
 }
+function available(session) {
+    opened(session);
+    const condition = session.health.condition();
+    if (condition !== null)
+        throw new Error(`authority-unavailable: ${condition}`);
+}
 function unavailableReceipt(session) {
-    const failure = session.condition ?? session.failure;
-    if (failure !== null)
+    if (session.failure !== null)
         return {
             kind: "failed",
             reason: "storage-unavailable",
-            detail: `closed after ${failure}`,
+            detail: `closed after ${session.failure}`,
         };
     if (session.closed)
         return { kind: "failed", reason: "storage-unavailable", detail: "closed" };
-    return undefined;
+    const condition = session.health.condition();
+    return condition === null
+        ? undefined
+        : { kind: "failed", reason: "authority-unavailable", detail: condition };
 }
 async function projected(session) {
     await refreshed(session);
     available(session);
-    return session.parts.reader.connection(loadProjections);
+    return session.parts.memory.read(session.parts.database);
 }
 async function deliver(session, signal) {
     if (!session.parts.deliveryConfigured)
         throw new Error("delivery-unconfigured");
     if (session.delivering)
         throw new Error("delivery-already-running");
-    available(session);
+    opened(session);
     session.delivering = true;
     try {
         await delivered(session, signal);
@@ -154,21 +151,19 @@ async function deliver(session, signal) {
         session.delivering = false;
     }
     await refreshed(session);
-    const failure = session.condition ?? session.failure;
+    const failure = session.failure ?? session.health.condition();
     if (failure !== null)
         throw new Error(`authority-unavailable: ${failure}`);
 }
 async function delivered(session, signal) {
     if (!(await session.parts.handlersSettled()))
         throw new Error("delivery-handler-in-flight");
-    available(session);
-    const watch = new AbortController();
+    opened(session);
     const timer = setInterval(() => {
-        if (session.parts.unhealthy() !== undefined)
-            watch.abort();
+        refreshed(session).catch(ignored);
     }, session.parts.watchMilliseconds);
     try {
-        session.running = session.parts.run(AbortSignal.any([signal, session.stopping.signal, watch.signal]));
+        session.running = session.parts.run(AbortSignal.any([signal, session.stopping.signal]));
         await session.running;
     }
     finally {
@@ -177,8 +172,7 @@ async function delivered(session, signal) {
 }
 async function executeCommand(session, input) {
     const { command, context } = input;
-    await refreshed(session);
-    const unavailable = unavailableReceipt(session);
+    const unavailable = await refusal(session);
     if (unavailable !== undefined)
         return unavailable;
     const given = serializable({
@@ -196,15 +190,32 @@ async function executeCommand(session, input) {
     if (oversized(request))
         return tooLarge;
     try {
-        return await healthyReceipt(session, await invoked(session, request));
+        return storeReceipt(await invoked(session, request));
     }
     catch (error) {
-        const receipt = failed(error);
-        await failClosed(session, receipt.reason === "storage-failure"
-            ? "storage failure"
-            : "authority failure");
-        return receipt;
+        return failedReceipt(session, error);
+    }
+}
+async function refusal(session) {
+    try {
+        await refreshed(session);
     }
+    catch (error) {
+        return failed(error);
+    }
+    return unavailableReceipt(session);
+}
+function failedReceipt(session, error) {
+    const receipt = failed(error);
+    if (timedOut(error))
+        return receipt;
+    failClosed(session, receipt.reason === "storage-failure"
+        ? "storage failure"
+        : "authority failure");
+    return receipt;
+}
+function timedOut(error) {
+    return error instanceof Error && error.name === "SyncTimeout";
 }
 async function invoked(session, request) {
     const invocation = session.parts.execute(request);
@@ -216,36 +227,3 @@ async function invoked(session, request) {
         session.invocations.delete(invocation);
     }
 }
-async function healthyReceipt(session, value) {
-    const receipt = storeReceipt(value);
-    await refreshed(session);
-    return receipt;
-}
-export function operationRecords(snapshot) {
-    const deliveries = Map.groupBy(snapshot.deliveries, (row) => row.operationId);
-    return snapshot.outbox.map((row) => ({
-        operationId: row.operationId,
-        kind: intentKind(row.kind),
-        runId: row.runId,
-        createdSequence: row.createdSequence,
-        releasedSequence: row.releasedSequence,
-        deliveries: (deliveries.get(row.operationId) ?? []).map(({ effectId, outcome, receipt, purpose, epoch }) => ({
-            effectId,
-            outcome,
-            receipt,
-            purpose,
-            epoch,
-        })),
-    }));
-}
-function eligible(snapshot) {
-    return eligibleIntents({
-        ...currentState(snapshot),
-        effects: {
-            operations: Object.fromEntries(snapshot.pending.map((operationId) => [
-                operationId,
-                { status: "dispatch-pending" },
-            ])),
-        },
-    });
-}

package/dist/src/broker/actor/authority-settings.d.ts

@@ -1,4 +1,4 @@
-import type { SolidObjectsConfiguration } from "solid-objects";
+import type { Logger, SolidObjectsConfiguration } from "solid-objects";
 export interface AuthorityTiming {
     readonly leaseMilliseconds: number;
     readonly renewalMilliseconds: number;
@@ -30,4 +30,5 @@ export declare function effectConfiguration(options: {
         readonly stop?: unknown;
     };
 }): PlanSettings;
+export declare function authorityLogger(log: ((line: string) => void) | undefined): Logger;
 export declare function librarySettings(timing: AuthorityTiming, log: ((line: string) => void) | undefined): Partial<SolidObjectsConfiguration>;

package/dist/src/broker/actor/authority-settings.js

@@ -47,14 +47,17 @@ export function effectConfiguration(options) {
 function standardError(line) {
     process.stderr.write(line);
 }
+export function authorityLogger(log) {
+    return lineLogger(log ?? standardError);
+}
 export function librarySettings(timing, log) {
     return {
         tableNamePrefix: "solid_",
         maxAttempts: 1,
         maxActivationDurationMilliseconds: 50,
-        maxPayloadBytes: 900_000 + commandLimits.stdoutBytes * 6,
-        maxResultBytes: 900_000 + commandLimits.stdoutBytes * 6,
-        logger: lineLogger(log ?? standardError),
+        maxPayloadBytes: 900_000 + commandLimits.stdoutBytes * 12,
+        maxResultBytes: 900_000 + commandLimits.stdoutBytes * 12,
+        logger: authorityLogger(log),
         leaseDurationMilliseconds: timing.leaseMilliseconds,
         leaseRenewalIntervalMilliseconds: timing.renewalMilliseconds,
         processHeartbeatIntervalMilliseconds: timing.heartbeatMilliseconds,

package/dist/src/broker/actor/authority-upgrades.d.ts

@@ -1,7 +1,8 @@
-import { loadDecided } from "../../storage/decision-snapshots.ts";
+import { upgradeCommandWaits, upgradeCommandWaitChecks } from "../command-wait-restore.ts";
+import { loadDecided, loadEffected, loadWaiting, loadCurrent } from "../../storage/decision-snapshots.ts";
 import { loadSnapshot } from "../../storage/domain-rows.ts";
 import { upgradeDecisions } from "../decision-upgrade.ts";
-import { upgradeEffects } from "../effect-upgrade.ts";
+import { upgradeEffects, upgradeEventIndexes, upgradeRunIndexes, upgradeSpendingIndexes } from "../effect-upgrade.ts";
 declare const upgrades: readonly [{
     readonly from: "adopted";
     readonly load: typeof loadSnapshot;
@@ -10,6 +11,26 @@ declare const upgrades: readonly [{
     readonly from: "decided";
     readonly load: typeof loadDecided;
     readonly apply: typeof upgradeEffects;
+}, {
+    readonly from: "effected";
+    readonly load: typeof loadEffected;
+    readonly apply: typeof upgradeCommandWaits;
+}, {
+    readonly from: "waiting";
+    readonly load: typeof loadWaiting;
+    readonly apply: typeof upgradeCommandWaitChecks;
+}, {
+    readonly from: "checked";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeSpendingIndexes;
+}, {
+    readonly from: "indexed";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeEventIndexes;
+}, {
+    readonly from: "event-indexed";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeRunIndexes;
 }];
 export type UpgradeStep = (typeof upgrades)[number];
 export declare function upgradeSteps(kind: string): readonly UpgradeStep[];

package/dist/src/broker/actor/authority-upgrades.js

@@ -1,10 +1,16 @@
-import { loadDecided } from "../../storage/decision-snapshots.js";
+import { upgradeCommandWaits, upgradeCommandWaitChecks, } from "../command-wait-restore.js";
+import { loadDecided, loadEffected, loadWaiting, loadCurrent, } from "../../storage/decision-snapshots.js";
 import { loadSnapshot } from "../../storage/domain-rows.js";
 import { upgradeDecisions } from "../decision-upgrade.js";
-import { upgradeEffects } from "../effect-upgrade.js";
+import { upgradeEffects, upgradeEventIndexes, upgradeRunIndexes, upgradeSpendingIndexes, } from "../effect-upgrade.js";
 const upgrades = [
     { from: "adopted", load: loadSnapshot, apply: upgradeDecisions },
     { from: "decided", load: loadDecided, apply: upgradeEffects },
+    { from: "effected", load: loadEffected, apply: upgradeCommandWaits },
+    { from: "waiting", load: loadWaiting, apply: upgradeCommandWaitChecks },
+    { from: "checked", load: loadCurrent, apply: upgradeSpendingIndexes },
+    { from: "indexed", load: loadCurrent, apply: upgradeEventIndexes },
+    { from: "event-indexed", load: loadCurrent, apply: upgradeRunIndexes },
 ];
 export function upgradeSteps(kind) {
     const first = upgrades.findIndex((step) => step.from === kind);

package/dist/src/broker/actor/authority-watchdog.d.ts

@@ -3,10 +3,14 @@ export interface WatchedEvent {
     readonly messageId: string | null;
     readonly attributes: Readonly<Record<string, unknown>>;
 }
+export interface WatchReport {
+    readonly kind: "stranded" | "dead-lettered";
+    readonly messageId: string;
+}
+export type MessageRuns = (messageIds: readonly string[]) => Promise<ReadonlyMap<string, string>>;
 export interface StrandingWatchdog {
     readonly observe: (event: WatchedEvent) => void;
-    readonly stranded: () => string | undefined;
-    readonly deadLettered: () => string | undefined;
+    readonly report: (runs: MessageRuns) => Promise<WatchReport | undefined>;
 }
 export declare function strandingWatchdog(bound: {
     readonly boundMilliseconds: number;

package/dist/src/broker/actor/authority-watchdog.js

@@ -1,3 +1,4 @@
+import { terminal } from "../../authority/transitions.js";
 const callbacks = ["deliveryCompleted", "deliveryFailed"];
 const finished = [
     "solid_objects.message.completed",
@@ -7,33 +8,54 @@ const finished = [
 export function strandingWatchdog(bound) {
     const clock = bound.clock ?? monotonic;
     const lost = new Map();
-    const dead = [];
-    let reported;
+    const dead = new Set();
     return {
         observe: (event) => {
             const { messageId } = event;
             if (messageId === null)
                 return;
             if (deadCallback(event))
-                dead.push(messageId);
+                dead.add(messageId);
             if (event.name === "solid_objects.activation.lost" &&
                 !lost.has(messageId))
                 lost.set(messageId, clock());
             if (finished.includes(event.name))
                 lost.delete(messageId);
         },
-        stranded: () => {
-            reported ??= overdue(lost, clock() - bound.boundMilliseconds);
-            return reported;
+        report: async (runs) => {
+            const overdue = overdueMessages(lost, clock() - bound.boundMilliseconds);
+            const suspects = [...dead, ...overdue];
+            if (suspects.length === 0)
+                return undefined;
+            const statuses = await runs(suspects);
+            const released = suspects.filter((messageId) => settled(statuses.get(messageId)));
+            for (const messageId of released) {
+                lost.delete(messageId);
+                dead.delete(messageId);
+            }
+            return firstReport({ dead: [...dead], stranded: overdue, lost });
         },
-        deadLettered: () => dead[0],
     };
 }
 function monotonic() {
     return performance.now();
 }
-function overdue(lost, limit) {
-    return [...lost].find(([, at]) => at < limit)?.[0];
+function overdueMessages(lost, limit) {
+    return [...lost]
+        .filter(([, at]) => at < limit)
+        .map(([messageId]) => messageId);
+}
+function settled(status) {
+    return terminal.some((candidate) => candidate === status);
+}
+function firstReport(found) {
+    const deadLetter = found.dead[0];
+    if (deadLetter !== undefined)
+        return { kind: "dead-lettered", messageId: deadLetter };
+    const stranded = found.stranded.find((messageId) => found.lost.has(messageId));
+    return stranded === undefined
+        ? undefined
+        : { kind: "stranded", messageId: stranded };
 }
 function deadCallback(event) {
     return (event.name === "solid_objects.dead_letter.created" &&

package/dist/src/broker/actor/authority.js

@@ -76,6 +76,7 @@ async function begun(opening) {
             reason: "unresolved-delivery",
             detail: condition,
         };
+    await parts.memory.load(owned.database);
     return integer(opened, "epoch");
 }
 async function start(opening) {
@@ -99,6 +100,7 @@ async function start(opening) {
         authority: authoritySession({
             database,
             reader: parts.reader,
+            memory: parts.memory,
             execute: (request) => authority.execute(json(request)),
             run: (signal) => parts.runtime.run(signal),
             stop: parts.stop,
@@ -106,8 +108,8 @@ async function start(opening) {
             deliveryConfigured: options.delivery !== undefined,
             epoch,
             ...parts.limits,
-            unhealthy: parts.unhealthy,
             condition: parts.condition,
+            logger: parts.logger,
             watchMilliseconds: parts.watchMilliseconds,
             subscribe: (listener) => parts.subscribe(actorId, listener),
         }),

package/dist/src/broker/actor/delivery-intents.js

@@ -1,3 +1,4 @@
+import { commandWaitDeadline } from "../command-wait-restore.js";
 import { createHash } from "node:crypto";
 import { delivers, isSettled } from "../../authority/gates.js";
 import { eligibleIntents, } from "../../authority/reducer.js";
@@ -26,6 +27,7 @@ export function protectedIntent(state, attempt) {
         .filter((lease) => lease.resourceKey === operation.leaseKey)
         .map((lease) => ({
         ...intent,
+        ...commandWaitDeadline(state.commandWaits ?? {}, operation.operationId),
         workflow,
         purpose,
         inputDigest: operation.inputDigest,

package/dist/src/broker/actor/effect-delivery.d.ts

@@ -16,4 +16,4 @@ export interface Attempt {
     readonly effectId: string;
 }
 export declare function deliverEffect(parts: EffectDelivery, attempt: Attempt): Promise<JsonObject>;
-export declare function protectedArguments(value: JsonObject): EffectArguments;
+export declare function protectedArguments(value: JsonObject): EffectArguments & Pick<ProtectedIntent, "commandDeadline">;

package/dist/src/broker/actor/effect-delivery.js

@@ -1,3 +1,4 @@
+import { dispatchCommandWait, readCommandWaitDeadline, } from "../command-wait-restore.js";
 import { dispatchDecision, } from "../../authority/dispatch-decision.js";
 import { integer, optionalInteger, optionalText, text, } from "../../storage/domain-rows.js";
 import { budgetRow } from "../../storage/effect-rows.js";
@@ -16,11 +17,16 @@ const absent = {
     budget: undefined,
 };
 async function dispatchRead(connection, attempt) {
-    const row = await connection.get("SELECT d.outcome, d.input_digest, a.lease_generation, o.released_sequence, r.status, r.workflow, p.status AS operation_status, p.capability, p.units, l.operation_id AS lease_owner, l.generation, l.expires_at, l.state AS lease_state, b.run_id, b.effect_limit, b.spent, b.capabilities, b.exhausted_sequence FROM deliveries d JOIN outbox o ON o.operation_id = d.operation_id JOIN runs r ON r.run_id = o.run_id JOIN dispatch_attempts a ON a.effect_id = d.effect_id JOIN operations p ON p.operation_id = d.operation_id JOIN leases l ON l.resource_key = p.lease_key JOIN run_budgets b ON b.run_id = o.run_id WHERE d.effect_id = ? AND d.operation_id = ?", [attempt.effectId, attempt.intent.operationId]);
+    const row = await connection.get("SELECT d.outcome, d.input_digest, a.lease_generation, o.released_sequence, r.status, r.workflow, p.status AS operation_status, p.capability, p.units, p.method, p.target, p.payload, l.operation_id AS lease_owner, l.generation, l.expires_at, l.state AS lease_state, b.run_id, b.effect_limit, b.spent, b.capabilities, b.exhausted_sequence FROM deliveries d JOIN outbox o ON o.operation_id = d.operation_id JOIN runs r ON r.run_id = o.run_id JOIN dispatch_attempts a ON a.effect_id = d.effect_id JOIN operations p ON p.operation_id = d.operation_id JOIN leases l ON l.resource_key = p.lease_key JOIN run_budgets b ON b.run_id = o.run_id WHERE d.effect_id = ? AND d.operation_id = ?", [attempt.effectId, attempt.intent.operationId]);
     if (row === undefined)
         return absent;
     const workflow = text(row, "workflow");
     return {
+        ...(await dispatchCommandWait(connection, {
+            runId: attempt.intent.runId,
+            workflow,
+            operationId: attempt.intent.operationId,
+        })),
         attempt: {
             outcome: optionalText(row, "outcome"),
             inputDigest: text(row, "input_digest"),
@@ -37,6 +43,9 @@ async function dispatchRead(connection, attempt) {
             status: text(row, "operation_status"),
             capability: text(row, "capability"),
             units: integer(row, "units"),
+            method: text(row, "method"),
+            target: text(row, "target"),
+            payload: text(row, "payload"),
         },
         budget: storedBudget(budgetRow(row)),
         lease: {
@@ -52,7 +61,8 @@ async function called(parts, call) {
     const adapterCall = parts.delivery[name]?.bind(parts.delivery);
     if (adapterCall === undefined)
         return { kind: "unknown", reason: "stop-not-supported" };
-    return settlement(await parts.calls.call(intent.operationId, (signal) => adapterCall(intent, signal)), intent.capability === "command"
+    const deadline = await parts.reader.connection((connection) => readCommandWaitDeadline(connection, intent));
+    return settlement(await parts.calls.call(intent.operationId, (signal) => adapterCall({ ...intent, ...deadline }, signal)), intent.capability === "command"
         ? commandLimits.resultBytes
         : parts.effects.resultBytes);
 }
@@ -126,5 +136,8 @@ export function protectedArguments(value) {
         method: text(value, "method"),
         target: text(value, "target"),
         payload: text(value, "payload"),
+        ...(typeof value["commandDeadline"] === "string"
+            ? { commandDeadline: value["commandDeadline"] }
+            : {}),
     };
 }

package/dist/src/broker/actor/effect-plan.js

@@ -1,3 +1,5 @@
+import { operationPhase } from "../../operation-identity.js";
+import { endedCommandWaitCheck } from "../../authority/command-waits.js";
 import { isInstant } from "../../authority/instants.js";
 import { promotions } from "../../authority/leases.js";
 import { execute } from "../../authority/reducer.js";
@@ -45,7 +47,9 @@ function closure(state, operation) {
         return ["run-ended"];
     const exhausted = (state.effects.budgets[operation.runId]?.exhaustedSequence ?? null) !==
         null;
-    return exhausted ? ["budget-exhausted"] : [];
+    return exhausted && operationPhase(operation.operationId) !== "command-wait"
+        ? ["budget-exhausted"]
+        : [];
 }
 function unblocked(state, followUp) {
     const steps = [];
@@ -95,6 +99,9 @@ const outcomes = {
     }),
     completed: (attempt) => stated({
         outcome: "completed",
+        ...(attempt.worker === "command"
+            ? { commandOutput: attempt.result }
+            : {}),
         result: attempt.result === null ? null : documentDigest(attempt.result),
         reason: attempt.reason,
         ...identified(attempt),
@@ -155,7 +162,9 @@ function acceptedNow(reconciling) {
 export function reconciliations(state, reconciling) {
     const accepted = acceptedNow(reconciling);
     const rejected = rejectedReports(reconciling.results);
-    const due = (operation) => state.runs[operation.runId]?.status !== "cancelled" && live(operation);
+    const due = (operation) => state.runs[operation.runId]?.status !== "cancelled" &&
+        !endedCommandWaitCheck(state.commandWaits ?? {}, operation.operationId) &&
+        live(operation);
     const live = (operation) => (operation.status === "recovering" &&
         !accepted.has(operation.operationId)) ||
         (operation.status === "blocked" && rejected.has(operation.operationId));

package/dist/src/broker/actor/effect-queries.d.ts

@@ -8,6 +8,10 @@ export interface OperationView extends OperationRow {
 }
 export type StoredDisposition = Omit<Evidence, "sequence">;
 export interface EffectQueries {
+    readonly workerTurns: (window: {
+        readonly after: string;
+        readonly through: string;
+    }) => Promise<readonly OperationView[]>;
     readonly operation: (operationId: string) => Promise<OperationView | undefined>;
     readonly operations: (runId: string) => Promise<readonly OperationView[]>;
     readonly evidence: (runId: string) => Promise<readonly number[]>;

package/dist/src/broker/actor/effect-queries.js

@@ -34,7 +34,9 @@ const viewColumns = operationColumns
     .map((name) => `o.${name}`)
     .join(", ");
 const waitCause = "CASE WHEN o.status != 'waiting' THEN NULL WHEN r.status != 'running' THEN 'run-not-running' WHEN EXISTS (SELECT 1 FROM leases h WHERE h.resource_key = o.lease_key AND h.state != 'released') THEN 'resource' ELSE 'ready' END";
-const viewed = `SELECT ${viewColumns}, l.generation AS lease_generation, ${waitCause} AS wait_cause FROM operations o JOIN runs r ON r.run_id = o.run_id LEFT JOIN leases l ON l.resource_key = o.lease_key AND l.operation_id = o.operation_id AND l.state != 'released'`;
+const viewSelection = `SELECT ${viewColumns}, l.generation AS lease_generation, ${waitCause} AS wait_cause`;
+const viewJoins = "JOIN runs r ON r.run_id = o.run_id LEFT JOIN leases l ON l.resource_key = o.lease_key AND l.operation_id = o.operation_id AND l.state != 'released'";
+const viewed = `${viewSelection} FROM operations o ${viewJoins}`;
 function operationView(row) {
     return {
         ...operationRow(row),
@@ -52,6 +54,11 @@ async function operations(connection, runId) {
     const rows = await connection.all(`${viewed} WHERE o.run_id = ? ORDER BY o.requested_sequence`, [runId]);
     return rows.map(operationView);
 }
+async function workerTurns(connection, window) {
+    const settled = await connection.all(`${viewSelection} FROM events e CROSS JOIN operations o ${viewJoins} WHERE o.settled_sequence = e.sequence AND o.capability = 'claude' AND julianday(e.recorded_at) > julianday(?) AND julianday(e.recorded_at) <= julianday(?)`, [window.after, window.through]);
+    const open = await connection.all(`${viewSelection} FROM operations o INDEXED BY operations_worker_open ${viewJoins} WHERE o.capability = 'claude' AND o.settled_sequence IS NULL AND o.status NOT IN ('refused', 'completed', 'failed', 'stopped', 'cancelled')`);
+    return [...settled, ...open].map(operationView);
+}
 async function evidence(connection, runId) {
     const rows = await connection.all("SELECT e.sequence FROM effect_evidence e JOIN events v ON v.sequence = e.sequence WHERE e.run_id = ? AND (e.disposition IN ('rejected', 'denied', 'refused') OR (v.type = 'operation.reconciled' AND json_extract(v.payload, '$.outcome') = 'unknown')) ORDER BY e.sequence", [runId]);
     return rows.map((row) => integer(row, "sequence"));
@@ -74,6 +81,7 @@ async function budget(connection, runId) {
 }
 export function effectQueries(read, settings) {
     return {
+        workerTurns: (window) => read((connection) => workerTurns(connection, window)),
         stops: (runId) => read((connection) => stops(connection, runId)),
         closure: (runId) => read((connection) => storedClosure(connection, { runId, stopLimit: settings.stopLimit })),
         settledAt: (runId) => read((connection) => settledAt(connection, runId)),

package/dist/src/broker/actor/effect-turn.js

@@ -1,3 +1,4 @@
+import { readCommandWaitDeadline } from "../command-wait-restore.js";
 import { detailColumns, detailRow } from "../../storage/decision-rows.js";
 import { integer, runRow, text, } from "../../storage/domain-rows.js";
 import { storedRun } from "../authority-restore.js";
@@ -57,6 +58,7 @@ export async function pendingEffect(connection, attempt) {
     return {
         ...intent,
         purpose,
+        ...(await readCommandWaitDeadline(connection, intent)),
         inputDigest: text(row, "input_digest"),
         leaseGeneration: integer(row, "generation"),
         capability: text(row, "capability"),