cloving 1.2.0 → 1.4.0

package/dist/src/authority/status.js

@@ -88,8 +88,10 @@ const unavailablePin = assessed("infrastructure-failure", {
     nextAction: "restore-pinned-workflow",
     humanInputCanHelp: true,
 });
-function worldConflict(input) {
+function workflowFailure(input) {
     const code = input.details?.observation?.reason;
+    if (code === "pinned-workflow-unavailable")
+        return unavailablePin;
     if (code !== "world-run-conflict")
         return undefined;
     return assessed("infrastructure-failure", {
@@ -103,7 +105,7 @@ function queued(input) {
         return unknownEffect;
     if (blocked(input.operations))
         return unavailablePin;
-    const conflict = worldConflict(input);
+    const conflict = workflowFailure(input);
     if (conflict !== undefined)
         return conflict;
     if (dispatched(input.operations))
@@ -141,10 +143,13 @@ function running(input) {
         return effect;
     if (blocked(input.operations))
         return unavailablePin;
-    return worldConflict(input) ?? observed(input);
+    return workflowFailure(input) ?? observed(input);
 }
 function observed(input) {
     const observation = input.details?.observation ?? null;
+    const wait = runWaitAssessment(observation);
+    if (wait !== undefined)
+        return wait;
     if (observation !== null &&
         Object.hasOwn(observedActions, observation.condition))
         return observedWork(observation);
@@ -160,7 +165,20 @@ function observed(input) {
         humanInputCanHelp: false,
     });
 }
+function runWaitAssessment(observation) {
+    if (observation?.condition === "waiting-run")
+        return assessed("waiting-run", {
+            code: "awaiting-run",
+            detail: `The run waits for run ${observation.evidence ?? "unknown"}.`,
+            nextAction: "await-run",
+            nextActionAt: observation.retryAt,
+            humanInputCanHelp: false,
+        });
+    return undefined;
+}
 function waiting(input) {
+    if (input.details?.observation?.reason === "pinned-workflow-unavailable")
+        return unavailablePin;
     const pending = input.gates.filter((gate) => gate.outcome === null);
     const question = pending.some((gate) => gate.kind === "question");
     const approval = pending.some((gate) => gate.kind === "approval");
@@ -226,6 +244,7 @@ export function runView(input) {
         workflow: run.workflow,
         state: run.status,
         condition: assessment.condition,
+        awaitedRunId: awaitedRun(assessment, observation),
         revision: run.revision,
         latestSequence: input.latestSequence,
         pendingDecisions: input.gates
@@ -247,6 +266,7 @@ export function runView(input) {
             attempts: observation?.attempt ?? 0,
             limit: observation?.limit ?? null,
         },
+        commandWaits: input.commandWaits,
         budget: input.budget,
         effects: effectViews(input.effects),
         nextAction: assessment.nextAction,
@@ -254,3 +274,8 @@ export function runView(input) {
         humanInputCanHelp: assessment.humanInputCanHelp,
     };
 }
+function awaitedRun(assessment, observation) {
+    return assessment.condition === "waiting-run"
+        ? (observation?.evidence ?? null)
+        : null;
+}

package/dist/src/authority/stop-sequence.d.ts

@@ -5,6 +5,7 @@ export interface AttemptSummary {
     readonly stagedSequence: number;
 }
 export interface StopSubject {
+    readonly stopSequence?: number;
     readonly operationId: string;
     readonly kind: string;
     readonly attempts: readonly AttemptSummary[];

package/dist/src/authority/stop-sequence.js

@@ -39,7 +39,8 @@ const stopPurposes = ["stop", "reconcile"];
 export function stopCount(subject, cancelledSequence) {
     return subject.attempts.filter((attempt) => attempt.outcome !== null &&
         stopPurposes.includes(attempt.purpose) &&
-        attempt.stagedSequence >= cancelledSequence).length;
+        attempt.stagedSequence >=
+            Math.max(cancelledSequence, subject.stopSequence ?? 0)).length;
 }
 const uncertainStatuses = ["recovering", "blocked"];
 function uncertain(subject) {

package/dist/src/authority/transitions.d.ts

@@ -1,3 +1,4 @@
+import type { CommandWaitsState } from "./command-waits.ts";
 import type { RunDetails } from "./details.ts";
 import type { GateState } from "./gates.ts";
 import type { EffectState } from "./operations.ts";
@@ -11,7 +12,7 @@ export interface RunState {
     readonly gates: readonly string[];
 }
 export interface EffectIntent {
-    readonly kind: "start-run" | "continue-run" | "protected-effect";
+    readonly kind: "start-run" | "continue-run" | "resume-hook" | "protected-effect";
     readonly runId: string;
     readonly operationId: string;
 }
@@ -26,9 +27,10 @@ export interface AuthorityState {
     readonly gates: Readonly<Record<string, GateState>>;
     readonly details: Readonly<Record<string, RunDetails>>;
     readonly questions: number;
+    readonly commandWaits?: CommandWaitsState;
     readonly effects: EffectState;
 }
-export declare const eventTypes: readonly ["run.created", "run.started", "gate.opened", "gate.wait-registered", "gate.decided", "gate.expired", "run.paused", "run.resumed", "run.completed", "run.failed", "run.cancelled", "run.observed", "step.dispatched", "effect.requested", "lease.granted", "lease.expired", "effect.reported", "operation.reconciled"];
+export declare const eventTypes: readonly ["run.created", "run.started", "gate.opened", "gate.wait-registered", "gate.decided", "gate.expired", "run.paused", "run.resumed", "run.completed", "run.failed", "run.cancelled", "run.observed", "step.dispatched", "effect.requested", "lease.granted", "lease.expired", "effect.reported", "operation.reconciled", "schedule.skipped", "command-wait.opened", "command-wait.checked", "command-wait.expired"];
 export type EventType = (typeof eventTypes)[number];
 export interface AuthorityEvent {
     readonly sequence: number;
@@ -39,8 +41,8 @@ export interface AuthorityEvent {
     readonly request: string;
     readonly payload: Readonly<Record<string, string>>;
 }
-export declare const numberFields: readonly ["expectedRevision", "generation", "attempt", "limit", "effectLimit", "units", "recoveryLimit", "leaseGeneration", "sourceGeneration", "capacity"];
-export declare const textFields: readonly ["parentRunId", "chainId", "workflow", "submission", "input", "resolver", "gateId", "gateKind", "subject", "prompt", "expiresAt", "decision", "answer", "condition", "reason", "evidence", "retryAt", "operationId", "receipt", "capabilities", "capability", "method", "target", "payload", "inputDigest", "resource", "transport", "source", "worker", "outcome", "outcomeReason", "promotionId", "ticketStatus", "issueUpdate", "result"];
+export declare const numberFields: readonly ["expectedRevision", "expectedObservation", "generation", "attempt", "limit", "effectLimit", "units", "recoveryLimit", "leaseGeneration", "sourceGeneration", "capacity", "slot", "maximumWaitSeconds", "pollEverySeconds"];
+export declare const textFields: readonly ["parentRunId", "chainId", "workflow", "workflowId", "submission", "input", "resolver", "gateId", "gateKind", "waitMode", "subject", "prompt", "expiresAt", "decision", "answer", "condition", "reason", "evidence", "retryAt", "operationId", "receipt", "capabilities", "capability", "method", "target", "payload", "inputDigest", "resource", "transport", "source", "worker", "outcome", "outcomeReason", "promotionId", "ticketStatus", "issueUpdate", "result", "actor", "waitName", "commandOutput"];
 export type NumberField = (typeof numberFields)[number];
 export type TextField = (typeof textFields)[number];
 export type AuthorityCommand = {
@@ -48,7 +50,7 @@ export type AuthorityCommand = {
     readonly commandId: string;
     readonly runId: string;
 } & Readonly<Partial<Record<NumberField, number>>> & Readonly<Partial<Record<TextField, string>>>;
-export type Move = Exclude<EventType, "run.created" | "step.dispatched" | "gate.wait-registered" | "run.observed" | "effect.requested" | "lease.granted" | "lease.expired" | "effect.reported" | "operation.reconciled">;
+export type Move = Exclude<EventType, "run.created" | "step.dispatched" | "gate.wait-registered" | "run.observed" | "effect.requested" | "lease.granted" | "lease.expired" | "effect.reported" | "operation.reconciled" | "schedule.skipped" | "command-wait.opened" | "command-wait.checked" | "command-wait.expired">;
 export declare const terminal: readonly RunStatus[];
 export type Transition = {
     readonly kind: "moved";

package/dist/src/authority/transitions.js

@@ -17,9 +17,14 @@ export const eventTypes = [
     "lease.expired",
     "effect.reported",
     "operation.reconciled",
+    "schedule.skipped",
+    "command-wait.opened",
+    "command-wait.checked",
+    "command-wait.expired",
 ];
 export const numberFields = [
     "expectedRevision",
+    "expectedObservation",
     "generation",
     "attempt",
     "limit",
@@ -29,16 +34,21 @@ export const numberFields = [
     "leaseGeneration",
     "sourceGeneration",
     "capacity",
+    "slot",
+    "maximumWaitSeconds",
+    "pollEverySeconds",
 ];
 export const textFields = [
     "parentRunId",
     "chainId",
     "workflow",
+    "workflowId",
     "submission",
     "input",
     "resolver",
     "gateId",
     "gateKind",
+    "waitMode",
     "subject",
     "prompt",
     "expiresAt",
@@ -66,6 +76,9 @@ export const textFields = [
     "ticketStatus",
     "issueUpdate",
     "result",
+    "actor",
+    "waitName",
+    "commandOutput",
 ];
 export const terminal = [
     "completed",

package/dist/src/broker/actor/authority-delivery.js

@@ -8,6 +8,7 @@ import { pinned } from "./pin-check.js";
 const eligibleStatus = {
     "start-run": "queued",
     "continue-run": "running",
+    "resume-hook": "running",
 };
 function admission(parts, attempt) {
     const { intent, effectId } = attempt;

package/dist/src/broker/actor/authority-queries.d.ts

@@ -1,5 +1,6 @@
 import type { Database } from "solid-objects";
-import type { AuthorityCommand, AuthorityEvent } from "../../authority/transitions.ts";
+import type { ExecutionManifest } from "../../workflow-admission.ts";
+import type { AuthorityCommand, AuthorityEvent, RunState } from "../../authority/transitions.ts";
 import { type SqlConnection } from "../../storage/domain-rows.ts";
 import { type RunLinks, type RunOrigins } from "../../authority/handoff-chain.ts";
 export interface CommandRecord {
@@ -31,13 +32,19 @@ export interface CompletionCursor {
     readonly atOrBefore: number;
     readonly terminal?: boolean;
 }
+export interface RunRecord extends RunState {
+    readonly admissionCommandId: string | null;
+    readonly sequence: number;
+}
 export interface AuthorityQueries {
+    readonly run: (runId: string) => Promise<RunRecord | undefined>;
+    readonly manifest: (runId: string) => Promise<ExecutionManifest | undefined>;
     readonly document: (digest: string) => Promise<string | undefined>;
     readonly receipt: (commandId: string) => Promise<CommandRecord | undefined>;
     readonly eventsAfter: (cursor: EventCursor) => Promise<EventPage>;
     readonly latestCompletion: (cursor: CompletionCursor) => Promise<PublishedEvent | undefined>;
     readonly sequences: () => Promise<Sequences>;
-    readonly workflowIds: () => Promise<Readonly<Record<string, string>>>;
+    readonly workflowIds: (runIds?: readonly string[]) => Promise<Readonly<Record<string, string>>>;
     readonly runOrigins: () => Promise<RunOrigins>;
     readonly runLinks: () => Promise<Readonly<Record<string, RunLinks>>>;
 }

package/dist/src/broker/actor/authority-queries.js

@@ -1,9 +1,21 @@
+import { readExecutionManifest } from "../../execution-pin.js";
 import { parseCommand } from "../../authority/commands.js";
 import { documentBody } from "../../storage/decision-rows.js";
-import { integer, optionalText, latestSequence, text, } from "../../storage/domain-rows.js";
-import { storedEvents } from "../authority-restore.js";
+import { integer, optionalText, latestSequence, runRow, text, } from "../../storage/domain-rows.js";
+import { storedEvents, storedRun } from "../authority-restore.js";
 import { documentDigest } from "../decision-restore.js";
 import { runLinks, } from "../../authority/handoff-chain.js";
+async function run(connection, runId) {
+    const row = await connection.get("SELECT r.run_id, r.status, r.revision, r.workflow, r.paused_from, r.gates, r.sequence, c.command_id AS admission_command_id FROM runs r LEFT JOIN events c ON c.run_id = r.run_id AND c.type = 'run.created' WHERE r.run_id = ? ORDER BY c.sequence LIMIT 1", [runId]);
+    if (row === undefined)
+        return undefined;
+    const record = runRow(row);
+    return {
+        ...storedRun(record),
+        sequence: record.sequence,
+        admissionCommandId: optionalText(row, "admission_command_id"),
+    };
+}
 async function runOrigins(connection, cursor) {
     const rows = await connection.all("SELECT run_id, json_extract(payload, '$.parentRunId') AS parent_run_id, coalesce(json_extract(payload, '$.chainId'), run_id) AS chain_id FROM events WHERE type = 'run.created' AND sequence > ? AND sequence <= ? ORDER BY sequence", [cursor.after, cursor.through]);
     return Object.fromEntries(rows.map((row) => [
@@ -76,8 +88,13 @@ async function sequences(connection) {
         times: Object.fromEntries(rows.map((row) => [text(row, "run_id"), text(row, "recorded_at")])),
     };
 }
-async function workflowIds(connection) {
-    const rows = await connection.all("SELECT d.run_id, json_extract(s.body, '$.workflowId') AS workflow_id FROM run_details d JOIN documents s ON s.digest = d.submission WHERE json_valid(s.body) AND json_type(s.body, '$.workflowId') = 'text'");
+async function workflowIds(connection, runIds) {
+    if (runIds?.length === 0)
+        return {};
+    const scoped = runIds === undefined
+        ? ""
+        : ` AND d.run_id IN (${runIds.map(() => "?").join(", ")})`;
+    const rows = await connection.all(`SELECT d.run_id, json_extract(s.body, '$.workflowId') AS workflow_id FROM run_details d JOIN documents s ON s.digest = d.submission WHERE json_valid(s.body) AND json_type(s.body, '$.workflowId') = 'text'${scoped}`, runIds);
     return Object.fromEntries(rows.map((row) => [text(row, "run_id"), text(row, "workflow_id")]));
 }
 function cachedOrigins(read) {
@@ -122,12 +139,19 @@ export function authorityQueries(read) {
         return links;
     };
     return {
+        run: (runId) => read((connection) => run(connection, runId)),
+        manifest: (runId) => read(async (connection) => {
+            const row = await connection.get("SELECT d.manifest FROM runs r JOIN definitions d ON d.pin = r.workflow WHERE r.run_id = ?", [runId]);
+            return row === undefined
+                ? undefined
+                : readExecutionManifest(JSON.parse(text(row, "manifest")));
+        }),
         document: (digest) => read((connection) => documentBody(connection, digest)),
         receipt: (commandId) => read((connection) => receipt(connection, commandId)),
         eventsAfter: (cursor) => read((connection) => eventsAfter(connection, cursor)),
         latestCompletion: (cursor) => read((connection) => latestCompletion(connection, cursor)),
         sequences: () => read(sequences),
-        workflowIds: () => read(workflowIds),
+        workflowIds: (runIds) => read((connection) => workflowIds(connection, runIds)),
         runOrigins: origins,
         runLinks: currentLinks,
     };

package/dist/src/broker/actor/authority-receipts.js

@@ -1,5 +1,6 @@
 import { integer, text } from "../../storage/domain-rows.js";
 import { intentKind } from "../authority-restore.js";
+import { commandLimits } from "../../command-settings.js";
 export const unserializable = {
     kind: "rejected",
     reason: "invalid-command",
@@ -16,7 +17,16 @@ export const tooLarge = {
     detail: `A request has at most ${String(maximumRequestBytes)} bytes.`,
 };
 export function oversized(request) {
-    return Buffer.byteLength(JSON.stringify(request)) > maximumRequestBytes;
+    const command = request["command"];
+    const report = typeof command === "object" &&
+        command !== null &&
+        !Array.isArray(command) &&
+        command["kind"] === "report-effect" &&
+        command["worker"] === "command";
+    const limit = report
+        ? maximumRequestBytes + commandLimits.stdoutBytes * 12
+        : maximumRequestBytes;
+    return Buffer.byteLength(JSON.stringify(request)) > limit;
 }
 export function failed(error) {
     const name = String(Reflect.get(Object(error), "name"));

package/dist/src/broker/actor/authority-session.js

@@ -1,5 +1,5 @@
 import { eligibleIntents, } from "../../authority/reducer.js";
-import { loadProjections, } from "../../storage/decision-snapshots.js";
+import { loadOperationSnapshot, loadProjections, } from "../../storage/decision-snapshots.js";
 import { loadEvents } from "../../storage/domain-rows.js";
 import { pinnedResumption } from "../authority-admission.js";
 import { currentState, intentKind, storedDefinitions, storedEvents, } from "../authority-restore.js";
@@ -40,7 +40,7 @@ export function authoritySession(parts) {
             return storedEvents(await parts.reader.connection(loadEvents));
         },
         eligibleIntents: async () => eligible(await projected(session)),
-        operations: async () => operationRecords(await projected(session)),
+        operations: async () => operationRecords(await read(loadOperationSnapshot)),
         resumePinned: async (runId, manifest) => {
             const snapshot = await projected(session);
             return pinnedResumption(currentState(snapshot), {

package/dist/src/broker/actor/authority-settings.js

@@ -1,5 +1,6 @@
 import { recoveryBound } from "../../authority/commands.js";
 import { lineLogger } from "./authority-logger.js";
+import { commandLimits } from "../../command-settings.js";
 export const authorityTiming = {
     leaseMilliseconds: 2_000,
     renewalMilliseconds: 500,
@@ -50,6 +51,9 @@ export function librarySettings(timing, log) {
     return {
         tableNamePrefix: "solid_",
         maxAttempts: 1,
+        maxActivationDurationMilliseconds: 50,
+        maxPayloadBytes: 900_000 + commandLimits.stdoutBytes * 12,
+        maxResultBytes: 900_000 + commandLimits.stdoutBytes * 12,
         logger: lineLogger(log ?? standardError),
         leaseDurationMilliseconds: timing.leaseMilliseconds,
         leaseRenewalIntervalMilliseconds: timing.renewalMilliseconds,

package/dist/src/broker/actor/authority-upgrades.d.ts

@@ -1,7 +1,8 @@
-import { loadDecided } from "../../storage/decision-snapshots.ts";
+import { upgradeCommandWaits, upgradeCommandWaitChecks } from "../command-wait-restore.ts";
+import { loadDecided, loadEffected, loadWaiting, loadCurrent } from "../../storage/decision-snapshots.ts";
 import { loadSnapshot } from "../../storage/domain-rows.ts";
 import { upgradeDecisions } from "../decision-upgrade.ts";
-import { upgradeEffects } from "../effect-upgrade.ts";
+import { upgradeEffects, upgradeEventIndexes, upgradeSpendingIndexes } from "../effect-upgrade.ts";
 declare const upgrades: readonly [{
     readonly from: "adopted";
     readonly load: typeof loadSnapshot;
@@ -10,6 +11,22 @@ declare const upgrades: readonly [{
     readonly from: "decided";
     readonly load: typeof loadDecided;
     readonly apply: typeof upgradeEffects;
+}, {
+    readonly from: "effected";
+    readonly load: typeof loadEffected;
+    readonly apply: typeof upgradeCommandWaits;
+}, {
+    readonly from: "waiting";
+    readonly load: typeof loadWaiting;
+    readonly apply: typeof upgradeCommandWaitChecks;
+}, {
+    readonly from: "checked";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeSpendingIndexes;
+}, {
+    readonly from: "indexed";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeEventIndexes;
 }];
 export type UpgradeStep = (typeof upgrades)[number];
 export declare function upgradeSteps(kind: string): readonly UpgradeStep[];

package/dist/src/broker/actor/authority-upgrades.js

@@ -1,10 +1,15 @@
-import { loadDecided } from "../../storage/decision-snapshots.js";
+import { upgradeCommandWaits, upgradeCommandWaitChecks, } from "../command-wait-restore.js";
+import { loadDecided, loadEffected, loadWaiting, loadCurrent, } from "../../storage/decision-snapshots.js";
 import { loadSnapshot } from "../../storage/domain-rows.js";
 import { upgradeDecisions } from "../decision-upgrade.js";
-import { upgradeEffects } from "../effect-upgrade.js";
+import { upgradeEffects, upgradeEventIndexes, upgradeSpendingIndexes, } from "../effect-upgrade.js";
 const upgrades = [
     { from: "adopted", load: loadSnapshot, apply: upgradeDecisions },
     { from: "decided", load: loadDecided, apply: upgradeEffects },
+    { from: "effected", load: loadEffected, apply: upgradeCommandWaits },
+    { from: "waiting", load: loadWaiting, apply: upgradeCommandWaitChecks },
+    { from: "checked", load: loadCurrent, apply: upgradeSpendingIndexes },
+    { from: "indexed", load: loadCurrent, apply: upgradeEventIndexes },
 ];
 export function upgradeSteps(kind) {
     const first = upgrades.findIndex((step) => step.from === kind);

package/dist/src/broker/actor/delivery-intents.d.ts

@@ -18,6 +18,7 @@ export interface DeliveryIntent extends EffectIntent, Partial<DeliveryDecision>,
     readonly workflow: string;
     readonly inputDigest: string;
     readonly purpose: Purpose;
+    readonly commandDeadline?: string;
 }
 export type ProtectedIntent = DeliveryIntent & EffectArguments;
 export type Undigested = Omit<DeliveryIntent, "inputDigest" | "purpose">;
@@ -54,7 +55,7 @@ export interface Delivery {
     readonly retry?: Delivery["reconcile"];
     readonly stop?: (intent: DeliveryIntent, signal: AbortSignal) => Promise<AdapterOutcome>;
 }
-export type AvailableManifest = (pinned: ExecutionManifest) => ExecutionManifest | undefined;
+export type AvailableManifest = (pinned: ExecutionManifest) => ExecutionManifest | undefined | Promise<ExecutionManifest | undefined>;
 export declare function intentDigest(intent: Undigested): string;
 export declare function deliveryIntent(intent: Undigested): DeliveryIntent;
 export declare function protectedIntent(state: AuthorityState, attempt: {

package/dist/src/broker/actor/delivery-intents.js

@@ -1,7 +1,8 @@
+import { commandWaitDeadline } from "../command-wait-restore.js";
 import { createHash } from "node:crypto";
-import { continues, isSettled } from "../../authority/gates.js";
+import { delivers, isSettled } from "../../authority/gates.js";
 import { eligibleIntents, } from "../../authority/reducer.js";
-import { gateContinuation } from "../../authority/reduction.js";
+import { gateDelivery } from "../../authority/reduction.js";
 export function intentDigest(intent) {
     const base = [intent.kind, intent.runId, intent.operationId, intent.workflow];
     const bound = intent.gateId === undefined
@@ -26,6 +27,7 @@ export function protectedIntent(state, attempt) {
         .filter((lease) => lease.resourceKey === operation.leaseKey)
         .map((lease) => ({
         ...intent,
+        ...commandWaitDeadline(state.commandWaits ?? {}, operation.operationId),
         workflow,
         purpose,
         inputDigest: operation.inputDigest,
@@ -37,7 +39,7 @@ export function protectedIntent(state, attempt) {
     })));
 }
 export function gateDecision(gate) {
-    if (gate === undefined || gate.kind === "untyped" || !continues(gate))
+    if (gate === undefined || gate.kind === "untyped" || !delivers(gate))
         return null;
     return {
         gateId: gate.gateId,
@@ -51,7 +53,7 @@ function workflowOf(state, runId) {
 function decisionOf(state, intent) {
     return gateDecision(Object.values(state.gates).find((gate) => gate.runId === intent.runId &&
         isSettled(gate) &&
-        gateContinuation(gate).operationId === intent.operationId));
+        gateDelivery(gate).operationId === intent.operationId));
 }
 export function observingIntent(state, attempt) {
     const { intent, purpose } = attempt;

package/dist/src/broker/actor/effect-delivery.d.ts

@@ -16,4 +16,4 @@ export interface Attempt {
     readonly effectId: string;
 }
 export declare function deliverEffect(parts: EffectDelivery, attempt: Attempt): Promise<JsonObject>;
-export declare function protectedArguments(value: JsonObject): EffectArguments;
+export declare function protectedArguments(value: JsonObject): EffectArguments & Pick<ProtectedIntent, "commandDeadline">;

package/dist/src/broker/actor/effect-delivery.js

@@ -1,9 +1,11 @@
+import { dispatchCommandWait, readCommandWaitDeadline, } from "../command-wait-restore.js";
 import { dispatchDecision, } from "../../authority/dispatch-decision.js";
 import { integer, optionalInteger, optionalText, text, } from "../../storage/domain-rows.js";
 import { budgetRow } from "../../storage/effect-rows.js";
 import { effectInputDigest, storedBudget } from "../effect-restore.js";
 import { settlement } from "./adapter-outcomes.js";
 import { pinned } from "./pin-check.js";
+import { commandLimits } from "../../command-settings.js";
 const absent = {
     attempt: undefined,
     released: false,
@@ -15,11 +17,16 @@ const absent = {
     budget: undefined,
 };
 async function dispatchRead(connection, attempt) {
-    const row = await connection.get("SELECT d.outcome, d.input_digest, a.lease_generation, o.released_sequence, r.status, r.workflow, p.status AS operation_status, p.capability, p.units, l.operation_id AS lease_owner, l.generation, l.expires_at, l.state AS lease_state, b.run_id, b.effect_limit, b.spent, b.capabilities, b.exhausted_sequence FROM deliveries d JOIN outbox o ON o.operation_id = d.operation_id JOIN runs r ON r.run_id = o.run_id JOIN dispatch_attempts a ON a.effect_id = d.effect_id JOIN operations p ON p.operation_id = d.operation_id JOIN leases l ON l.resource_key = p.lease_key JOIN run_budgets b ON b.run_id = o.run_id WHERE d.effect_id = ? AND d.operation_id = ?", [attempt.effectId, attempt.intent.operationId]);
+    const row = await connection.get("SELECT d.outcome, d.input_digest, a.lease_generation, o.released_sequence, r.status, r.workflow, p.status AS operation_status, p.capability, p.units, p.method, p.target, p.payload, l.operation_id AS lease_owner, l.generation, l.expires_at, l.state AS lease_state, b.run_id, b.effect_limit, b.spent, b.capabilities, b.exhausted_sequence FROM deliveries d JOIN outbox o ON o.operation_id = d.operation_id JOIN runs r ON r.run_id = o.run_id JOIN dispatch_attempts a ON a.effect_id = d.effect_id JOIN operations p ON p.operation_id = d.operation_id JOIN leases l ON l.resource_key = p.lease_key JOIN run_budgets b ON b.run_id = o.run_id WHERE d.effect_id = ? AND d.operation_id = ?", [attempt.effectId, attempt.intent.operationId]);
     if (row === undefined)
         return absent;
     const workflow = text(row, "workflow");
     return {
+        ...(await dispatchCommandWait(connection, {
+            runId: attempt.intent.runId,
+            workflow,
+            operationId: attempt.intent.operationId,
+        })),
         attempt: {
             outcome: optionalText(row, "outcome"),
             inputDigest: text(row, "input_digest"),
@@ -36,6 +43,9 @@ async function dispatchRead(connection, attempt) {
             status: text(row, "operation_status"),
             capability: text(row, "capability"),
             units: integer(row, "units"),
+            method: text(row, "method"),
+            target: text(row, "target"),
+            payload: text(row, "payload"),
         },
         budget: storedBudget(budgetRow(row)),
         lease: {
@@ -51,7 +61,10 @@ async function called(parts, call) {
     const adapterCall = parts.delivery[name]?.bind(parts.delivery);
     if (adapterCall === undefined)
         return { kind: "unknown", reason: "stop-not-supported" };
-    return settlement(await parts.calls.call(intent.operationId, (signal) => adapterCall(intent, signal)), parts.effects.resultBytes);
+    const deadline = await parts.reader.connection((connection) => readCommandWaitDeadline(connection, intent));
+    return settlement(await parts.calls.call(intent.operationId, (signal) => adapterCall({ ...intent, ...deadline }, signal)), intent.capability === "command"
+        ? commandLimits.resultBytes
+        : parts.effects.resultBytes);
 }
 const confirmations = ["stopped", "completed", "failed"];
 async function stopRequested(parts, intent) {
@@ -123,5 +136,8 @@ export function protectedArguments(value) {
         method: text(value, "method"),
         target: text(value, "target"),
         payload: text(value, "payload"),
+        ...(typeof value["commandDeadline"] === "string"
+            ? { commandDeadline: value["commandDeadline"] }
+            : {}),
     };
 }

package/dist/src/broker/actor/effect-plan.js

@@ -1,3 +1,5 @@
+import { operationPhase } from "../../operation-identity.js";
+import { endedCommandWaitCheck } from "../../authority/command-waits.js";
 import { isInstant } from "../../authority/instants.js";
 import { promotions } from "../../authority/leases.js";
 import { execute } from "../../authority/reducer.js";
@@ -45,7 +47,9 @@ function closure(state, operation) {
         return ["run-ended"];
     const exhausted = (state.effects.budgets[operation.runId]?.exhaustedSequence ?? null) !==
         null;
-    return exhausted ? ["budget-exhausted"] : [];
+    return exhausted && operationPhase(operation.operationId) !== "command-wait"
+        ? ["budget-exhausted"]
+        : [];
 }
 function unblocked(state, followUp) {
     const steps = [];
@@ -95,6 +99,9 @@ const outcomes = {
     }),
     completed: (attempt) => stated({
         outcome: "completed",
+        ...(attempt.worker === "command"
+            ? { commandOutput: attempt.result }
+            : {}),
         result: attempt.result === null ? null : documentDigest(attempt.result),
         reason: attempt.reason,
         ...identified(attempt),
@@ -155,7 +162,9 @@ function acceptedNow(reconciling) {
 export function reconciliations(state, reconciling) {
     const accepted = acceptedNow(reconciling);
     const rejected = rejectedReports(reconciling.results);
-    const due = (operation) => state.runs[operation.runId]?.status !== "cancelled" && live(operation);
+    const due = (operation) => state.runs[operation.runId]?.status !== "cancelled" &&
+        !endedCommandWaitCheck(state.commandWaits ?? {}, operation.operationId) &&
+        live(operation);
     const live = (operation) => (operation.status === "recovering" &&
         !accepted.has(operation.operationId)) ||
         (operation.status === "blocked" && rejected.has(operation.operationId));

package/dist/src/broker/actor/effect-queries.d.ts

@@ -8,6 +8,10 @@ export interface OperationView extends OperationRow {
 }
 export type StoredDisposition = Omit<Evidence, "sequence">;
 export interface EffectQueries {
+    readonly workerTurns: (window: {
+        readonly after: string;
+        readonly through: string;
+    }) => Promise<readonly OperationView[]>;
     readonly operation: (operationId: string) => Promise<OperationView | undefined>;
     readonly operations: (runId: string) => Promise<readonly OperationView[]>;
     readonly evidence: (runId: string) => Promise<readonly number[]>;
@@ -16,6 +20,7 @@ export interface EffectQueries {
     readonly dueLeases: (instant: string) => Promise<readonly DueLease[]>;
     readonly stops: (runId: string) => Promise<readonly string[]>;
     readonly closure: (runId: string) => Promise<RunClosure>;
+    readonly settledAt: (runId: string) => Promise<number | null>;
 }
 export interface DueLease {
     readonly runId: string;

package/dist/src/broker/actor/effect-queries.js

@@ -3,6 +3,12 @@ import { integer, optionalInteger, optionalText, text, } from "../../storage/dom
 import { budgetColumns, budgetRow, evidenceColumns, evidenceRow, operationColumns, operationRow, } from "../../storage/effect-rows.js";
 import { storedBudget } from "../effect-restore.js";
 import { stopping } from "./stop-turn.js";
+async function settledAt(connection, runId) {
+    const rows = await connection.all("SELECT recorded_at FROM events WHERE sequence IN (SELECT settled_sequence FROM operations WHERE run_id = ? UNION ALL SELECT min(e.sequence) FROM events e JOIN stopped_operations s ON s.run_id = e.run_id AND s.operation_id = json_extract(e.payload, '$.operationId') WHERE e.run_id = ? AND (EXISTS (SELECT 1 FROM deliveries d WHERE d.operation_id = s.operation_id AND d.outcome = 'sent') OR (SELECT d.outcome FROM deliveries d WHERE d.operation_id = s.operation_id ORDER BY d.staged_sequence DESC, d.rowid DESC LIMIT 1) NOT IN ('withheld', 'blocked', 'absent')) AND ((e.type = 'run.observed' AND json_extract(e.payload, '$.condition') = 'stopped') OR (e.type = 'operation.reconciled' AND json_extract(e.payload, '$.outcome') = 'stopped')) GROUP BY s.operation_id)", [runId, runId]);
+    if (rows.length === 0)
+        return null;
+    return rows.reduce((latest, row) => Math.max(latest, Date.parse(text(row, "recorded_at"))), -Infinity);
+}
 async function storedClosure(connection, scope) {
     const { subjects, cancelledSequence } = await stopping(connection, scope.runId);
     return runClosure(subjects, {
@@ -28,7 +34,9 @@ const viewColumns = operationColumns
     .map((name) => `o.${name}`)
     .join(", ");
 const waitCause = "CASE WHEN o.status != 'waiting' THEN NULL WHEN r.status != 'running' THEN 'run-not-running' WHEN EXISTS (SELECT 1 FROM leases h WHERE h.resource_key = o.lease_key AND h.state != 'released') THEN 'resource' ELSE 'ready' END";
-const viewed = `SELECT ${viewColumns}, l.generation AS lease_generation, ${waitCause} AS wait_cause FROM operations o JOIN runs r ON r.run_id = o.run_id LEFT JOIN leases l ON l.resource_key = o.lease_key AND l.operation_id = o.operation_id AND l.state != 'released'`;
+const viewSelection = `SELECT ${viewColumns}, l.generation AS lease_generation, ${waitCause} AS wait_cause`;
+const viewJoins = "JOIN runs r ON r.run_id = o.run_id LEFT JOIN leases l ON l.resource_key = o.lease_key AND l.operation_id = o.operation_id AND l.state != 'released'";
+const viewed = `${viewSelection} FROM operations o ${viewJoins}`;
 function operationView(row) {
     return {
         ...operationRow(row),
@@ -46,6 +54,11 @@ async function operations(connection, runId) {
     const rows = await connection.all(`${viewed} WHERE o.run_id = ? ORDER BY o.requested_sequence`, [runId]);
     return rows.map(operationView);
 }
+async function workerTurns(connection, window) {
+    const settled = await connection.all(`${viewSelection} FROM events e CROSS JOIN operations o ${viewJoins} WHERE o.settled_sequence = e.sequence AND o.capability = 'claude' AND julianday(e.recorded_at) > julianday(?) AND julianday(e.recorded_at) <= julianday(?)`, [window.after, window.through]);
+    const open = await connection.all(`${viewSelection} FROM operations o INDEXED BY operations_worker_open ${viewJoins} WHERE o.capability = 'claude' AND o.settled_sequence IS NULL AND o.status NOT IN ('refused', 'completed', 'failed', 'stopped', 'cancelled')`);
+    return [...settled, ...open].map(operationView);
+}
 async function evidence(connection, runId) {
     const rows = await connection.all("SELECT e.sequence FROM effect_evidence e JOIN events v ON v.sequence = e.sequence WHERE e.run_id = ? AND (e.disposition IN ('rejected', 'denied', 'refused') OR (v.type = 'operation.reconciled' AND json_extract(v.payload, '$.outcome') = 'unknown')) ORDER BY e.sequence", [runId]);
     return rows.map((row) => integer(row, "sequence"));
@@ -68,8 +81,10 @@ async function budget(connection, runId) {
 }
 export function effectQueries(read, settings) {
     return {
+        workerTurns: (window) => read((connection) => workerTurns(connection, window)),
         stops: (runId) => read((connection) => stops(connection, runId)),
         closure: (runId) => read((connection) => storedClosure(connection, { runId, stopLimit: settings.stopLimit })),
+        settledAt: (runId) => read((connection) => settledAt(connection, runId)),
         budget: (runId) => read((connection) => budget(connection, runId)),
         dueLeases: (instant) => read((connection) => dueLeases(connection, instant)),
         operation: (operationId) => read((connection) => operation(connection, operationId)),