cloving 0.3.20 → 1.0.1

package/dist/src/broker/actor/effect-plan.js

@@ -0,0 +1,173 @@
+import { isInstant } from "../../authority/instants.js";
+import { promotions } from "../../authority/leases.js";
+import { execute } from "../../authority/reducer.js";
+import { documentDigest } from "../decision-restore.js";
+import { operationDigest } from "../operation-digest.js";
+import { protectedIntent } from "./delivery-intents.js";
+function leaseGrant(state, grant) {
+    const { operation, recordedAt, effects } = grant;
+    const generation = (state.effects.leases[operation.leaseKey]?.generation ?? 0) + 1;
+    return {
+        kind: "grant-lease",
+        commandId: `lease:${operationDigest(operation.operationId)}:${String(generation)}`,
+        runId: operation.runId,
+        operationId: operation.operationId,
+        leaseGeneration: generation,
+        expiresAt: new Date(Date.parse(recordedAt) + effects.leaseMilliseconds).toISOString(),
+        capacity: effects.capacity,
+    };
+}
+function recordedStep(before, proposed) {
+    return [execute(before, proposed)]
+        .filter((result) => result.kind === "accepted")
+        .map((result) => ({ result, before }));
+}
+function granted(before, grant) {
+    return recordedStep(before, {
+        command: leaseGrant(before, grant),
+        recordedAt: grant.recordedAt,
+    });
+}
+export function promoted(state, promotion) {
+    if (!isInstant(promotion.recordedAt))
+        return [];
+    const steps = [];
+    for (const operation of promotions(state, promotion.effects.capacity))
+        steps.push(...granted(steps.at(-1)?.result.state ?? state, {
+            ...promotion,
+            operation,
+        }));
+    return steps;
+}
+const terminal = ["completed", "failed", "cancelled"];
+function closure(state, operation) {
+    if (terminal.includes(state.runs[operation.runId]?.status ?? ""))
+        return ["run-ended"];
+    const exhausted = (state.effects.budgets[operation.runId]?.exhaustedSequence ?? null) !==
+        null;
+    return exhausted ? ["budget-exhausted"] : [];
+}
+function unblocked(state, followUp) {
+    const steps = [];
+    for (const [operationId, effectId] of followUp.blocked) {
+        const before = steps.at(-1)?.result.state ?? state;
+        const pending = Object.values(before.effects.operations).filter((operation) => operation.operationId === operationId &&
+            operation.status === "dispatch-pending" &&
+            operation.receipt === null);
+        for (const operation of pending)
+            for (const reason of closure(before, operation))
+                steps.push(...recordedStep(before, {
+                    command: {
+                        kind: "record-outcome",
+                        commandId: `outcome:${effectId}`,
+                        runId: operation.runId,
+                        operationId,
+                        leaseGeneration: before.effects.leases[operation.leaseKey]?.generation ?? 0,
+                        outcome: "not-started",
+                        reason,
+                    },
+                    recordedAt: followUp.recordedAt,
+                }));
+    }
+    return steps;
+}
+export function followUps(state, followUp) {
+    const settled = unblocked(state, followUp);
+    return [
+        ...settled,
+        ...promoted(settled.at(-1)?.result.state ?? state, followUp),
+    ];
+}
+function stated(fields) {
+    return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== null));
+}
+function identified(attempt) {
+    return { receipt: attempt.receipt, worker: attempt.worker };
+}
+function unknown(reason) {
+    return { outcome: "unknown", reason };
+}
+const outcomes = {
+    accepted: (attempt) => ({
+        outcome: "accepted",
+        receipt: attempt.receipt,
+        worker: attempt.worker,
+    }),
+    completed: (attempt) => stated({
+        outcome: "completed",
+        result: attempt.result === null ? null : documentDigest(attempt.result),
+        reason: attempt.reason,
+        ...identified(attempt),
+    }),
+    failed: (attempt) => stated({
+        outcome: "failed",
+        reason: attempt.reason,
+        result: attempt.result === null ? null : documentDigest(attempt.result),
+        ...identified(attempt),
+    }),
+    stopped: (attempt) => stated({ outcome: "stopped", ...identified(attempt) }),
+    unknown: (attempt) => unknown(attempt.reason ?? "adapter-unknown"),
+    thrown: () => unknown("handler-failure"),
+    "stale-epoch": () => unknown("stale-epoch"),
+    absent: (attempt) => attempt.storedReceipt === null
+        ? { outcome: "not-started", reason: "absent" }
+        : unknown("absent-with-receipt"),
+    withheld: (attempt) => attempt.purpose === "dispatch"
+        ? { outcome: "not-started", reason: "withheld" }
+        : undefined,
+    refused: () => ({ outcome: "not-started", reason: "budget-exhausted" }),
+};
+export function outcomeCommand(attempt) {
+    const fields = outcomes[attempt.outcome]?.(attempt);
+    if (fields === undefined)
+        return undefined;
+    return {
+        kind: "record-outcome",
+        commandId: `outcome:${attempt.effectId}`,
+        runId: attempt.runId,
+        operationId: attempt.operationId,
+        leaseGeneration: attempt.leaseGeneration,
+        ...fields,
+    };
+}
+export function resultDocuments(attempt) {
+    const kept = ["completed", "failed"].includes(attempt.outcome);
+    if (!kept || attempt.result === null)
+        return [];
+    return [{ digest: documentDigest(attempt.result), body: attempt.result }];
+}
+function named(reconciling) {
+    const operations = reconciling.results.flatMap((result) => result.events.flatMap((event) => event.payload["operationId"] ?? []));
+    return [...new Set([...operations, ...reconciling.closed])];
+}
+export function rejectedReports(results) {
+    return new Set(results
+        .filter((result) => result.events.some((event) => event.type === "effect.reported") &&
+        result.disposition?.disposition === "rejected")
+        .flatMap((result) => result.disposition?.operationId ?? []));
+}
+function acceptedNow(reconciling) {
+    return new Set(reconciling.results.flatMap((result) => result.events
+        .filter((event) => event.type === "operation.reconciled" &&
+        event.payload["outcome"] === "accepted")
+        .flatMap((event) => event.payload["operationId"] ?? [])));
+}
+export function reconciliations(state, reconciling) {
+    const accepted = acceptedNow(reconciling);
+    const rejected = rejectedReports(reconciling.results);
+    const due = (operation) => state.runs[operation.runId]?.status !== "cancelled" && live(operation);
+    const live = (operation) => (operation.status === "recovering" &&
+        !accepted.has(operation.operationId)) ||
+        (operation.status === "blocked" && rejected.has(operation.operationId));
+    return named(reconciling)
+        .filter((operationId) => !reconciling.open.has(operationId))
+        .flatMap((operationId) => Object.values(state.effects.operations).filter((operation) => operation.operationId === operationId && due(operation)))
+        .flatMap((operation) => protectedIntent(state, {
+        intent: {
+            kind: "protected-effect",
+            runId: operation.runId,
+            operationId: operation.operationId,
+        },
+        purpose: "reconcile",
+    }));
+}

package/dist/src/broker/actor/effect-queries.d.ts

@@ -0,0 +1,27 @@
+import type { Evidence, RunBudget } from "../../authority/operations.ts";
+import { type RunClosure } from "../../authority/stop-sequence.ts";
+import { type OperationRow } from "../../storage/effect-rows.ts";
+import type { Read } from "./authority-queries.ts";
+export interface OperationView extends OperationRow {
+    readonly leaseGeneration: number | null;
+    readonly waitCause: string | null;
+}
+export type StoredDisposition = Omit<Evidence, "sequence">;
+export interface EffectQueries {
+    readonly operation: (operationId: string) => Promise<OperationView | undefined>;
+    readonly operations: (runId: string) => Promise<readonly OperationView[]>;
+    readonly evidence: (runId: string) => Promise<readonly number[]>;
+    readonly disposition: (sequence: number) => Promise<StoredDisposition | undefined>;
+    readonly budget: (runId: string) => Promise<RunBudget | undefined>;
+    readonly dueLeases: (instant: string) => Promise<readonly DueLease[]>;
+    readonly stops: (runId: string) => Promise<readonly string[]>;
+    readonly closure: (runId: string) => Promise<RunClosure>;
+}
+export interface DueLease {
+    readonly runId: string;
+    readonly operationId: string;
+    readonly generation: number;
+}
+export declare function effectQueries(read: Read, settings: {
+    readonly stopLimit: number;
+}): EffectQueries;

package/dist/src/broker/actor/effect-queries.js

@@ -0,0 +1,80 @@
+import { runClosure } from "../../authority/stop-sequence.js";
+import { integer, optionalInteger, optionalText, text, } from "../../storage/domain-rows.js";
+import { budgetColumns, budgetRow, evidenceColumns, evidenceRow, operationColumns, operationRow, } from "../../storage/effect-rows.js";
+import { storedBudget } from "../effect-restore.js";
+import { stopping } from "./stop-turn.js";
+async function storedClosure(connection, scope) {
+    const { subjects, cancelledSequence } = await stopping(connection, scope.runId);
+    return runClosure(subjects, {
+        cancelledSequence: cancelledSequence ?? 0,
+        stopLimit: scope.stopLimit,
+    });
+}
+async function stops(connection, runId) {
+    const rows = await connection.all("SELECT operation_id FROM stopped_operations WHERE run_id = ? ORDER BY operation_id", [runId]);
+    return rows.map((row) => text(row, "operation_id"));
+}
+const dueBatch = 100;
+async function dueLeases(connection, instant) {
+    const rows = await connection.all("SELECT o.run_id, l.operation_id, l.generation FROM leases l JOIN operations o ON o.operation_id = l.operation_id WHERE l.state = 'held' AND l.expires_at <= ? ORDER BY l.expires_at, l.resource_key LIMIT ?", [new Date(instant).toISOString(), dueBatch]);
+    return rows.map((row) => ({
+        runId: text(row, "run_id"),
+        operationId: text(row, "operation_id"),
+        generation: integer(row, "generation"),
+    }));
+}
+const viewColumns = operationColumns
+    .split(", ")
+    .map((name) => `o.${name}`)
+    .join(", ");
+const waitCause = "CASE WHEN o.status != 'waiting' THEN NULL WHEN r.status != 'running' THEN 'run-not-running' WHEN EXISTS (SELECT 1 FROM leases h WHERE h.resource_key = o.lease_key AND h.state != 'released') THEN 'resource' ELSE 'ready' END";
+const viewed = `SELECT ${viewColumns}, l.generation AS lease_generation, ${waitCause} AS wait_cause FROM operations o JOIN runs r ON r.run_id = o.run_id LEFT JOIN leases l ON l.resource_key = o.lease_key AND l.operation_id = o.operation_id AND l.state != 'released'`;
+function operationView(row) {
+    return {
+        ...operationRow(row),
+        leaseGeneration: optionalInteger(row, "lease_generation"),
+        waitCause: optionalText(row, "wait_cause"),
+    };
+}
+async function operation(connection, operationId) {
+    const row = await connection.get(`${viewed} WHERE o.operation_id = ?`, [
+        operationId,
+    ]);
+    return row === undefined ? undefined : operationView(row);
+}
+async function operations(connection, runId) {
+    const rows = await connection.all(`${viewed} WHERE o.run_id = ? ORDER BY o.requested_sequence`, [runId]);
+    return rows.map(operationView);
+}
+async function evidence(connection, runId) {
+    const rows = await connection.all("SELECT e.sequence FROM effect_evidence e JOIN events v ON v.sequence = e.sequence WHERE e.run_id = ? AND (e.disposition IN ('rejected', 'denied', 'refused') OR (v.type = 'operation.reconciled' AND json_extract(v.payload, '$.outcome') = 'unknown')) ORDER BY e.sequence", [runId]);
+    return rows.map((row) => integer(row, "sequence"));
+}
+async function disposition(connection, sequence) {
+    const row = await connection.get(`SELECT ${evidenceColumns} FROM effect_evidence WHERE sequence = ?`, [sequence]);
+    if (row === undefined)
+        return undefined;
+    const stored = evidenceRow(row);
+    return {
+        runId: stored.runId,
+        operationId: stored.operationId,
+        disposition: stored.disposition,
+        reason: stored.reason,
+    };
+}
+async function budget(connection, runId) {
+    const row = await connection.get(`SELECT ${budgetColumns} FROM run_budgets WHERE run_id = ?`, [runId]);
+    return row === undefined ? undefined : storedBudget(budgetRow(row));
+}
+export function effectQueries(read, settings) {
+    return {
+        stops: (runId) => read((connection) => stops(connection, runId)),
+        closure: (runId) => read((connection) => storedClosure(connection, { runId, stopLimit: settings.stopLimit })),
+        budget: (runId) => read((connection) => budget(connection, runId)),
+        dueLeases: (instant) => read((connection) => dueLeases(connection, instant)),
+        operation: (operationId) => read((connection) => operation(connection, operationId)),
+        operations: (runId) => read((connection) => operations(connection, runId)),
+        evidence: (runId) => read((connection) => evidence(connection, runId)),
+        disposition: (sequence) => read((connection) => disposition(connection, sequence)),
+    };
+}

package/dist/src/broker/actor/effect-turn.d.ts

@@ -0,0 +1,25 @@
+import type { RunDetails } from "../../authority/details.ts";
+import type { EffectState } from "../../authority/operations.ts";
+import type { EffectIntent, RunState } from "../../authority/reducer.ts";
+import { type SqlConnection } from "../../storage/domain-rows.ts";
+import type { DeliveryIntent, Purpose, Undigested } from "./delivery-intents.ts";
+export interface EffectScope {
+    readonly runId: string;
+    readonly operations: readonly string[];
+    readonly keys: readonly string[];
+    readonly capacity: number;
+}
+export declare function effectTurn(connection: SqlConnection, scope: EffectScope): Promise<EffectState>;
+export declare function pendingEffect(connection: SqlConnection, attempt: {
+    readonly intent: Undigested;
+    readonly purpose: Purpose;
+}): Promise<DeliveryIntent>;
+export interface Neighbours {
+    readonly runs: Readonly<Record<string, RunState>>;
+    readonly details: Readonly<Record<string, RunDetails>>;
+    readonly intents: readonly EffectIntent[];
+}
+export declare function neighbours(connection: SqlConnection, scope: {
+    readonly runId: string;
+    readonly effects: EffectState;
+}): Promise<Neighbours>;

package/dist/src/broker/actor/effect-turn.js

@@ -0,0 +1,84 @@
+import { detailColumns, detailRow } from "../../storage/decision-rows.js";
+import { integer, runRow, text, } from "../../storage/domain-rows.js";
+import { storedRun } from "../authority-restore.js";
+import { storedDetails } from "../decision-restore.js";
+import { budgetColumns, budgetRow, leaseColumns, leaseRow, operationColumns, operationRow, } from "../../storage/effect-rows.js";
+import { storedEffects } from "../effect-restore.js";
+const candidateColumns = operationColumns
+    .split(", ")
+    .map((name) => `p.${name}`)
+    .join(", ");
+const oldestElsewhere = "p.status = 'waiting' AND r.status = 'running' AND p.run_id != ? AND NOT EXISTS (SELECT 1 FROM operations q JOIN runs s ON s.run_id = q.run_id WHERE q.lease_key = p.lease_key AND q.status = 'waiting' AND s.status = 'running' AND q.run_id != ? AND q.wait_sequence < p.wait_sequence)";
+async function candidates(connection, scope) {
+    const { runId, capacity } = scope;
+    const source = `SELECT ${candidateColumns} FROM operations p JOIN runs r ON r.run_id = p.run_id WHERE ${oldestElsewhere}`;
+    return [
+        ...(await connection.all(`${source} AND NOT EXISTS (SELECT 1 FROM leases l WHERE l.resource_key = p.lease_key AND l.state != 'released') ORDER BY p.wait_sequence LIMIT ?`, [runId, runId, capacity])),
+        ...(await connection.all(`${source} AND p.lease_key IN (SELECT l.resource_key FROM leases l JOIN operations o ON o.operation_id = l.operation_id WHERE o.run_id = ? AND l.state != 'released') ORDER BY p.wait_sequence`, [runId, runId, runId])),
+    ];
+}
+const open = "status NOT IN ('refused', 'completed', 'failed', 'stopped', 'cancelled')";
+async function operations(connection, scope) {
+    const rows = [
+        ...(await connection.all(`SELECT ${operationColumns} FROM operations WHERE run_id = ? AND ${open} ORDER BY requested_sequence`, [scope.runId])),
+    ];
+    for (const operationId of scope.operations)
+        rows.push(...(await connection.all(`SELECT ${operationColumns} FROM operations WHERE operation_id = ?`, [operationId])));
+    rows.push(...(await candidates(connection, scope)));
+    return [
+        ...new Map(rows.map(operationRow).map((row) => [row.operationId, row])).values(),
+    ];
+}
+async function leases(connection, keys) {
+    const rows = [];
+    for (const key of new Set(keys))
+        rows.push(...(await connection.all(`SELECT ${leaseColumns} FROM leases WHERE resource_key = ?`, [key])));
+    return rows.map(leaseRow);
+}
+export async function effectTurn(connection, scope) {
+    const loaded = await operations(connection, scope);
+    const budgets = [];
+    for (const runId of new Set([scope.runId, ...loaded.map((row) => row.runId)]))
+        budgets.push(...(await connection.all(`SELECT ${budgetColumns} FROM run_budgets WHERE run_id = ?`, [runId])));
+    return storedEffects({
+        budgets: budgets.map(budgetRow),
+        operations: loaded,
+        leases: await leases(connection, [
+            ...loaded.map((row) => row.leaseKey),
+            ...scope.keys,
+        ]),
+        stops: [],
+        holding: integer(await connection.get("SELECT count(*) AS holding FROM operations WHERE status IN ('dispatch-pending', 'running', 'recovering', 'blocked')"), "holding"),
+    });
+}
+export async function pendingEffect(connection, attempt) {
+    const { intent, purpose } = attempt;
+    const row = await connection.get("SELECT p.input_digest, p.capability, p.method, p.target, p.payload, l.generation FROM operations p JOIN leases l ON l.resource_key = p.lease_key WHERE p.operation_id = ?", [intent.operationId]);
+    return {
+        ...intent,
+        purpose,
+        inputDigest: text(row, "input_digest"),
+        leaseGeneration: integer(row, "generation"),
+        capability: text(row, "capability"),
+        method: text(row, "method"),
+        target: text(row, "target"),
+        payload: text(row, "payload"),
+    };
+}
+export async function neighbours(connection, scope) {
+    const others = Object.values(scope.effects.operations).filter((operation) => operation.runId !== scope.runId);
+    const runs = {};
+    const details = {};
+    for (const runId of new Set(others.map((operation) => operation.runId))) {
+        runs[runId] = storedRun(runRow(await connection.get("SELECT run_id, status, revision, workflow, paused_from, gates, sequence FROM runs WHERE run_id = ?", [runId])));
+        details[runId] = storedDetails(detailRow(await connection.get(`SELECT ${detailColumns} FROM run_details WHERE run_id = ?`, [runId])));
+    }
+    const intents = [];
+    for (const operation of others)
+        intents.push(...(await connection.all("SELECT operation_id FROM outbox WHERE operation_id = ? AND released_sequence IS NULL", [operation.operationId])).map(() => ({
+            kind: "protected-effect",
+            runId: operation.runId,
+            operationId: operation.operationId,
+        })));
+    return { runs, details, intents };
+}

package/dist/src/broker/actor/pin-check.d.ts

@@ -0,0 +1,6 @@
+import { type SqlConnection } from "../../storage/domain-rows.ts";
+import type { AvailableManifest } from "./delivery-intents.ts";
+export declare function pinned(connection: SqlConnection, check: {
+    readonly workflow: string;
+    readonly available: AvailableManifest | undefined;
+}): Promise<boolean>;

package/dist/src/broker/actor/pin-check.js

@@ -0,0 +1,12 @@
+import { readExecutionManifest, resumePinned } from "../../execution-pin.js";
+import { text } from "../../storage/domain-rows.js";
+export async function pinned(connection, check) {
+    const { workflow, available } = check;
+    if (available === undefined)
+        return true;
+    const stored = readExecutionManifest(JSON.parse(text(await connection.get("SELECT manifest FROM definitions WHERE pin = ?", [
+        workflow,
+    ]), "manifest")));
+    const current = available(stored);
+    return (current !== undefined && resumePinned(stored, current).kind === "admitted");
+}

package/dist/src/broker/actor/runtime-parts.d.ts

@@ -0,0 +1,23 @@
+import { type ActorClass, type Database, type InstrumentationEvent, type SolidObjectsConfiguration, type SolidObjectsRuntime } from "solid-objects";
+import { type AuthorityTiming } from "./authority-settings.ts";
+import type { StrandingWatchdog } from "./authority-watchdog.ts";
+export interface FailedDelivery {
+    readonly operationId: string;
+    readonly effectId: string;
+}
+export declare function failureCounter(): (delivery: FailedDelivery) => number;
+export declare function wallClock(): string;
+export declare function runtimeSettings(parts: {
+    readonly timing: AuthorityTiming;
+    readonly options: {
+        readonly log?: (line: string) => void;
+        readonly instrumentation?: (event: InstrumentationEvent) => void;
+    };
+    readonly broker: object;
+    readonly watchdog: StrandingWatchdog;
+}): Partial<SolidObjectsConfiguration>;
+export declare function registeredRuntime(registration: {
+    readonly database: Database;
+    readonly actor: ActorClass;
+    readonly settings: Partial<SolidObjectsConfiguration>;
+}): SolidObjectsRuntime;

package/dist/src/broker/actor/runtime-parts.js

@@ -0,0 +1,34 @@
+import { createRuntime, } from "solid-objects";
+import { persistChange } from "../../storage/domain-changes.js";
+import { persistAction } from "./authority-actor.js";
+import { brokerAuthorization } from "./authority-policy.js";
+import { librarySettings } from "./authority-settings.js";
+export function failureCounter() {
+    const failed = new Map();
+    return ({ operationId, effectId }) => {
+        const effects = failed.get(operationId) ?? new Set();
+        failed.set(operationId, effects.add(effectId));
+        return effects.size;
+    };
+}
+export function wallClock() {
+    return new Date().toISOString();
+}
+export function runtimeSettings(parts) {
+    const { timing, options, broker, watchdog } = parts;
+    return {
+        ...librarySettings(timing, options.log),
+        ...brokerAuthorization(broker),
+        instrumentation: (event) => {
+            watchdog.observe(event);
+            options.instrumentation?.(event);
+        },
+    };
+}
+export function registeredRuntime(registration) {
+    const { database, actor, settings } = registration;
+    const runtime = createRuntime({ database, ...settings });
+    runtime.register(actor);
+    runtime.registerCommitAction(persistAction, (change, context) => persistChange(change, context.connection));
+    return runtime;
+}

package/dist/src/broker/actor/stop-plan.d.ts

@@ -0,0 +1,20 @@
+import type { AuthorityState } from "../../authority/reducer.ts";
+import { type StopSubject } from "../../authority/stop-sequence.ts";
+import { type DeliveryIntent } from "./delivery-intents.ts";
+import type { Stopping } from "./stop-turn.ts";
+export interface ClosedRow {
+    readonly effectId: string;
+    readonly outcome: string;
+}
+export interface StopPlanning {
+    readonly stopping: Stopping;
+    readonly runId: string;
+    readonly closed: readonly ClosedRow[];
+    readonly cancelledNow: readonly number[];
+    readonly stopLimit: number;
+    readonly stopSupported: boolean;
+    readonly opening: boolean;
+    readonly rejected: ReadonlySet<string>;
+}
+export declare function presentSubjects(state: AuthorityState, planning: Pick<StopPlanning, "stopping" | "closed">): readonly StopSubject[];
+export declare function stopIntents(state: AuthorityState, planning: StopPlanning): readonly DeliveryIntent[];

package/dist/src/broker/actor/stop-plan.js

@@ -0,0 +1,46 @@
+import { stopAttempts, } from "../../authority/stop-sequence.js";
+import { observingIntent, protectedIntent, } from "./delivery-intents.js";
+function present(state, planning) {
+    const { subject, closed } = planning;
+    const operation = state.effects.operations[subject.operationId];
+    return {
+        ...subject,
+        attempts: subject.attempts.map((attempt) => ({
+            ...attempt,
+            outcome: closed.find((row) => row.effectId === attempt.effectId)?.outcome ??
+                attempt.outcome,
+        })),
+        status: operation?.status ?? subject.status,
+        receipt: operation?.receipt ?? subject.receipt,
+        stopped: subject.stopped || state.effects.stops[subject.operationId] !== undefined,
+    };
+}
+const kinds = ["start-run", "continue-run"];
+function intentOf(state, attempt) {
+    const { runId, operationId, purpose } = attempt;
+    const kind = kinds.find((candidate) => candidate === attempt.kind);
+    if (kind === undefined)
+        return protectedIntent(state, {
+            intent: { kind: "protected-effect", runId, operationId },
+            purpose,
+        });
+    return [
+        observingIntent(state, { intent: { kind, runId, operationId }, purpose }),
+    ];
+}
+export function presentSubjects(state, planning) {
+    return planning.stopping.subjects.map((subject) => present(state, { subject, closed: planning.closed }));
+}
+export function stopIntents(state, planning) {
+    const { runId, stopping } = planning;
+    if (state.runs[runId]?.status !== "cancelled")
+        return [];
+    const subjects = presentSubjects(state, planning);
+    return stopAttempts(subjects, {
+        cancelledSequence: planning.cancelledNow[0] ?? stopping.cancelledSequence ?? 0,
+        stopLimit: planning.stopLimit,
+        stopSupported: planning.stopSupported,
+        opening: planning.opening,
+        rejected: planning.rejected,
+    }).flatMap((attempt) => intentOf(state, { ...attempt, runId }));
+}

package/dist/src/broker/actor/stop-turn.d.ts

@@ -0,0 +1,17 @@
+import type { StopSubject } from "../../authority/stop-sequence.ts";
+import { type SqlConnection } from "../../storage/domain-rows.ts";
+export interface Stopping {
+    readonly subjects: readonly StopSubject[];
+    readonly cancelledSequence: number | null;
+}
+export declare const noStopping: Stopping;
+export declare function stopping(connection: SqlConnection, runId: string): Promise<Stopping>;
+export declare function turnStopping(connection: SqlConnection, scope: {
+    readonly runId: string;
+    readonly status: string | undefined;
+    readonly kind: string;
+}): Promise<Stopping>;
+export declare function droppedRuns(connection: SqlConnection, rule: {
+    readonly limit: number;
+    readonly stopSupported: boolean;
+}): Promise<readonly string[]>;

package/dist/src/broker/actor/stop-turn.js

@@ -0,0 +1,42 @@
+import { integer, optionalInteger, optionalText, text, } from "../../storage/domain-rows.js";
+export const noStopping = { subjects: [], cancelledSequence: null };
+function attemptOf(row) {
+    return {
+        effectId: text(row, "effect_id"),
+        purpose: text(row, "purpose"),
+        outcome: optionalText(row, "outcome"),
+        stagedSequence: integer(row, "staged_sequence"),
+    };
+}
+const closed = "NOT EXISTS (SELECT 1 FROM deliveries x WHERE x.operation_id = o.operation_id AND x.outcome IS NULL)";
+const reported = "EXISTS (SELECT 1 FROM stopped_operations s WHERE s.operation_id = o.operation_id)";
+const finalStatuses = "'refused', 'completed', 'failed', 'stopped', 'cancelled'";
+export async function stopping(connection, runId) {
+    const rows = await connection.all(`SELECT o.operation_id, o.kind, d.effect_id, coalesce(a.purpose, 'dispatch') AS purpose, d.outcome, d.staged_sequence, p.status, p.receipt, ${reported} AS stopped FROM outbox o JOIN deliveries d ON d.operation_id = o.operation_id LEFT JOIN dispatch_attempts a ON a.effect_id = d.effect_id LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = ? AND ((coalesce(p.status, '') NOT IN (${finalStatuses}) AND NOT ${reported}) OR NOT ${closed}) ORDER BY d.staged_sequence, d.rowid`, [runId]);
+    const grouped = Map.groupBy(rows, (row) => text(row, "operation_id"));
+    return {
+        subjects: [...grouped].map(([operationId, attempts]) => ({
+            operationId,
+            kind: text(attempts[0], "kind"),
+            attempts: attempts.map(attemptOf),
+            status: optionalText(attempts[0], "status"),
+            receipt: optionalText(attempts[0], "receipt"),
+            stopped: integer(attempts[0], "stopped") === 1,
+        })),
+        cancelledSequence: optionalInteger(await connection.get("SELECT max(sequence) AS sequence FROM events WHERE run_id = ? AND type = 'run.cancelled'", [runId]), "sequence"),
+    };
+}
+export function turnStopping(connection, scope) {
+    const dropping = scope.status === "cancelled" || scope.kind === "cancel-run";
+    return dropping
+        ? stopping(connection, scope.runId)
+        : Promise.resolve(noStopping);
+}
+const sent = "EXISTS (SELECT 1 FROM deliveries d WHERE d.operation_id = o.operation_id AND d.outcome = 'sent')";
+const last = "(SELECT d.outcome FROM deliveries d WHERE d.operation_id = o.operation_id ORDER BY d.staged_sequence DESC, d.rowid DESC LIMIT 1)";
+const effectDue = `o.kind = 'protected-effect' AND p.status NOT IN (${finalStatuses}) AND (p.receipt IS NULL OR ?1 = 1 OR p.status IN ('recovering', 'blocked'))`;
+const runDue = `o.kind != 'protected-effect' AND NOT ${reported} AND ((${sent} AND ?1 = 1) OR (NOT ${sent} AND ${last} NOT IN ('withheld', 'blocked', 'absent')))`;
+export async function droppedRuns(connection, rule) {
+    const rows = await connection.all(`SELECT r.run_id FROM runs r WHERE r.status = 'cancelled' AND EXISTS (SELECT 1 FROM outbox o LEFT JOIN operations p ON p.operation_id = o.operation_id WHERE o.run_id = r.run_id AND ${closed} AND ((${effectDue}) OR (${runDue}))) ORDER BY r.run_id LIMIT ?2`, [rule.stopSupported ? 1 : 0, rule.limit]);
+    return rows.map((row) => text(row, "run_id"));
+}

package/dist/src/broker/actor/turn-draft.d.ts

@@ -0,0 +1,31 @@
+import type { AuthorityState, CommandResult } from "../../authority/reducer.ts";
+export type Row = Readonly<Record<string, unknown>>;
+export interface ChangeDraft {
+    readonly previous: number;
+    readonly sequence: number;
+    readonly definitions: readonly Row[];
+    readonly events: readonly Row[];
+    readonly runs: readonly Row[];
+    readonly commands: readonly Row[];
+    readonly added: readonly Row[];
+    readonly released: readonly Row[];
+    readonly closed: readonly Row[];
+    readonly gates: readonly Row[];
+    readonly details: readonly Row[];
+    readonly documents: readonly Row[];
+    readonly budgets: readonly Row[];
+    readonly operations: readonly Row[];
+    readonly leases: readonly Row[];
+    readonly evidence: readonly Row[];
+    readonly stops: readonly Row[];
+    readonly completes_adoption?: true;
+}
+export type Accepted = Extract<CommandResult, {
+    kind: "accepted";
+}>;
+export declare function unchanged(state: AuthorityState, closed: readonly Row[]): ChangeDraft;
+export declare function unchangedAt(sequence: number, closed: readonly Row[]): ChangeDraft;
+export declare function draft(result: Accepted, extra: Pick<ChangeDraft, "definitions" | "closed" | "documents"> & {
+    readonly before: AuthorityState;
+}): ChangeDraft;
+export declare function merged(first: ChangeDraft, later: readonly ChangeDraft[]): ChangeDraft;