cloving 1.4.0 → 1.5.0

package/dist/src/broker/actor/authority-session.js

@@ -1,23 +1,22 @@
-import { eligibleIntents, } from "../../authority/reducer.js";
-import { loadOperationSnapshot, loadProjections, } from "../../storage/decision-snapshots.js";
 import { loadEvents } from "../../storage/domain-rows.js";
 import { pinnedResumption } from "../authority-admission.js";
-import { currentState, intentKind, storedDefinitions, storedEvents, } from "../authority-restore.js";
+import { storedDefinitions, storedEvents, } from "../authority-restore.js";
 import { checkedRequest } from "./authority-actor.js";
 import { subscriptions, } from "./authority-changes.js";
 import { authorityQueries, guardedRead, } from "./authority-queries.js";
 import { effectQueries } from "./effect-queries.js";
 import { backupInto } from "../../storage/domain-schema.js";
 import { observeSettings, } from "./authority-connection.js";
+import { authorityHealth } from "./authority-health.js";
 import { failed, serializable, storeReceipt, invalidCommand, oversized, tooLarge, unserializable, } from "./authority-receipts.js";
+export { operationRecords } from "./projection-memory.js";
 export function authoritySession(parts) {
     const session = {
         parts,
         stopping: new AbortController(),
         delivering: false,
         failure: null,
-        condition: null,
-        describing: null,
+        health: authorityHealth(parts),
         closed: false,
         running: Promise.resolve(),
         invocations: new Set(),
@@ -33,18 +32,18 @@ export function authoritySession(parts) {
     });
     return {
         execute: (command, context) => executeCommand(session, { command, context }),
-        state: async () => currentState(await projected(session)),
+        state: async () => (await projected(session)).state(),
         events: async () => {
             await refreshed(session);
             available(session);
             return storedEvents(await parts.reader.connection(loadEvents));
         },
-        eligibleIntents: async () => eligible(await projected(session)),
-        operations: async () => operationRecords(await read(loadOperationSnapshot)),
+        eligibleIntents: async () => (await projected(session)).eligible(),
+        operations: async () => (await projected(session)).operations(),
         resumePinned: async (runId, manifest) => {
-            const snapshot = await projected(session);
-            return pinnedResumption(currentState(snapshot), {
-                definitions: storedDefinitions(snapshot),
+            const current = await projected(session);
+            return pinnedResumption(current.state(), {
+                definitions: storedDefinitions(current.snapshot),
                 runId,
                 available: manifest,
             });
@@ -56,7 +55,11 @@ export function authoritySession(parts) {
         },
         epoch: parts.epoch,
         recoveryLimit: parts.recoveryLimit,
-        queries: authorityQueries(read),
+        queries: {
+            ...authorityQueries(read),
+            sequences: async () => (await projected(session)).sequences(),
+            workflowIds: async (runIds) => (await projected(session)).workflowIds(runIds),
+        },
         effects: effectQueries(read, { stopLimit: parts.stopLimit }),
         deliver: (signal) => deliver(session, signal),
         changes: { subscribe: session.changes.add },
@@ -82,29 +85,16 @@ async function stopped(session) {
     }
 }
 async function refreshed(session) {
-    if (session.parts.unhealthy() === undefined)
+    if (session.failure !== null || session.closed)
         return;
-    await failClosed(session, "authority failure");
+    await session.health.refresh();
 }
-async function failClosed(session, failure) {
+function failClosed(session, failure) {
     session.failure ??= failure;
-    await conditionRead(session);
-    close(session).catch(() => undefined);
-}
-async function conditionRead(session) {
-    if (session.condition !== null && session.describing === null)
-        return;
-    session.describing ??= described(session).finally(() => {
-        session.describing = null;
-    });
-    await session.describing;
+    close(session).catch(ignored);
 }
-async function described(session) {
-    const detected = session.parts.unhealthy();
-    if (detected === undefined)
-        return;
-    session.condition = detected;
-    session.condition = (await session.parts.condition()) ?? detected;
+function ignored() {
+    return undefined;
 }
 async function backup(session, target) {
     await refreshed(session);
@@ -116,36 +106,43 @@ async function backup(session, target) {
         throw new Error(`backup-failed: ${String(Reflect.get(Object(error), "message"))}`, { cause: error });
     }
 }
-function available(session) {
-    const failure = session.condition ?? session.failure;
-    if (failure !== null)
-        throw new Error(`authority-unavailable: ${failure}`);
+function opened(session) {
+    if (session.failure !== null)
+        throw new Error(`authority-unavailable: ${session.failure}`);
     if (session.closed)
         throw new Error("authority-unavailable: closed");
 }
+function available(session) {
+    opened(session);
+    const condition = session.health.condition();
+    if (condition !== null)
+        throw new Error(`authority-unavailable: ${condition}`);
+}
 function unavailableReceipt(session) {
-    const failure = session.condition ?? session.failure;
-    if (failure !== null)
+    if (session.failure !== null)
         return {
             kind: "failed",
             reason: "storage-unavailable",
-            detail: `closed after ${failure}`,
+            detail: `closed after ${session.failure}`,
         };
     if (session.closed)
         return { kind: "failed", reason: "storage-unavailable", detail: "closed" };
-    return undefined;
+    const condition = session.health.condition();
+    return condition === null
+        ? undefined
+        : { kind: "failed", reason: "authority-unavailable", detail: condition };
 }
 async function projected(session) {
     await refreshed(session);
     available(session);
-    return session.parts.reader.connection(loadProjections);
+    return session.parts.memory.read(session.parts.database);
 }
 async function deliver(session, signal) {
     if (!session.parts.deliveryConfigured)
         throw new Error("delivery-unconfigured");
     if (session.delivering)
         throw new Error("delivery-already-running");
-    available(session);
+    opened(session);
     session.delivering = true;
     try {
         await delivered(session, signal);
@@ -154,21 +151,19 @@ async function deliver(session, signal) {
         session.delivering = false;
     }
     await refreshed(session);
-    const failure = session.condition ?? session.failure;
+    const failure = session.failure ?? session.health.condition();
     if (failure !== null)
         throw new Error(`authority-unavailable: ${failure}`);
 }
 async function delivered(session, signal) {
     if (!(await session.parts.handlersSettled()))
         throw new Error("delivery-handler-in-flight");
-    available(session);
-    const watch = new AbortController();
+    opened(session);
     const timer = setInterval(() => {
-        if (session.parts.unhealthy() !== undefined)
-            watch.abort();
+        refreshed(session).catch(ignored);
     }, session.parts.watchMilliseconds);
     try {
-        session.running = session.parts.run(AbortSignal.any([signal, session.stopping.signal, watch.signal]));
+        session.running = session.parts.run(AbortSignal.any([signal, session.stopping.signal]));
         await session.running;
     }
     finally {
@@ -177,8 +172,7 @@ async function delivered(session, signal) {
 }
 async function executeCommand(session, input) {
     const { command, context } = input;
-    await refreshed(session);
-    const unavailable = unavailableReceipt(session);
+    const unavailable = await refusal(session);
     if (unavailable !== undefined)
         return unavailable;
     const given = serializable({
@@ -196,15 +190,32 @@ async function executeCommand(session, input) {
     if (oversized(request))
         return tooLarge;
     try {
-        return await healthyReceipt(session, await invoked(session, request));
+        return storeReceipt(await invoked(session, request));
     }
     catch (error) {
-        const receipt = failed(error);
-        await failClosed(session, receipt.reason === "storage-failure"
-            ? "storage failure"
-            : "authority failure");
-        return receipt;
+        return failedReceipt(session, error);
+    }
+}
+async function refusal(session) {
+    try {
+        await refreshed(session);
     }
+    catch (error) {
+        return failed(error);
+    }
+    return unavailableReceipt(session);
+}
+function failedReceipt(session, error) {
+    const receipt = failed(error);
+    if (timedOut(error))
+        return receipt;
+    failClosed(session, receipt.reason === "storage-failure"
+        ? "storage failure"
+        : "authority failure");
+    return receipt;
+}
+function timedOut(error) {
+    return error instanceof Error && error.name === "SyncTimeout";
 }
 async function invoked(session, request) {
     const invocation = session.parts.execute(request);
@@ -216,36 +227,3 @@ async function invoked(session, request) {
         session.invocations.delete(invocation);
     }
 }
-async function healthyReceipt(session, value) {
-    const receipt = storeReceipt(value);
-    await refreshed(session);
-    return receipt;
-}
-export function operationRecords(snapshot) {
-    const deliveries = Map.groupBy(snapshot.deliveries, (row) => row.operationId);
-    return snapshot.outbox.map((row) => ({
-        operationId: row.operationId,
-        kind: intentKind(row.kind),
-        runId: row.runId,
-        createdSequence: row.createdSequence,
-        releasedSequence: row.releasedSequence,
-        deliveries: (deliveries.get(row.operationId) ?? []).map(({ effectId, outcome, receipt, purpose, epoch }) => ({
-            effectId,
-            outcome,
-            receipt,
-            purpose,
-            epoch,
-        })),
-    }));
-}
-function eligible(snapshot) {
-    return eligibleIntents({
-        ...currentState(snapshot),
-        effects: {
-            operations: Object.fromEntries(snapshot.pending.map((operationId) => [
-                operationId,
-                { status: "dispatch-pending" },
-            ])),
-        },
-    });
-}

package/dist/src/broker/actor/authority-settings.d.ts

@@ -1,4 +1,4 @@
-import type { SolidObjectsConfiguration } from "solid-objects";
+import type { Logger, SolidObjectsConfiguration } from "solid-objects";
 export interface AuthorityTiming {
     readonly leaseMilliseconds: number;
     readonly renewalMilliseconds: number;
@@ -30,4 +30,5 @@ export declare function effectConfiguration(options: {
         readonly stop?: unknown;
     };
 }): PlanSettings;
+export declare function authorityLogger(log: ((line: string) => void) | undefined): Logger;
 export declare function librarySettings(timing: AuthorityTiming, log: ((line: string) => void) | undefined): Partial<SolidObjectsConfiguration>;

package/dist/src/broker/actor/authority-settings.js

@@ -47,6 +47,9 @@ export function effectConfiguration(options) {
 function standardError(line) {
     process.stderr.write(line);
 }
+export function authorityLogger(log) {
+    return lineLogger(log ?? standardError);
+}
 export function librarySettings(timing, log) {
     return {
         tableNamePrefix: "solid_",
@@ -54,7 +57,7 @@ export function librarySettings(timing, log) {
         maxActivationDurationMilliseconds: 50,
         maxPayloadBytes: 900_000 + commandLimits.stdoutBytes * 12,
         maxResultBytes: 900_000 + commandLimits.stdoutBytes * 12,
-        logger: lineLogger(log ?? standardError),
+        logger: authorityLogger(log),
         leaseDurationMilliseconds: timing.leaseMilliseconds,
         leaseRenewalIntervalMilliseconds: timing.renewalMilliseconds,
         processHeartbeatIntervalMilliseconds: timing.heartbeatMilliseconds,

package/dist/src/broker/actor/authority-upgrades.d.ts

@@ -2,7 +2,7 @@ import { upgradeCommandWaits, upgradeCommandWaitChecks } from "../command-wait-r
 import { loadDecided, loadEffected, loadWaiting, loadCurrent } from "../../storage/decision-snapshots.ts";
 import { loadSnapshot } from "../../storage/domain-rows.ts";
 import { upgradeDecisions } from "../decision-upgrade.ts";
-import { upgradeEffects, upgradeEventIndexes, upgradeSpendingIndexes } from "../effect-upgrade.ts";
+import { upgradeEffects, upgradeEventIndexes, upgradeRunIndexes, upgradeSpendingIndexes } from "../effect-upgrade.ts";
 declare const upgrades: readonly [{
     readonly from: "adopted";
     readonly load: typeof loadSnapshot;
@@ -27,6 +27,10 @@ declare const upgrades: readonly [{
     readonly from: "indexed";
     readonly load: typeof loadCurrent;
     readonly apply: typeof upgradeEventIndexes;
+}, {
+    readonly from: "event-indexed";
+    readonly load: typeof loadCurrent;
+    readonly apply: typeof upgradeRunIndexes;
 }];
 export type UpgradeStep = (typeof upgrades)[number];
 export declare function upgradeSteps(kind: string): readonly UpgradeStep[];

package/dist/src/broker/actor/authority-upgrades.js

@@ -2,7 +2,7 @@ import { upgradeCommandWaits, upgradeCommandWaitChecks, } from "../command-wait-
 import { loadDecided, loadEffected, loadWaiting, loadCurrent, } from "../../storage/decision-snapshots.js";
 import { loadSnapshot } from "../../storage/domain-rows.js";
 import { upgradeDecisions } from "../decision-upgrade.js";
-import { upgradeEffects, upgradeEventIndexes, upgradeSpendingIndexes, } from "../effect-upgrade.js";
+import { upgradeEffects, upgradeEventIndexes, upgradeRunIndexes, upgradeSpendingIndexes, } from "../effect-upgrade.js";
 const upgrades = [
     { from: "adopted", load: loadSnapshot, apply: upgradeDecisions },
     { from: "decided", load: loadDecided, apply: upgradeEffects },
@@ -10,6 +10,7 @@ const upgrades = [
     { from: "waiting", load: loadWaiting, apply: upgradeCommandWaitChecks },
     { from: "checked", load: loadCurrent, apply: upgradeSpendingIndexes },
     { from: "indexed", load: loadCurrent, apply: upgradeEventIndexes },
+    { from: "event-indexed", load: loadCurrent, apply: upgradeRunIndexes },
 ];
 export function upgradeSteps(kind) {
     const first = upgrades.findIndex((step) => step.from === kind);

package/dist/src/broker/actor/authority-watchdog.d.ts

@@ -3,10 +3,14 @@ export interface WatchedEvent {
     readonly messageId: string | null;
     readonly attributes: Readonly<Record<string, unknown>>;
 }
+export interface WatchReport {
+    readonly kind: "stranded" | "dead-lettered";
+    readonly messageId: string;
+}
+export type MessageRuns = (messageIds: readonly string[]) => Promise<ReadonlyMap<string, string>>;
 export interface StrandingWatchdog {
     readonly observe: (event: WatchedEvent) => void;
-    readonly stranded: () => string | undefined;
-    readonly deadLettered: () => string | undefined;
+    readonly report: (runs: MessageRuns) => Promise<WatchReport | undefined>;
 }
 export declare function strandingWatchdog(bound: {
     readonly boundMilliseconds: number;

package/dist/src/broker/actor/authority-watchdog.js

@@ -1,3 +1,4 @@
+import { terminal } from "../../authority/transitions.js";
 const callbacks = ["deliveryCompleted", "deliveryFailed"];
 const finished = [
     "solid_objects.message.completed",
@@ -7,33 +8,54 @@ const finished = [
 export function strandingWatchdog(bound) {
     const clock = bound.clock ?? monotonic;
     const lost = new Map();
-    const dead = [];
-    let reported;
+    const dead = new Set();
     return {
         observe: (event) => {
             const { messageId } = event;
             if (messageId === null)
                 return;
             if (deadCallback(event))
-                dead.push(messageId);
+                dead.add(messageId);
             if (event.name === "solid_objects.activation.lost" &&
                 !lost.has(messageId))
                 lost.set(messageId, clock());
             if (finished.includes(event.name))
                 lost.delete(messageId);
         },
-        stranded: () => {
-            reported ??= overdue(lost, clock() - bound.boundMilliseconds);
-            return reported;
+        report: async (runs) => {
+            const overdue = overdueMessages(lost, clock() - bound.boundMilliseconds);
+            const suspects = [...dead, ...overdue];
+            if (suspects.length === 0)
+                return undefined;
+            const statuses = await runs(suspects);
+            const released = suspects.filter((messageId) => settled(statuses.get(messageId)));
+            for (const messageId of released) {
+                lost.delete(messageId);
+                dead.delete(messageId);
+            }
+            return firstReport({ dead: [...dead], stranded: overdue, lost });
         },
-        deadLettered: () => dead[0],
     };
 }
 function monotonic() {
     return performance.now();
 }
-function overdue(lost, limit) {
-    return [...lost].find(([, at]) => at < limit)?.[0];
+function overdueMessages(lost, limit) {
+    return [...lost]
+        .filter(([, at]) => at < limit)
+        .map(([messageId]) => messageId);
+}
+function settled(status) {
+    return terminal.some((candidate) => candidate === status);
+}
+function firstReport(found) {
+    const deadLetter = found.dead[0];
+    if (deadLetter !== undefined)
+        return { kind: "dead-lettered", messageId: deadLetter };
+    const stranded = found.stranded.find((messageId) => found.lost.has(messageId));
+    return stranded === undefined
+        ? undefined
+        : { kind: "stranded", messageId: stranded };
 }
 function deadCallback(event) {
     return (event.name === "solid_objects.dead_letter.created" &&

package/dist/src/broker/actor/authority.js

@@ -76,6 +76,7 @@ async function begun(opening) {
             reason: "unresolved-delivery",
             detail: condition,
         };
+    await parts.memory.load(owned.database);
     return integer(opened, "epoch");
 }
 async function start(opening) {
@@ -99,6 +100,7 @@ async function start(opening) {
         authority: authoritySession({
             database,
             reader: parts.reader,
+            memory: parts.memory,
             execute: (request) => authority.execute(json(request)),
             run: (signal) => parts.runtime.run(signal),
             stop: parts.stop,
@@ -106,8 +108,8 @@ async function start(opening) {
             deliveryConfigured: options.delivery !== undefined,
             epoch,
             ...parts.limits,
-            unhealthy: parts.unhealthy,
             condition: parts.condition,
+            logger: parts.logger,
             watchMilliseconds: parts.watchMilliseconds,
             subscribe: (listener) => parts.subscribe(actorId, listener),
         }),

package/dist/src/broker/actor/projection-memory.d.ts

@@ -0,0 +1,43 @@
+import type { Database, Logger } from "solid-objects";
+import { type EffectIntent } from "../../authority/reducer.ts";
+import { type ProjectionSnapshot } from "../../storage/decision-snapshots.ts";
+import type { DomainChange } from "../../storage/domain-changes.ts";
+import { type StoredState } from "../authority-restore.ts";
+import type { Sequences } from "./authority-queries.ts";
+export interface OperationRecord {
+    readonly operationId: string;
+    readonly kind: EffectIntent["kind"];
+    readonly runId: string;
+    readonly createdSequence: number;
+    readonly releasedSequence: number | null;
+    readonly deliveries: readonly {
+        readonly effectId: string;
+        readonly outcome: string | null;
+        readonly receipt: string | null;
+        readonly purpose: string;
+        readonly epoch: number | null;
+    }[];
+}
+export interface Projection {
+    readonly snapshot: ProjectionSnapshot;
+    readonly state: () => StoredState;
+    readonly eligible: () => readonly EffectIntent[];
+    readonly operations: () => readonly OperationRecord[];
+    readonly sequences: () => Sequences;
+    readonly workflowIds: (runIds?: readonly string[]) => Readonly<Record<string, string>>;
+}
+type Source = Pick<Database, "connection">;
+export interface ProjectionMemory {
+    readonly record: (change: RecordedChange) => void;
+    readonly load: (database: Source) => Promise<void>;
+    readonly read: (database: Source) => Promise<Projection>;
+}
+export interface RecordedChange {
+    readonly messageId: string;
+    readonly change: DomainChange;
+}
+export declare function projectionMemory(options: {
+    readonly logger: Logger;
+}): ProjectionMemory;
+export declare function operationRecords(snapshot: Pick<ProjectionSnapshot, "outbox" | "deliveries">): readonly OperationRecord[];
+export {};