cloving 0.3.20 → 1.1.0

package/dist/src/broker/actor/authority-session.d.ts

@@ -0,0 +1,67 @@
+import type { Database, JsonObject } from "solid-objects";
+import { type AuthorityEvent, type EffectIntent } from "../../authority/reducer.ts";
+import { type ProjectionSnapshot } from "../../storage/decision-snapshots.ts";
+import type { AdmissionResult, ExecutionManifest } from "../../workflow-admission.ts";
+import { type StoredState } from "../authority-restore.ts";
+import { type SequenceChanges, type SequenceListener } from "./authority-changes.ts";
+import { type AuthorityQueries } from "./authority-queries.ts";
+import { type EffectQueries } from "./effect-queries.ts";
+import { type ConnectionSettings } from "./authority-connection.ts";
+import { type StoreReceipt } from "./authority-receipts.ts";
+export type { StoreReceipt } from "./authority-receipts.ts";
+export type { StoredState } from "../authority-restore.ts";
+export interface CommandContext {
+    readonly recordedAt: string;
+    readonly manifest?: ExecutionManifest;
+    readonly epoch?: number;
+    readonly documents?: readonly string[];
+}
+export interface OperationRecord {
+    readonly operationId: string;
+    readonly kind: EffectIntent["kind"];
+    readonly runId: string;
+    readonly createdSequence: number;
+    readonly releasedSequence: number | null;
+    readonly deliveries: readonly {
+        readonly effectId: string;
+        readonly outcome: string | null;
+        readonly receipt: string | null;
+        readonly purpose: string;
+        readonly epoch: number | null;
+    }[];
+}
+export interface Authority {
+    readonly execute: (command: unknown, context: CommandContext) => Promise<StoreReceipt>;
+    readonly state: () => Promise<StoredState>;
+    readonly events: () => Promise<readonly AuthorityEvent[]>;
+    readonly eligibleIntents: () => Promise<readonly EffectIntent[]>;
+    readonly operations: () => Promise<readonly OperationRecord[]>;
+    readonly resumePinned: (runId: string, available: ExecutionManifest) => Promise<AdmissionResult>;
+    readonly settings: () => Promise<ConnectionSettings>;
+    readonly epoch: number;
+    readonly recoveryLimit: number;
+    readonly queries: AuthorityQueries;
+    readonly effects: EffectQueries;
+    readonly deliver: (signal: AbortSignal) => Promise<void>;
+    readonly changes: SequenceChanges;
+    readonly backup: (target: string) => Promise<void>;
+    readonly close: () => Promise<void>;
+}
+export interface SessionParts {
+    readonly database: Database;
+    readonly reader: Database;
+    readonly execute: (request: JsonObject) => Promise<unknown>;
+    readonly run: (signal: AbortSignal) => Promise<void>;
+    readonly stop: () => Promise<void>;
+    readonly handlersSettled: () => Promise<boolean>;
+    readonly deliveryConfigured: boolean;
+    readonly epoch: number;
+    readonly stopLimit: number;
+    readonly recoveryLimit: number;
+    readonly unhealthy: () => string | undefined;
+    readonly condition: () => Promise<string | undefined>;
+    readonly watchMilliseconds: number;
+    readonly subscribe: (listener: SequenceListener) => () => void;
+}
+export declare function authoritySession(parts: SessionParts): Authority;
+export declare function operationRecords(snapshot: Pick<ProjectionSnapshot, "outbox" | "deliveries">): readonly OperationRecord[];

package/dist/src/broker/actor/authority-session.js

@@ -0,0 +1,251 @@
+import { eligibleIntents, } from "../../authority/reducer.js";
+import { loadProjections, } from "../../storage/decision-snapshots.js";
+import { loadEvents } from "../../storage/domain-rows.js";
+import { pinnedResumption } from "../authority-admission.js";
+import { currentState, intentKind, storedDefinitions, storedEvents, } from "../authority-restore.js";
+import { checkedRequest } from "./authority-actor.js";
+import { subscriptions, } from "./authority-changes.js";
+import { authorityQueries, guardedRead, } from "./authority-queries.js";
+import { effectQueries } from "./effect-queries.js";
+import { backupInto } from "../../storage/domain-schema.js";
+import { observeSettings, } from "./authority-connection.js";
+import { failed, serializable, storeReceipt, invalidCommand, oversized, tooLarge, unserializable, } from "./authority-receipts.js";
+export function authoritySession(parts) {
+    const session = {
+        parts,
+        stopping: new AbortController(),
+        delivering: false,
+        failure: null,
+        condition: null,
+        describing: null,
+        closed: false,
+        running: Promise.resolve(),
+        invocations: new Set(),
+        changes: subscriptions(parts.subscribe),
+        closing: null,
+    };
+    const read = guardedRead({
+        reader: parts.reader,
+        ready: async () => {
+            await refreshed(session);
+            available(session);
+        },
+    });
+    return {
+        execute: (command, context) => executeCommand(session, { command, context }),
+        state: async () => currentState(await projected(session)),
+        events: async () => {
+            await refreshed(session);
+            available(session);
+            return storedEvents(await parts.reader.connection(loadEvents));
+        },
+        eligibleIntents: async () => eligible(await projected(session)),
+        operations: async () => operationRecords(await projected(session)),
+        resumePinned: async (runId, manifest) => {
+            const snapshot = await projected(session);
+            return pinnedResumption(currentState(snapshot), {
+                definitions: storedDefinitions(snapshot),
+                runId,
+                available: manifest,
+            });
+        },
+        settings: async () => {
+            await refreshed(session);
+            available(session);
+            return observeSettings(parts.database);
+        },
+        epoch: parts.epoch,
+        recoveryLimit: parts.recoveryLimit,
+        queries: authorityQueries(read),
+        effects: effectQueries(read, { stopLimit: parts.stopLimit }),
+        deliver: (signal) => deliver(session, signal),
+        changes: { subscribe: session.changes.add },
+        backup: (target) => backup(session, target),
+        close: () => close(session),
+    };
+}
+function close(session) {
+    session.closed = true;
+    session.closing ??= stopped(session);
+    return session.closing;
+}
+async function stopped(session) {
+    try {
+        session.stopping.abort();
+        session.changes.close();
+        await Promise.allSettled([session.running, ...session.invocations]);
+        await session.parts.stop();
+    }
+    catch (error) {
+        session.closing = null;
+        throw error;
+    }
+}
+async function refreshed(session) {
+    if (session.parts.unhealthy() === undefined)
+        return;
+    await failClosed(session, "authority failure");
+}
+async function failClosed(session, failure) {
+    session.failure ??= failure;
+    await conditionRead(session);
+    close(session).catch(() => undefined);
+}
+async function conditionRead(session) {
+    if (session.condition !== null && session.describing === null)
+        return;
+    session.describing ??= described(session).finally(() => {
+        session.describing = null;
+    });
+    await session.describing;
+}
+async function described(session) {
+    const detected = session.parts.unhealthy();
+    if (detected === undefined)
+        return;
+    session.condition = detected;
+    session.condition = (await session.parts.condition()) ?? detected;
+}
+async function backup(session, target) {
+    await refreshed(session);
+    available(session);
+    try {
+        await session.parts.database.connection((connection) => backupInto(connection, target));
+    }
+    catch (error) {
+        throw new Error(`backup-failed: ${String(Reflect.get(Object(error), "message"))}`, { cause: error });
+    }
+}
+function available(session) {
+    const failure = session.condition ?? session.failure;
+    if (failure !== null)
+        throw new Error(`authority-unavailable: ${failure}`);
+    if (session.closed)
+        throw new Error("authority-unavailable: closed");
+}
+function unavailableReceipt(session) {
+    const failure = session.condition ?? session.failure;
+    if (failure !== null)
+        return {
+            kind: "failed",
+            reason: "storage-unavailable",
+            detail: `closed after ${failure}`,
+        };
+    if (session.closed)
+        return { kind: "failed", reason: "storage-unavailable", detail: "closed" };
+    return undefined;
+}
+async function projected(session) {
+    await refreshed(session);
+    available(session);
+    return session.parts.reader.connection(loadProjections);
+}
+async function deliver(session, signal) {
+    if (!session.parts.deliveryConfigured)
+        throw new Error("delivery-unconfigured");
+    if (session.delivering)
+        throw new Error("delivery-already-running");
+    available(session);
+    session.delivering = true;
+    try {
+        await delivered(session, signal);
+    }
+    finally {
+        session.delivering = false;
+    }
+    await refreshed(session);
+    const failure = session.condition ?? session.failure;
+    if (failure !== null)
+        throw new Error(`authority-unavailable: ${failure}`);
+}
+async function delivered(session, signal) {
+    if (!(await session.parts.handlersSettled()))
+        throw new Error("delivery-handler-in-flight");
+    available(session);
+    const watch = new AbortController();
+    const timer = setInterval(() => {
+        if (session.parts.unhealthy() !== undefined)
+            watch.abort();
+    }, session.parts.watchMilliseconds);
+    try {
+        session.running = session.parts.run(AbortSignal.any([signal, session.stopping.signal, watch.signal]));
+        await session.running;
+    }
+    finally {
+        clearInterval(timer);
+    }
+}
+async function executeCommand(session, input) {
+    const { command, context } = input;
+    await refreshed(session);
+    const unavailable = unavailableReceipt(session);
+    if (unavailable !== undefined)
+        return unavailable;
+    const given = serializable({
+        command,
+        recordedAt: context.recordedAt,
+        manifest: context.manifest ?? null,
+        epoch: context.epoch ?? null,
+        documents: context.documents ?? [],
+    });
+    if (given === undefined)
+        return unserializable;
+    const request = checkedRequest(given);
+    if (request === undefined)
+        return invalidCommand;
+    if (oversized(request))
+        return tooLarge;
+    try {
+        return await healthyReceipt(session, await invoked(session, request));
+    }
+    catch (error) {
+        const receipt = failed(error);
+        await failClosed(session, receipt.reason === "storage-failure"
+            ? "storage failure"
+            : "authority failure");
+        return receipt;
+    }
+}
+async function invoked(session, request) {
+    const invocation = session.parts.execute(request);
+    session.invocations.add(invocation);
+    try {
+        return await invocation;
+    }
+    finally {
+        session.invocations.delete(invocation);
+    }
+}
+async function healthyReceipt(session, value) {
+    const receipt = storeReceipt(value);
+    await refreshed(session);
+    return receipt;
+}
+export function operationRecords(snapshot) {
+    const deliveries = Map.groupBy(snapshot.deliveries, (row) => row.operationId);
+    return snapshot.outbox.map((row) => ({
+        operationId: row.operationId,
+        kind: intentKind(row.kind),
+        runId: row.runId,
+        createdSequence: row.createdSequence,
+        releasedSequence: row.releasedSequence,
+        deliveries: (deliveries.get(row.operationId) ?? []).map(({ effectId, outcome, receipt, purpose, epoch }) => ({
+            effectId,
+            outcome,
+            receipt,
+            purpose,
+            epoch,
+        })),
+    }));
+}
+function eligible(snapshot) {
+    return eligibleIntents({
+        ...currentState(snapshot),
+        effects: {
+            operations: Object.fromEntries(snapshot.pending.map((operationId) => [
+                operationId,
+                { status: "dispatch-pending" },
+            ])),
+        },
+    });
+}

package/dist/src/broker/actor/authority-settings.d.ts

@@ -0,0 +1,33 @@
+import type { SolidObjectsConfiguration } from "solid-objects";
+export interface AuthorityTiming {
+    readonly leaseMilliseconds: number;
+    readonly renewalMilliseconds: number;
+    readonly heartbeatMilliseconds: number;
+    readonly aliveThresholdMilliseconds: number;
+    readonly cleanupIntervalMilliseconds: number;
+    readonly callTimeoutMilliseconds: number;
+    readonly messageRetentionMilliseconds: number;
+    readonly retentionIntervalMilliseconds: number;
+    readonly shutdownMilliseconds: number;
+}
+export declare const authorityTiming: AuthorityTiming;
+export interface EffectSettings {
+    readonly capacity: number;
+    readonly leaseMilliseconds: number;
+    readonly adapterMilliseconds: number;
+    readonly resultBytes: number;
+    readonly deliveryAttempts: number;
+    readonly stopLimit: number;
+    readonly recoveryLimit: number;
+}
+export interface PlanSettings extends EffectSettings {
+    readonly stopSupported: boolean;
+}
+export declare const effectSettings: PlanSettings;
+export declare function effectConfiguration(options: {
+    readonly effects?: Partial<EffectSettings>;
+    readonly delivery?: {
+        readonly stop?: unknown;
+    };
+}): PlanSettings;
+export declare function librarySettings(timing: AuthorityTiming, log: ((line: string) => void) | undefined): Partial<SolidObjectsConfiguration>;

package/dist/src/broker/actor/authority-settings.js

@@ -0,0 +1,63 @@
+import { recoveryBound } from "../../authority/commands.js";
+import { lineLogger } from "./authority-logger.js";
+export const authorityTiming = {
+    leaseMilliseconds: 2_000,
+    renewalMilliseconds: 500,
+    heartbeatMilliseconds: 1_000,
+    aliveThresholdMilliseconds: 3_000,
+    cleanupIntervalMilliseconds: 1_000,
+    callTimeoutMilliseconds: 10_000,
+    messageRetentionMilliseconds: 30 * 86_400_000,
+    retentionIntervalMilliseconds: 3_600_000,
+    shutdownMilliseconds: 10_000,
+};
+export const effectSettings = {
+    capacity: 4,
+    leaseMilliseconds: 600_000,
+    adapterMilliseconds: 30_000,
+    resultBytes: 262_144,
+    deliveryAttempts: 2,
+    stopLimit: 3,
+    recoveryLimit: 3,
+    stopSupported: false,
+};
+const maximumCapacity = 8;
+export function effectConfiguration(options) {
+    const settings = {
+        ...effectSettings,
+        ...options.effects,
+        stopSupported: options.delivery?.stop !== undefined,
+    };
+    const positive = [
+        settings.capacity,
+        settings.leaseMilliseconds,
+        settings.adapterMilliseconds,
+        settings.resultBytes,
+        settings.deliveryAttempts,
+        settings.stopLimit,
+        settings.recoveryLimit,
+    ].every((value) => Number.isInteger(value) && value >= 1);
+    const bounded = settings.capacity <= maximumCapacity &&
+        settings.recoveryLimit <= recoveryBound;
+    if (!positive || !bounded)
+        throw new Error("invalid-effect-settings");
+    return settings;
+}
+function standardError(line) {
+    process.stderr.write(line);
+}
+export function librarySettings(timing, log) {
+    return {
+        tableNamePrefix: "solid_",
+        maxAttempts: 1,
+        logger: lineLogger(log ?? standardError),
+        leaseDurationMilliseconds: timing.leaseMilliseconds,
+        leaseRenewalIntervalMilliseconds: timing.renewalMilliseconds,
+        processHeartbeatIntervalMilliseconds: timing.heartbeatMilliseconds,
+        processAliveThresholdMilliseconds: timing.aliveThresholdMilliseconds,
+        deadProcessCleanupIntervalMilliseconds: timing.cleanupIntervalMilliseconds,
+        messageRetentionMilliseconds: timing.messageRetentionMilliseconds,
+        retentionIntervalMilliseconds: timing.retentionIntervalMilliseconds,
+        shutdownTimeoutMilliseconds: timing.shutdownMilliseconds,
+    };
+}

package/dist/src/broker/actor/authority-upgrades.d.ts

@@ -0,0 +1,16 @@
+import { loadDecided } from "../../storage/decision-snapshots.ts";
+import { loadSnapshot } from "../../storage/domain-rows.ts";
+import { upgradeDecisions } from "../decision-upgrade.ts";
+import { upgradeEffects } from "../effect-upgrade.ts";
+declare const upgrades: readonly [{
+    readonly from: "adopted";
+    readonly load: typeof loadSnapshot;
+    readonly apply: typeof upgradeDecisions;
+}, {
+    readonly from: "decided";
+    readonly load: typeof loadDecided;
+    readonly apply: typeof upgradeEffects;
+}];
+export type UpgradeStep = (typeof upgrades)[number];
+export declare function upgradeSteps(kind: string): readonly UpgradeStep[];
+export {};

package/dist/src/broker/actor/authority-upgrades.js

@@ -0,0 +1,12 @@
+import { loadDecided } from "../../storage/decision-snapshots.js";
+import { loadSnapshot } from "../../storage/domain-rows.js";
+import { upgradeDecisions } from "../decision-upgrade.js";
+import { upgradeEffects } from "../effect-upgrade.js";
+const upgrades = [
+    { from: "adopted", load: loadSnapshot, apply: upgradeDecisions },
+    { from: "decided", load: loadDecided, apply: upgradeEffects },
+];
+export function upgradeSteps(kind) {
+    const first = upgrades.findIndex((step) => step.from === kind);
+    return first === -1 ? [] : upgrades.slice(first);
+}

package/dist/src/broker/actor/authority-watchdog.d.ts

@@ -0,0 +1,14 @@
+export interface WatchedEvent {
+    readonly name: string;
+    readonly messageId: string | null;
+    readonly attributes: Readonly<Record<string, unknown>>;
+}
+export interface StrandingWatchdog {
+    readonly observe: (event: WatchedEvent) => void;
+    readonly stranded: () => string | undefined;
+    readonly deadLettered: () => string | undefined;
+}
+export declare function strandingWatchdog(bound: {
+    readonly boundMilliseconds: number;
+    readonly clock?: () => number;
+}): StrandingWatchdog;

package/dist/src/broker/actor/authority-watchdog.js

@@ -0,0 +1,41 @@
+const callbacks = ["deliveryCompleted", "deliveryFailed"];
+const finished = [
+    "solid_objects.message.completed",
+    "solid_objects.message.failed",
+    "solid_objects.message.rejected",
+];
+export function strandingWatchdog(bound) {
+    const clock = bound.clock ?? monotonic;
+    const lost = new Map();
+    const dead = [];
+    let reported;
+    return {
+        observe: (event) => {
+            const { messageId } = event;
+            if (messageId === null)
+                return;
+            if (deadCallback(event))
+                dead.push(messageId);
+            if (event.name === "solid_objects.activation.lost" &&
+                !lost.has(messageId))
+                lost.set(messageId, clock());
+            if (finished.includes(event.name))
+                lost.delete(messageId);
+        },
+        stranded: () => {
+            reported ??= overdue(lost, clock() - bound.boundMilliseconds);
+            return reported;
+        },
+        deadLettered: () => dead[0],
+    };
+}
+function monotonic() {
+    return performance.now();
+}
+function overdue(lost, limit) {
+    return [...lost].find(([, at]) => at < limit)?.[0];
+}
+function deadCallback(event) {
+    return (event.name === "solid_objects.dead_letter.created" &&
+        callbacks.includes(String(event.attributes["operation"])));
+}

package/dist/src/broker/actor/authority.d.ts

@@ -0,0 +1,14 @@
+import { type AuthorityOptions, type Refusal } from "./authority-runtime.ts";
+import { type Authority } from "./authority-session.ts";
+export type { AdapterOutcome, Delivery, DeliveryIntent, Reconciliation, } from "./authority-delivery.ts";
+export type { AuthorityOptions, AuthorityTiming, Refusal, } from "./authority-runtime.ts";
+export type { EffectSettings } from "./authority-settings.ts";
+export type { OperationView, StoredDisposition } from "./effect-queries.ts";
+export { authorityTiming } from "./authority-runtime.ts";
+export type { AuthorityQueries, CommandRecord, EventPage, PublishedEvent, } from "./authority-queries.ts";
+export type { Authority, CommandContext, OperationRecord, StoreReceipt, StoredState, } from "./authority-session.ts";
+export type AuthorityOpening = {
+    readonly kind: "opened";
+    readonly authority: Authority;
+} | Refusal;
+export declare function openAuthority(directory: string, options?: AuthorityOptions): Promise<AuthorityOpening>;

package/dist/src/broker/actor/authority.js

@@ -0,0 +1,115 @@
+import { loadCurrent } from "../../storage/decision-snapshots.js";
+import { integer } from "../../storage/domain-rows.js";
+import { authorityIdentity, createSchema, } from "../../storage/domain-schema.js";
+import { restore } from "../authority-restore.js";
+import { ownedRuntime, recoverDeliveries, refused, } from "./authority-runtime.js";
+import { json } from "./authority-receipts.js";
+import { upgradeSteps } from "./authority-upgrades.js";
+import { authoritySession } from "./authority-session.js";
+export { authorityTiming } from "./authority-runtime.js";
+export async function openAuthority(directory, options = {}) {
+    const owned = await ownedRuntime(directory, options);
+    if (owned.kind === "refused")
+        return owned;
+    let actorId;
+    try {
+        actorId = await validated(owned);
+    }
+    catch (error) {
+        actorId = refused("storage-failure", error);
+    }
+    if (typeof actorId !== "string") {
+        await owned.parts.stop();
+        return actorId;
+    }
+    return start({ owned, actorId, options });
+}
+async function validated(owned) {
+    const { database, schema } = owned;
+    if (schema.kind === "new")
+        await database.transaction(createSchema);
+    if (schema.kind === "baseline")
+        return conversionRequired("schema version 1");
+    const upgrade = await upgraded(owned);
+    if (upgrade !== undefined)
+        return upgrade;
+    const restored = restore(await database.connection(loadCurrent));
+    if (restored.kind === "refused")
+        return restored;
+    const identity = await database.connection(authorityIdentity);
+    if (identity.pendingAdoption !== null)
+        return conversionRequired("adoption pending");
+    return identity.actorId;
+}
+async function upgraded(owned) {
+    const { database, schema } = owned;
+    for (const step of upgradeSteps(schema.kind)) {
+        const restored = restore(await database.connection(step.load));
+        if (restored.kind === "refused")
+            return restored;
+        const identity = await database.connection(authorityIdentity);
+        if (identity.pendingAdoption !== null)
+            return conversionRequired("adoption pending");
+        await step.apply(database, restored.state);
+    }
+    return undefined;
+}
+function conversionRequired(detail) {
+    return { kind: "refused", reason: "conversion-required", detail };
+}
+async function begun(opening) {
+    const { owned, actorId } = opening;
+    const { parts } = owned;
+    const authority = parts.invoker(actorId);
+    await parts.runtime.install();
+    const stored = await parts.storedActorRefusal(actorId);
+    if (stored !== undefined)
+        return stored;
+    await parts.releaseEarlierOwners();
+    await parts.retryDeliveryCallbacks();
+    const opened = await authority.beginEpoch();
+    await recoverDeliveries(authority);
+    const condition = await parts.condition();
+    if (condition !== undefined)
+        return {
+            kind: "refused",
+            reason: "unresolved-delivery",
+            detail: condition,
+        };
+    return integer(opened, "epoch");
+}
+async function start(opening) {
+    const { owned, actorId, options } = opening;
+    const { database, parts } = owned;
+    let epoch;
+    try {
+        epoch = await begun({ owned, actorId });
+    }
+    catch (error) {
+        epoch = refused("authority-unavailable", error);
+    }
+    if (typeof epoch !== "number") {
+        await parts.stop();
+        return epoch;
+    }
+    parts.setEpoch(epoch);
+    const authority = parts.invoker(actorId);
+    return {
+        kind: "opened",
+        authority: authoritySession({
+            database,
+            reader: parts.reader,
+            execute: (request) => authority.execute(json(request)),
+            run: (signal) => parts.runtime.run(signal),
+            stop: parts.stop,
+            handlersSettled: parts.handlersSettled,
+            deliveryConfigured: options.delivery !== undefined,
+            epoch,
+            ...parts.limits,
+            unhealthy: parts.unhealthy,
+            condition: parts.condition,
+            watchMilliseconds: parts.watchMilliseconds,
+            subscribe: (listener) => parts.subscribe(actorId, listener),
+        }),
+    };
+}