@evelinka-ai/core 0.1.6 → 0.3.0

package/dist/lib/coordination/reconciler.d.ts

@@ -0,0 +1,156 @@
+import { type LeaseRow, type ReapResult } from "./head-lease.js";
+import type { DeferOpts, QuarantineOpts } from "./work-queue.js";
+export declare const UNCONFIRMED_GRACE_MS: number;
+export declare const CLOSED_TOMBSTONE_TTL_MS: number;
+export declare const CLAIM_ROUNDS_PER_PASS = 5;
+export declare const CLAIM_ROWS_PER_PASS = 30;
+export declare const TRANSIENT_BACKOFF_MS = 60000;
+export type TargetState = {
+    kind: "open";
+    head: string | null;
+} | {
+    kind: "closed";
+} | {
+    kind: "missing";
+} | {
+    kind: "invisible";
+} | {
+    kind: "unavailable";
+};
+export type DispatchOutcome = "dispatched" | "deduped" | "queued" | "refused";
+export interface ReconcilerSettings {
+    cadenceMs: number;
+    dispatchCap: number;
+    restartBudget: number;
+    quarantineMs: number;
+    settleMs: number;
+}
+export interface ReconcilerQueue<K, Item extends QueueItem<K>, Reason extends string> {
+    claimDue(limit: number): Promise<Item[]>;
+    list(): Promise<Item[] | null>;
+    read(key: K): Promise<Item | null | undefined>;
+    upsert(item: K & {
+        targetHead: string;
+        baseHead?: string | null;
+        reason: Reason;
+        countAttempt?: boolean;
+        lastError?: string | null;
+    }): Promise<boolean>;
+    complete(key: K, head: string | null): Promise<boolean>;
+    defer(key: K, opts: DeferOpts<Reason>): Promise<string | null>;
+    quarantine(key: K, opts: QuarantineOpts): Promise<string | null>;
+}
+export type QueueItem<K> = K & {
+    targetHead: string;
+    baseHead: string | null;
+    reason: string;
+    attempts: number;
+};
+export interface LostHead<K> {
+    key: K;
+    head: string;
+    name: string;
+    lease: LeaseRow;
+    genuine: boolean;
+    counted: boolean;
+    nextAttemptAt: string;
+    readAttempts: () => Promise<number | null>;
+}
+export interface UnconfirmedHead<K> {
+    key: K | null;
+    head: string;
+    base: string | null;
+    at: string;
+}
+export interface ReconcilerPolicy<K, Item extends QueueItem<K>, Reason extends string, D, U extends UnconfirmedHead<K> = UnconfirmedHead<K>> {
+    now(): number;
+    settings(): ReconcilerSettings;
+    queue: ReconcilerQueue<K, Item, Reason>;
+    leases: {
+        reap(): Promise<LeaseRow[] | ReapResult>;
+        list(): Promise<LeaseRow[] | null>;
+    };
+    keys: {
+        fromLease(itemKey: string): K | null;
+        toLease(key: K, head: string): string;
+        id(key: K): string;
+        of(item: Item): K;
+    };
+    reasons: {
+        restart: Reason;
+        refused: Reason;
+        reconcile: Reason;
+        successor: Reason;
+        external: ReadonlySet<Reason>;
+    };
+    lost: {
+        requeued(info: LostHead<K>): Promise<void>;
+        quarantined(info: LostHead<K> & {
+            attempts: number;
+            until: string | null;
+        }): Promise<void>;
+        budgetExhausted(info: {
+            attempts: number;
+            budget: number;
+            head: string;
+            genuine: boolean;
+        }): string;
+    };
+    recovery: {
+        candidates(): Promise<Array<{
+            key: K | null;
+            head: string;
+            finish(): Promise<boolean>;
+        }>>;
+    };
+    unconfirmed: {
+        list(): U[];
+        maxAgeMs: number;
+        settled(u: U & {
+            key: K;
+        }): Promise<boolean>;
+        requeued(u: U & {
+            key: K;
+        }, nextAttemptAt: string): Promise<void>;
+    };
+    items: {
+        name(key: K): string | null;
+        state(name: string, key: K): Promise<TargetState>;
+        confirmedHead(name: string, key: K): string | null;
+        forget(name: string, key: K): boolean;
+        invisible(name: string, key: K, head: string, until: string | null): Promise<void>;
+        tombstoned(name: string, key: K, kind: "closed" | "missing"): Promise<void>;
+        toDispatch(item: Item, name: string): D;
+        dispatch(d: D): Promise<{
+            outcome: DispatchOutcome;
+        }>;
+        texts: {
+            noName: string;
+            invisible: string;
+            closed: string;
+            missing: string;
+        };
+    };
+    sweeps: Record<string, () => Promise<number>>;
+}
+export interface ReconcilerCounters {
+    skipped: boolean;
+    reaped: number;
+    requeued: number;
+    quarantined: number;
+    tombstoned: number;
+    dispatched: number;
+    deferred: number;
+    swept: Record<string, number>;
+    recovered: number;
+    deferredReap: number;
+    tombstoneDeferred: number;
+    recoverySkipped: boolean;
+    unconfirmedSkipped: boolean;
+}
+export interface ReconcilerState {
+    lastPassAt: number;
+}
+export declare const createReconcilerState: () => ReconcilerState;
+export declare function runReconcilerPass<K, Item extends QueueItem<K>, Reason extends string, D, U extends UnconfirmedHead<K>>(policy: ReconcilerPolicy<K, Item, Reason, D, U>, state: ReconcilerState): Promise<ReconcilerCounters>;
+//# sourceMappingURL=reconciler.d.ts.map

package/dist/lib/coordination/reconciler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconciler.d.ts","sourceRoot":"","sources":["../../../src/lib/coordination/reconciler.ts"],"names":[],"mappings":"AAgDA,OAAO,EAAE,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAiB,MAAM,iBAAiB,CAAC;AAChF,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAMjE,eAAO,MAAM,oBAAoB,QAAa,CAAC;AAK/C,eAAO,MAAM,uBAAuB,QAAsB,CAAC;AAI3D,eAAO,MAAM,qBAAqB,IAAI,CAAC;AACvC,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAG3C,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,CAAC;AAE5B,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,WAAW,eAAe,CAAC,CAAC,EAAE,IAAI,SAAS,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,SAAS,MAAM;IAClF,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC;IAE/B,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IAC/C,MAAM,CACJ,IAAI,EAAE,CAAC,GAAG;QACR,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC3B,GACA,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,QAAQ,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACxD,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC/D,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAClE;AAED,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,QAAQ,CAAC,CAAC;IACzB,GAAG,EAAE,CAAC,CAAC;IACP,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAEhB,OAAO,EAAE,OAAO,CAAC;IAEjB,OAAO,EAAE,OAAO,CAAC;IAEjB,aAAa,EAAE,MAAM,CAAC;IAEtB,YAAY,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB,CAC/B,CAAC,EACD,IAAI,SAAS,SAAS,CAAC,CAAC,CAAC,EACzB,MAAM,SAAS,MAAM,EACrB,CAAC,EACD,CAAC,SAAS,eAAe,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC;IAEjD,GAAG,IAAI,MAAM,CAAC;IACd,QAAQ,IAAI,kBAAkB,CAAC;IAC/B,KAAK,EAAE,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,MAAM,EAAE;QACN,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC;QACzC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;KACpC,CAAC;IACF,IAAI,EAAE;QAEJ,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;QAEtC,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,CAAC;QAEnB,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;KACnB,CAAC;IACF,OAAO,EAAE;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAElB,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;KAC/B,CAAC;IAEF,IAAI,EAAE;QACJ,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE3C,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;SAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE3F,eAAe,CAAC,IAAI,EAAE;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,OAAO,CAAC;SAClB,GAAG,MAAM,CAAC;KACZ,CAAC;IAEF,QAAQ,EAAE;QAER,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC;YAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;SAAE,CAAC,CAAC,CAAC;KAC3F,CAAC;IAEF,WAAW,EAAE;QACX,IAAI,IAAI,CAAC,EAAE,CAAC;QAEZ,QAAQ,EAAE,MAAM,CAAC;QAEjB,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG;YAAE,GAAG,EAAE,CAAC,CAAA;SAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG;YAAE,GAAG,EAAE,CAAC,CAAA;SAAE,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACnE,CAAC;IAEF,KAAK,EAAE;QAEL,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;QAC5B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;QAElD,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;QAEnD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC;QACtC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACnF,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5E,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAAC;QACxC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;YAAE,OAAO,EAAE,eAAe,CAAA;SAAE,CAAC,CAAC;QACtD,KAAK,EAAE;YACL,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC;KACH,CAAC;IAEF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IAEjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE9B,SAAS,EAAE,MAAM,CAAC;IAElB,YAAY,EAAE,MAAM,CAAC;IAErB,iBAAiB,EAAE,MAAM,CAAC;IAE1B,eAAe,EAAE,OAAO,CAAC;IAEzB,kBAAkB,EAAE,OAAO,CAAC;CAC7B;AAGD,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,qBAAqB,QAAO,eAAsC,CAAC;AAEhF,wBAAsB,iBAAiB,CACrC,CAAC,EACD,IAAI,SAAS,SAAS,CAAC,CAAC,CAAC,EACzB,MAAM,SAAS,MAAM,EACrB,CAAC,EACD,CAAC,SAAS,eAAe,CAAC,CAAC,CAAC,EAE5B,MAAM,EAAE,gBAAgB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,EAC/C,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,kBAAkB,CAAC,CAyL7B"}

package/dist/lib/coordination/reconciler.js

@@ -0,0 +1,287 @@
+import { logger } from "../observability/logger.js";
+import { splitLeaseKey } from "./head-lease.js";
+const log = logger.child({ module: "reconciler" });
+export const UNCONFIRMED_GRACE_MS = 3 * 60_000;
+export const CLOSED_TOMBSTONE_TTL_MS = 30 * 24 * 3_600_000;
+export const CLAIM_ROUNDS_PER_PASS = 5;
+export const CLAIM_ROWS_PER_PASS = 30;
+export const TRANSIENT_BACKOFF_MS = 60_000;
+export const createReconcilerState = () => ({ lastPassAt: 0 });
+export async function runReconcilerPass(policy, state) {
+    const result = {
+        skipped: false,
+        reaped: 0,
+        requeued: 0,
+        quarantined: 0,
+        tombstoned: 0,
+        dispatched: 0,
+        deferred: 0,
+        swept: {},
+        recovered: 0,
+        deferredReap: 0,
+        tombstoneDeferred: 0,
+        recoverySkipped: false,
+        unconfirmedSkipped: false,
+    };
+    const cfg = policy.settings();
+    const now = policy.now();
+    if (now - state.lastPassAt < cfg.cadenceMs) {
+        result.skipped = true;
+        return result;
+    }
+    state.lastPassAt = now;
+    const eta = (ms) => new Date(policy.now() + ms).toISOString();
+    const attemptsOf = async (key) => {
+        const row = await policy.queue.read(key);
+        return row ? row.attempts : null;
+    };
+    try {
+        const reap = await policy.leases.reap();
+        const reaped = Array.isArray(reap) ? reap : reap.reaped;
+        result.deferredReap = Array.isArray(reap) ? 0 : reap.deferred.length;
+        for (const lease of reaped) {
+            result.reaped++;
+            const split = splitLeaseKey(lease.key);
+            const key = split ? policy.keys.fromLease(split.itemKey) : null;
+            if (!split || key === null)
+                continue;
+            const { head } = split;
+            const genuine = lease.runId !== "";
+            const nextAttemptAt = eta(0);
+            const prior = genuine ? null : await policy.queue.read(key);
+            const repeatRefusal = !genuine &&
+                prior !== null &&
+                prior !== undefined &&
+                prior.targetHead === head &&
+                !policy.reasons.external.has(prior.reason);
+            const counted = genuine || repeatRefusal;
+            await policy.queue.upsert({
+                ...key,
+                targetHead: head,
+                reason: genuine ? policy.reasons.restart : policy.reasons.refused,
+                countAttempt: counted,
+                lastError: genuine
+                    ? lease.reapReason === "stalled"
+                        ? `run ${lease.runId.slice(0, 8)} stalled: the engine still reports it live but its lease has not renewed since ${lease.heartbeatAt} — reaped past the stall ceiling`
+                        : `run ${lease.runId.slice(0, 8)} lost with its process (boot ${lease.bootId.slice(0, 8)}, last heartbeat ${lease.heartbeatAt})`
+                    : "dispatched turn never started (reservation expired)",
+            });
+            result.requeued++;
+            const name = policy.items.name(key);
+            if (name === null)
+                continue;
+            const info = {
+                key,
+                head,
+                name,
+                lease,
+                genuine,
+                counted,
+                nextAttemptAt,
+                readAttempts: () => attemptsOf(key),
+            };
+            await policy.lost.requeued(info);
+            if (counted) {
+                const attempts = await attemptsOf(key);
+                if (attempts !== null && attempts > cfg.restartBudget) {
+                    const until = await policy.queue.quarantine(key, {
+                        reason: policy.lost.budgetExhausted({
+                            attempts,
+                            budget: cfg.restartBudget,
+                            head,
+                            genuine,
+                        }),
+                        ttlMs: cfg.quarantineMs,
+                    });
+                    result.quarantined++;
+                    await policy.lost.quarantined({ ...info, attempts, until });
+                }
+            }
+        }
+    }
+    catch (err) {
+        log.error({ err }, "reconciler: lease reap pass failed");
+    }
+    const leaseRows = await policy.leases.list();
+    const leaseKeys = new Set((leaseRows ?? []).map((l) => l.key));
+    try {
+        if (leaseRows === null) {
+            result.recoverySkipped = true;
+            log.warn("reconciler: lease inventory unavailable — recovery pass skipped");
+        }
+        for (const c of leaseRows === null ? [] : await policy.recovery.candidates()) {
+            if (c.key === null)
+                continue;
+            if (leaseKeys.has(policy.keys.toLease(c.key, c.head)))
+                continue;
+            if (await c.finish())
+                result.recovered++;
+        }
+    }
+    catch (err) {
+        log.error({ err }, "reconciler: recovered-run pass failed");
+    }
+    try {
+        const work = leaseRows === null ? null : await policy.queue.list();
+        if (leaseRows === null || work === null) {
+            result.unconfirmedSkipped = true;
+            log.warn({ leases: leaseRows !== null, queue: work !== null }, "reconciler: inventory unavailable — unconfirmed-heads pass skipped");
+        }
+        const queued = new Set((work ?? []).map((w) => policy.keys.id(w)));
+        for (const u of result.unconfirmedSkipped ? [] : policy.unconfirmed.list()) {
+            const age = now - Date.parse(u.at);
+            if (!Number.isFinite(age) || age <= UNCONFIRMED_GRACE_MS)
+                continue;
+            if (age > policy.unconfirmed.maxAgeMs)
+                continue;
+            if (u.key === null)
+                continue;
+            const known = { ...u, key: u.key };
+            if (queued.has(policy.keys.id(u.key)))
+                continue;
+            if (leaseKeys.has(policy.keys.toLease(u.key, u.head)))
+                continue;
+            if (await policy.unconfirmed.settled(known))
+                continue;
+            await policy.queue.upsert({
+                ...u.key,
+                targetHead: u.head,
+                baseHead: u.base,
+                reason: policy.reasons.reconcile,
+                lastError: "dispatched head never confirmed — no live run, no queue entry",
+            });
+            result.requeued++;
+            await policy.unconfirmed.requeued(known, eta(0));
+        }
+    }
+    catch (err) {
+        log.error({ err }, "reconciler: unconfirmed-head pass failed");
+    }
+    for (const [name, sweep] of Object.entries(policy.sweeps)) {
+        try {
+            result.swept[name] = await sweep();
+        }
+        catch (err) {
+            result.swept[name] = 0;
+            log.error({ err, sweep: name }, "reconciler: sweep failed");
+        }
+    }
+    let attempts = 0;
+    let rows = 0;
+    for (let round = 0; round < CLAIM_ROUNDS_PER_PASS; round++) {
+        const want = Math.min(cfg.dispatchCap - attempts, CLAIM_ROWS_PER_PASS - rows);
+        if (want <= 0)
+            break;
+        let items = [];
+        try {
+            items = await policy.queue.claimDue(want);
+        }
+        catch (err) {
+            log.error({ err }, "reconciler: claiming due work failed");
+            break;
+        }
+        if (!items.length)
+            break;
+        rows += items.length;
+        for (const item of items) {
+            attempts += await handleItem(policy, cfg, item, result);
+        }
+        if (items.length < want)
+            break;
+    }
+    return result;
+}
+async function handleItem(policy, cfg, item, result) {
+    const { queue, items } = policy;
+    const key = policy.keys.of(item);
+    let attempted = false;
+    try {
+        const name = items.name(key);
+        if (!name) {
+            await queue.quarantine(key, { reason: items.texts.noName, ttlMs: cfg.quarantineMs });
+            result.quarantined++;
+            return 0;
+        }
+        const target = await items.state(name, key);
+        if (target.kind === "unavailable") {
+            await queue.defer(key, {
+                reason: item.reason,
+                backoffMs: TRANSIENT_BACKOFF_MS,
+                error: "target lookup unavailable",
+            });
+            result.deferred++;
+            return 0;
+        }
+        if (target.kind === "invisible") {
+            const until = await queue.quarantine(key, {
+                reason: items.texts.invisible,
+                ttlMs: cfg.quarantineMs,
+            });
+            result.quarantined++;
+            await items.invisible(name, key, item.targetHead, until);
+            return 0;
+        }
+        if (target.kind === "closed" || target.kind === "missing") {
+            const parkedUntil = await queue.quarantine(key, {
+                reason: target.kind === "closed" ? items.texts.closed : items.texts.missing,
+                ttlMs: CLOSED_TOMBSTONE_TTL_MS,
+            });
+            const marked = items.forget(name, key);
+            if (parkedUntil === null || !marked) {
+                log.warn({ name, parkedUntil, marked }, "closed tombstone did not persist — deferred");
+                await queue.defer(key, {
+                    reason: item.reason,
+                    backoffMs: TRANSIENT_BACKOFF_MS,
+                    error: "closed tombstone not persisted",
+                });
+                result.tombstoneDeferred++;
+                return 0;
+            }
+            result.tombstoned++;
+            await items.tombstoned(name, key, target.kind);
+            return 0;
+        }
+        const head = target.head ?? item.targetHead;
+        if (head !== item.targetHead) {
+            await queue.upsert({
+                ...key,
+                targetHead: head,
+                baseHead: item.baseHead,
+                reason: policy.reasons.reconcile,
+            });
+        }
+        const confirmed = items.confirmedHead(name, key);
+        if (confirmed === head) {
+            await queue.complete(key, head);
+            return 0;
+        }
+        const d = items.toDispatch({ ...item, targetHead: head, baseHead: item.baseHead ?? confirmed }, name);
+        attempted = true;
+        const r = await items.dispatch(d);
+        if (r.outcome === "dispatched" || r.outcome === "deduped") {
+            await queue.complete(key, head);
+            result.dispatched += r.outcome === "dispatched" ? 1 : 0;
+        }
+        else if (r.outcome === "queued") {
+            await queue.defer(key, { reason: policy.reasons.successor, backoffMs: cfg.settleMs });
+            result.deferred++;
+        }
+        else {
+            result.deferred++;
+        }
+        return 1;
+    }
+    catch (err) {
+        log.error({ err, item: policy.keys.id(key) }, "reconciler: work item failed");
+        await queue
+            .defer(key, {
+            reason: item.reason,
+            backoffMs: TRANSIENT_BACKOFF_MS,
+            error: String(err),
+        })
+            .catch(() => null);
+        result.deferred++;
+        return attempted ? 1 : 0;
+    }
+}
+//# sourceMappingURL=reconciler.js.map

package/dist/lib/coordination/reconciler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconciler.js","sourceRoot":"","sources":["../../../src/lib/coordination/reconciler.ts"],"names":[],"mappings":"AA+CA,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAkC,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhF,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;AAInD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAG,MAAM,CAAC;AAK/C,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC;AAI3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAEtC,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAmL3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAoB,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAOrC,MAA+C,EAC/C,KAAsB;IAEtB,MAAM,MAAM,GAAuB;QACjC,OAAO,EAAE,KAAK;QACd,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,WAAW,EAAE,CAAC;QACd,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;QACf,iBAAiB,EAAE,CAAC;QACpB,eAAe,EAAE,KAAK;QACtB,kBAAkB,EAAE,KAAK;KAC1B,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,GAAG,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;IAEvB,MAAM,GAAG,GAAG,CAAC,EAAU,EAAU,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9E,MAAM,UAAU,GAAG,KAAK,EAAE,GAAM,EAA0B,EAAE;QAC1D,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC,CAAC;IAGF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACxD,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACrE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAChE,IAAI,CAAC,KAAK,IAAI,GAAG,KAAK,IAAI;gBAAE,SAAS;YACrC,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;YACvB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,KAAK,EAAE,CAAC;YACnC,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5D,MAAM,aAAa,GACjB,CAAC,OAAO;gBACR,KAAK,KAAK,IAAI;gBACd,KAAK,KAAK,SAAS;gBACnB,KAAK,CAAC,UAAU,KAAK,IAAI;gBACzB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAgB,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,OAAO,IAAI,aAAa,CAAC;YACzC,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBACxB,GAAG,GAAG;gBACN,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;gBACjE,YAAY,EAAE,OAAO;gBAIrB,SAAS,EAAE,OAAO;oBAChB,CAAC,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS;wBAC9B,CAAC,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,kFAAkF,KAAK,CAAC,WAAW,kCAAkC;wBACrK,CAAC,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,gCAAgC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,oBAAoB,KAAK,CAAC,WAAW,GAAG;oBAClI,CAAC,CAAC,qDAAqD;aAC1D,CAAC,CAAC;YACH,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,IAAI,KAAK,IAAI;gBAAE,SAAS;YAC5B,MAAM,IAAI,GAAgB;gBACxB,GAAG;gBACH,IAAI;gBACJ,IAAI;gBACJ,KAAK;gBACL,OAAO;gBACP,OAAO;gBACP,aAAa;gBACb,YAAY,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;aACpC,CAAC;YACF,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;oBACtD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE;wBAC/C,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC;4BAClC,QAAQ;4BACR,MAAM,EAAE,GAAG,CAAC,aAAa;4BACzB,IAAI;4BACJ,OAAO;yBACR,CAAC;wBACF,KAAK,EAAE,GAAG,CAAC,YAAY;qBACxB,CAAC,CAAC;oBACH,MAAM,CAAC,WAAW,EAAE,CAAC;oBACrB,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,oCAAoC,CAAC,CAAC;IAC3D,CAAC;IAGD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;YAC7E,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI;gBAAE,SAAS;YAC7B,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;gBAAE,SAAS;YAChE,IAAI,MAAM,CAAC,CAAC,MAAM,EAAE;gBAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uCAAuC,CAAC,CAAC;IAC9D,CAAC;IAGD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACnE,IAAI,SAAS,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,CAAC,kBAAkB,GAAG,IAAI,CAAC;YACjC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,IAAI,EAAE,EACpD,oEAAoE,CACrE,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3E,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,oBAAoB;gBAAE,SAAS;YACnE,IAAI,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ;gBAAE,SAAS;YAChD,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI;gBAAE,SAAS;YAC7B,MAAM,KAAK,GAAmB,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;YACnD,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAAE,SAAS;YAChD,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;gBAAE,SAAS;YAChE,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,SAAS;YACtD,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,CAAC,GAAG;gBACR,UAAU,EAAE,CAAC,CAAC,IAAI;gBAClB,QAAQ,EAAE,CAAC,CAAC,IAAI;gBAChB,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;gBAChC,SAAS,EAAE,+DAA+D;aAC3E,CAAC,CAAC;YACH,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,0CAA0C,CAAC,CAAC;IACjE,CAAC;IAGD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAMD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,qBAAqB,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,GAAG,QAAQ,EAAE,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC9E,IAAI,IAAI,IAAI,CAAC;YAAE,MAAM;QACrB,IAAI,KAAK,GAAW,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sCAAsC,CAAC,CAAC;YAC3D,MAAM;QACR,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,MAAM;QACzB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,QAAQ,IAAI,MAAM,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI;YAAE,MAAM;IACjC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAGD,KAAK,UAAU,UAAU,CACvB,MAA4C,EAC5C,GAAuB,EACvB,IAAU,EACV,MAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAGjC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC;YACrF,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAgB;gBAC7B,SAAS,EAAE,oBAAoB;gBAC/B,KAAK,EAAE,2BAA2B;aACnC,CAAC,CAAC;YACH,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE;gBACxC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS;gBAC7B,KAAK,EAAE,GAAG,CAAC,YAAY;aACxB,CAAC,CAAC;YACH,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACzD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAK1D,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE;gBAC9C,MAAM,EAAE,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO;gBAC3E,KAAK,EAAE,uBAAuB;aAC/B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,IAAI,WAAW,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAIpC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,6CAA6C,CAAC,CAAC;gBACvF,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAgB;oBAC7B,SAAS,EAAE,oBAAoB;oBAC/B,KAAK,EAAE,gCAAgC;iBACxC,CAAC,CAAC;gBACH,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,OAAO,CAAC,CAAC;YACX,CAAC;YACD,MAAM,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC;QAC5C,IAAI,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,GAAG,GAAG;gBACN,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;aACjC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAChC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CACxB,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE,EACnE,IAAI,CACL,CAAC;QACF,SAAS,GAAG,IAAI,CAAC;QACjB,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,OAAO,KAAK,YAAY,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1D,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,OAAO,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACtF,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,8BAA8B,CAAC,CAAC;QAC9E,MAAM,KAAK;aACR,KAAK,CAAC,GAAG,EAAE;YACV,MAAM,EAAE,IAAI,CAAC,MAAgB;YAC7B,SAAS,EAAE,oBAAoB;YAC/B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;SACnB,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACrB,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}

package/dist/lib/coordination/work-queue.d.ts

@@ -0,0 +1,81 @@
+import { type PgDeps, type QueryPool } from "./pg.js";
+export type WorkPool = QueryPool;
+export interface WorkDeps extends PgDeps {
+    now?: () => number;
+}
+export type PolicyColumns = Record<string, readonly string[]>;
+type PolicyValues<P extends PolicyColumns> = {
+    [C in keyof P]: P[C][number] | null;
+};
+export interface WorkQueueConfig<M extends Record<string, string>, P extends PolicyColumns, Reason extends string> {
+    table: string;
+    key: M;
+    policy?: P;
+    reasons: {
+        initial: Reason;
+        tombstone: Reason;
+        replaySafe: ReadonlySet<Reason>;
+    };
+    ledgerTtlMs?: number;
+    claimWindowMs?: number;
+}
+export type KeyOf<M extends Record<string, string>> = {
+    [F in keyof M]: number;
+};
+export type WorkItem<K, Reason extends string, P extends PolicyColumns> = K & PolicyValues<P> & {
+    targetHead: string;
+    baseHead: string | null;
+    reason: Reason;
+    nextAt: string;
+    attempts: number;
+    lastError: string | null;
+    quarantinedUntil: string | null;
+    quarantineReason: string | null;
+    dispatchedHead: string | null;
+    eventAt: string | null;
+    createdAt: string;
+    updatedAt: string;
+};
+export type UpsertInput<K, Reason extends string, P extends PolicyColumns> = K & Partial<PolicyValues<P>> & {
+    targetHead: string;
+    baseHead?: string | null;
+    reason: Reason;
+    nextAt?: string;
+    countAttempt?: boolean;
+    lastError?: string | null;
+    eventAt?: string | null;
+};
+export interface ReadWorkOpts {
+    includeDispatched?: boolean;
+}
+export interface DeferOpts<Reason extends string> {
+    reason: Reason;
+    backoffMs: number;
+    countAttempt?: boolean;
+    error?: string | null;
+}
+export interface QuarantineOpts {
+    reason: string;
+    ttlMs: number;
+}
+export declare const DISPATCHED_LEDGER_TTL_MS: number;
+export declare const CLAIM_WINDOW_MS = 90000;
+export interface WorkQueue<K, Reason extends string, P extends PolicyColumns> {
+    readonly table: string;
+    readonly ledgerTtlMs: number;
+    readonly claimWindowMs: number;
+    ensureSchema(p: QueryPool): Promise<void>;
+    upsert(item: UpsertInput<K, Reason, P>, deps?: WorkDeps): Promise<boolean>;
+    read(key: K, deps?: WorkDeps, opts?: ReadWorkOpts): Promise<WorkItem<K, Reason, P> | null | undefined>;
+    list(deps?: WorkDeps, opts?: ReadWorkOpts): Promise<WorkItem<K, Reason, P>[] | null>;
+    claimDue(limit: number, deps?: WorkDeps): Promise<WorkItem<K, Reason, P>[]>;
+    complete(key: K, head: string | null, deps?: WorkDeps): Promise<boolean>;
+    defer(key: K, opts: DeferOpts<Reason>, deps?: WorkDeps): Promise<string | null>;
+    quarantine(key: K, opts: QuarantineOpts, deps?: WorkDeps): Promise<string | null>;
+    purgeDispatched(ttlMs?: number, deps?: WorkDeps): Promise<number>;
+    tombstone(key: K, head: string, opts: QuarantineOpts, deps?: WorkDeps): Promise<boolean>;
+    clearQuarantine(key: K, deps?: WorkDeps): Promise<boolean | null>;
+}
+export declare function defineWorkQueue<M extends Record<string, string>, const P extends PolicyColumns, Reason extends string>(config: WorkQueueConfig<M, P, Reason>): WorkQueue<KeyOf<M>, Reason, P>;
+export {};
+//# sourceMappingURL=work-queue.d.ts.map

package/dist/lib/coordination/work-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"work-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/coordination/work-queue.ts"],"names":[],"mappings":"AAgCA,OAAO,EAA4B,KAAK,MAAM,EAAW,KAAK,SAAS,EAAE,MAAM,SAAS,CAAC;AAIzF,MAAM,MAAM,QAAQ,GAAG,SAAS,CAAC;AAEjC,MAAM,WAAW,QAAS,SAAQ,MAAM;IAEtC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAGD,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;AAC9D,KAAK,YAAY,CAAC,CAAC,SAAS,aAAa,IAAI;KAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI;CAAE,CAAC;AAErF,MAAM,WAAW,eAAe,CAC9B,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,CAAC,SAAS,aAAa,EACvB,MAAM,SAAS,MAAM;IAGrB,KAAK,EAAE,MAAM,CAAC;IAEd,GAAG,EAAE,CAAC,CAAC;IAMP,MAAM,CAAC,EAAE,CAAC,CAAC;IACX,OAAO,EAAE;QAEP,OAAO,EAAE,MAAM,CAAC;QAEhB,SAAS,EAAE,MAAM,CAAC;QAQlB,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;KACjC,CAAC;IAEF,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,KAAK,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI;KAAG,CAAC,IAAI,MAAM,CAAC,GAAG,MAAM;CAAE,CAAC;AAEjF,MAAM,MAAM,QAAQ,CAAC,CAAC,EAAE,MAAM,SAAS,MAAM,EAAE,CAAC,SAAS,aAAa,IAAI,CAAC,GACzE,YAAY,CAAC,CAAC,CAAC,GAAG;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAEhC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEJ,MAAM,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,SAAS,MAAM,EAAE,CAAC,SAAS,aAAa,IAAI,CAAC,GAC5E,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IAEf,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEJ,MAAM,WAAW,YAAY;IAE3B,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,SAAS,CAAC,MAAM,SAAS,MAAM;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAGD,eAAO,MAAM,wBAAwB,QAAqB,CAAC;AAK3D,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC,MAAM,WAAW,SAAS,CAAC,CAAC,EAAE,MAAM,SAAS,MAAM,EAAE,CAAC,SAAS,aAAa;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,YAAY,CAAC,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAW1C,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3E,IAAI,CACF,GAAG,EAAE,CAAC,EACN,IAAI,CAAC,EAAE,QAAQ,EACf,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACtD,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAOrF,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IAO5E,QAAQ,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzE,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAEhF,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAElF,eAAe,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAQlE,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAMzF,eAAe,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;CACnE;AAED,wBAAgB,eAAe,CAC7B,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,KAAK,CAAC,CAAC,SAAS,aAAa,EAC7B,MAAM,SAAS,MAAM,EACrB,MAAM,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAmVvE"}