@ixo/decision-oracle 0.0.0-stage → 0.2.0

package/dist/cron/backlog.js

@@ -0,0 +1,115 @@
+/**
+ * The backlog gauges, once per minute tick (observability spec §2.2): what is waiting, and how
+ * long the oldest of it has waited — the numbers an operator alerts on before a customer notices.
+ * Every query is served by an existing index or reads a table with one row per customer, so a
+ * tick costs a handful of index reads however large the ledgers grow:
+ *   - `collections.paused` — `watched_collections (status, lastScannedAt)`;
+ *   - `review.pending` — `decisions (status, createdAt)` (a fold moves the row off `pending_review`);
+ *   - `backlog.webhook.*` — `matrix_deliveries (kind, status, nextAttemptAt)`, the retry cron's own;
+ *   - `sources.paused` — `api_sources`, one row per source;
+ *   - `backlog.charge.unsettled.oldest_age_s` (with a facilitator configured) — a decided claim
+ *     with a receipt whose charge is still `reserved`: the facilitator has not accepted the receipt
+ *     (the reconcile's arm (b)). `api_charges (status, updatedAt)` narrows to the `reserved` rows
+ *     and each joins `decisions` by its primary key — but the reserved set is every live
+ *     reservation (claims in flight, reviews held for up to 30 days, failed claims waiting out
+ *     their 24 h), which grows with traffic, so this one is read after the five-minute reconcile
+ *     rather than every minute. Age from the decision row's `updatedAt` (the receipt landed with it).
+ * Not here: the oldest queued / processing ingest, for want of an index (`claim_ingests` has no
+ * `status` index, and migrations are not this lane's to add) — it would scan the whole ledger every
+ * minute.
+ *
+ * Best-effort: a gauge that cannot be read is a warn, never a failed tick.
+ */
+import { and, count, eq, inArray, isNotNull, lte, min } from "drizzle-orm";
+import { apiCharges, apiSources, decisions, matrixDeliveries, watchedCollections, } from "../db/schema.js";
+/**
+ * The chain lane's share: paused collections by reason, decisions waiting for a reviewer. Read
+ * here, emitted by `chainBacklog` — the operator status route (routes/v1/admin.ts) serves the same
+ * numbers live.
+ */
+export async function readChainBacklog(db) {
+    const [paused, [review]] = await Promise.all([
+        db
+            .select({ reason: watchedCollections.pausedReason, n: count() })
+            .from(watchedCollections)
+            .where(eq(watchedCollections.status, "paused"))
+            .groupBy(watchedCollections.pausedReason),
+        db.select({ n: count() }).from(decisions).where(eq(decisions.status, "pending_review")),
+    ]);
+    return { reviewPending: review?.n ?? 0, collectionsPaused: byReason(paused) };
+}
+/** The Decision API lane's share: webhooks due for an attempt, and paused sources by reason. */
+export async function readApiBacklog(db, now) {
+    const [[due], paused] = await Promise.all([
+        // The same predicate `deliverDueWebhooks` selects by, so `due` is exactly what it would take.
+        db
+            .select({ n: count(), oldest: min(matrixDeliveries.nextAttemptAt) })
+            .from(matrixDeliveries)
+            .where(and(eq(matrixDeliveries.kind, "webhook"), inArray(matrixDeliveries.status, ["pending", "failed"]), lte(matrixDeliveries.nextAttemptAt, now))),
+        db
+            .select({ reason: apiSources.pausedReason, n: count() })
+            .from(apiSources)
+            .where(eq(apiSources.status, "paused"))
+            .groupBy(apiSources.pausedReason),
+    ]);
+    return {
+        webhookDue: due?.n ?? 0,
+        webhookOldestAgeS: ageSeconds(now, due?.oldest),
+        sourcesPaused: byReason(paused),
+    };
+}
+export async function chainBacklog(ctx) {
+    await gauges(ctx, "chain", async () => {
+        const { reviewPending, collectionsPaused } = await readChainBacklog(ctx.db);
+        emitByReason(ctx.metrics, "collections.paused", collectionsPaused);
+        ctx.metrics.gauge("review.pending", reviewPending);
+    });
+}
+export async function apiBacklog(ctx, now) {
+    await gauges(ctx, "api", async () => {
+        const { webhookDue, webhookOldestAgeS, sourcesPaused } = await readApiBacklog(ctx.db, now);
+        ctx.metrics.gauge("backlog.webhook.due", webhookDue);
+        ctx.metrics.gauge("backlog.webhook.oldest_age_s", webhookOldestAgeS);
+        emitByReason(ctx.metrics, "sources.paused", sourcesPaused);
+    });
+}
+function byReason(rows) {
+    const out = { total: 0 };
+    for (const { reason, n } of rows) {
+        out[reason ?? "unknown"] = n;
+        out.total += n;
+    }
+    return out;
+}
+/** Decided claims whose receipt the facilitator has not accepted: how long the oldest has waited. */
+export async function chargeBacklog(ctx, now) {
+    await gauges(ctx, "charges", async () => {
+        const [row] = await ctx.db
+            .select({ oldest: min(decisions.updatedAt) })
+            .from(apiCharges)
+            .innerJoin(decisions, eq(decisions.claimId, apiCharges.claimId))
+            .where(and(eq(apiCharges.status, "reserved"), isNotNull(decisions.receiptJws)));
+        ctx.metrics.gauge("backlog.charge.unsettled.oldest_age_s", ageSeconds(now, row?.oldest));
+    });
+}
+/** One point per reason that has any, plus a `total` that is always written (0 included). */
+function emitByReason(metrics, name, counts) {
+    for (const [reason, n] of Object.entries(counts))
+        metrics.gauge(name, n, { outcome: reason });
+}
+/** Whole seconds since `oldest`; 0 when nothing waits. `min()` over a timestamp column comes back raw. */
+function ageSeconds(now, oldest) {
+    if (oldest == null)
+        return 0;
+    const at = oldest instanceof Date ? oldest.getTime() : Number(oldest);
+    return Math.max(0, Math.floor((now.getTime() - at) / 1000));
+}
+async function gauges(ctx, lane, read) {
+    try {
+        await read();
+    }
+    catch (err) {
+        ctx.logger.warn("cron.backlog.failed", { lane, detail: String(err) });
+    }
+}
+//# sourceMappingURL=backlog.js.map

package/dist/cron/backlog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backlog.js","sourceRoot":"","sources":["../../src/cron/backlog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAE3E,OAAO,EACL,UAAU,EACV,UAAU,EACV,SAAS,EACT,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAkBtB;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY;IACjD,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC3C,EAAE;aACC,MAAM,CAAC,EAAE,MAAM,EAAE,kBAAkB,CAAC,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;aAC/D,IAAI,CAAC,kBAAkB,CAAC;aACxB,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC9C,OAAO,CAAC,kBAAkB,CAAC,YAAY,CAAC;QAC3C,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KACxF,CAAC,CAAC;IACH,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAY,EAAE,GAAS;IAC1D,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,8FAA8F;QAC9F,EAAE;aACC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAAE,CAAC;aACnE,IAAI,CAAC,gBAAgB,CAAC;aACtB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,EACpC,OAAO,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EACvD,GAAG,CAAC,gBAAgB,CAAC,aAAa,EAAE,GAAG,CAAC,CACzC,CACF;QACH,EAAE;aACC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;aACvD,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aACtC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;KACpC,CAAC,CAAC;IACH,OAAO;QACL,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC;QACvB,iBAAiB,EAAE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAY;IAC7C,MAAM,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;QACpC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5E,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,CAAC,CAAC;QACnE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAY,EAAE,GAAS;IACtD,MAAM,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE;QAClC,MAAM,EAAE,UAAU,EAAE,iBAAiB,EAAE,aAAa,EAAE,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3F,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,UAAU,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,iBAAiB,CAAC,CAAC;QACrE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,EAAE,aAAa,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAA4C;IAC5D,MAAM,GAAG,GAAa,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACnC,KAAK,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7B,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qGAAqG;AACrG,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAY,EAAE,GAAS;IACzD,MAAM,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,IAAI,EAAE;QACtC,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;aAC5C,IAAI,CAAC,UAAU,CAAC;aAChB,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;aAC/D,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAClF,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,6FAA6F;AAC7F,SAAS,YAAY,CAAC,OAAgB,EAAE,IAAY,EAAE,MAAgB;IACpE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;AAChG,CAAC;AAED,0GAA0G;AAC1G,SAAS,UAAU,CAAC,GAAS,EAAE,MAAwC;IACrE,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,MAAM,YAAY,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,GAAY,EAAE,IAAY,EAAE,IAAyB;IACzE,IAAI,CAAC;QACH,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC"}

package/dist/cron/billing.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Un-pausing after the engine's 402 (ENGINE-CONTRACT.md "Billing"): a `needs_funds` at the engine
+ * paused EVERY watched collection with `pausedReason: "engine-credits"` (workflows/claim-ingest.ts)
+ * — the credits are the OWNER's, not the collection's. The engine flips its own `billingStatus`
+ * back when the owner's account covers what is owed (its billing cron, or a re-register); this
+ * tick asks it ONCE, only while something is paused, and resumes them all together. Never resumes
+ * an operator's pause (`pausedReason: null`) or an engine reason that isn't ours to clear.
+ */
+import type { CronCtx } from "./collection.js";
+export declare function resumeEngineCredits(ctx: CronCtx): Promise<number>;
+//# sourceMappingURL=billing.d.ts.map

package/dist/cron/billing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"billing.d.ts","sourceRoot":"","sources":["../../src/cron/billing.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAuEvE"}

package/dist/cron/billing.js

@@ -0,0 +1,69 @@
+/**
+ * Un-pausing after the engine's 402 (ENGINE-CONTRACT.md "Billing"): a `needs_funds` at the engine
+ * paused EVERY watched collection with `pausedReason: "engine-credits"` (workflows/claim-ingest.ts)
+ * — the credits are the OWNER's, not the collection's. The engine flips its own `billingStatus`
+ * back when the owner's account covers what is owed (its billing cron, or a re-register); this
+ * tick asks it ONCE, only while something is paused, and resumes them all together. Never resumes
+ * an operator's pause (`pausedReason: null`) or an engine reason that isn't ours to clear.
+ */
+import { and, eq } from "drizzle-orm";
+import { loadConfig } from "../config.js";
+import { watchedCollections } from "../db/schema.js";
+import { createEngineClient } from "../lib/engine/client.js";
+import { notifyCollectionResumed } from "../workflows/steps.js";
+export async function resumeEngineCredits(ctx) {
+    const { db, logger } = ctx;
+    const paused = await db
+        .select({ collectionId: watchedCollections.collectionId })
+        .from(watchedCollections)
+        .where(and(eq(watchedCollections.status, "paused"), eq(watchedCollections.pausedReason, "engine-credits")))
+        .limit(1);
+    if (paused.length === 0)
+        return 0;
+    // `ctx.deps.config` is `BlocksyncDeps`' narrow slice (cron/collection.ts) — the engine client
+    // (and the notice deps below) need the full `Config`, so it's reloaded from `ctx.env` rather
+    // than widening that type.
+    const config = loadConfig(ctx.env);
+    const engine = createEngineClient({
+        config,
+        trace: ctx.deps.trace,
+        logger,
+        cache: ctx.env.CACHE_KV,
+    });
+    let me;
+    try {
+        me = await engine.me();
+    }
+    catch (err) {
+        // Unreachable is not "funded": stay paused, say so at warn (once per tick, not per row).
+        logger.warn("cron.engine-credits.unreadable", { detail: String(err) });
+        return 0;
+    }
+    if (me.billingStatus !== "ok") {
+        logger.debug("cron.engine-credits.still-paused", {
+            topUpUrl: me.manageSubscriptionUrl ?? null,
+        });
+        return 0;
+    }
+    const rows = await db
+        .update(watchedCollections)
+        .set({ status: "active", pausedReason: null })
+        .where(and(eq(watchedCollections.status, "paused"), eq(watchedCollections.pausedReason, "engine-credits")))
+        .returning({ collectionId: watchedCollections.collectionId });
+    logger.info("cron.engine-credits.resumed", { collections: rows.map((r) => r.collectionId) });
+    // The pause (claim-ingest's `pauseCollection`) told the owner once, told-once; the un-pause owes
+    // its pair — else an owner who was told "stopped" is never told "you're back". Fail-soft: a
+    // notice failure must never undo or abort a resume that already committed.
+    for (const { collectionId } of rows) {
+        await notifyCollectionResumed(db, {
+            logger,
+            metrics: ctx.metrics,
+            config,
+            trace: ctx.deps.trace,
+            cache: ctx.env.CACHE_KV,
+            email: ctx.env.EMAIL,
+        }, collectionId).catch((err) => logger.warn("collection.notice.failed", { collectionId, detail: String(err) }));
+    }
+    return rows.length;
+}
+//# sourceMappingURL=billing.js.map

package/dist/cron/billing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"billing.js","sourceRoot":"","sources":["../../src/cron/billing.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAG7D,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAY;IACpD,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,EAAE;SACpB,MAAM,CAAC,EAAE,YAAY,EAAE,kBAAkB,CAAC,YAAY,EAAE,CAAC;SACzD,IAAI,CAAC,kBAAkB,CAAC;SACxB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,EACvC,EAAE,CAAC,kBAAkB,CAAC,YAAY,EAAE,gBAAgB,CAAC,CACtD,CACF;SACA,KAAK,CAAC,CAAC,CAAC,CAAC;IACZ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAElC,8FAA8F;IAC9F,6FAA6F;IAC7F,2BAA2B;IAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAChC,MAAM;QACN,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;QACrB,MAAM;QACN,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ;KACxB,CAAC,CAAC;IACH,IAAI,EAAyC,CAAC;IAC9C,IAAI,CAAC;QACH,EAAE,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,yFAAyF;QACzF,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,EAAE,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;YAC/C,QAAQ,EAAE,EAAE,CAAC,qBAAqB,IAAI,IAAI;SAC3C,CAAC,CAAC;QACH,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,EAAE;SAClB,MAAM,CAAC,kBAAkB,CAAC;SAC1B,GAAG,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;SAC7C,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,EACvC,EAAE,CAAC,kBAAkB,CAAC,YAAY,EAAE,gBAAgB,CAAC,CACtD,CACF;SACA,SAAS,CAAC,EAAE,YAAY,EAAE,kBAAkB,CAAC,YAAY,EAAE,CAAC,CAAC;IAChE,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAE7F,iGAAiG;IACjG,4FAA4F;IAC5F,2EAA2E;IAC3E,KAAK,MAAM,EAAE,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC;QACpC,MAAM,uBAAuB,CAC3B,EAAE,EACF;YACE,MAAM;YACN,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,MAAM;YACN,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;YACrB,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ;YACvB,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK;SACrB,EACD,YAAY,CACb,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACd,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAC/E,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC"}

package/dist/cron/collection.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Processing ONE watched collection per cron tick (README "How a claim flows" step 2):
+ *
+ *   fetchNewClaims → filterUnseen → startInstances → advanceCursor → stampScanned
+ *
+ * The cron only DISCOVERS claims and STARTS workflow instances — the workflow owns its own
+ * `claim_ingests` row for its whole lifecycle (see workflows/claim-ingest.ts), so there is no
+ * ledger writing, releasing, or healing here. Duplicate protection is two-layer and simple:
+ *   - `filterUnseen`: a claim with a ledger row was already started (the workflow durably writes
+ *     its row as its first step) — skip it.
+ *   - `createBatch` is idempotent: an instance id that already exists (within retention) is
+ *     skipped by the runtime, so even two overlapping ticks racing the same page converge on
+ *     exactly one instance per claim.
+ *
+ * Cursor rules (unchanged from the engine's proven design):
+ *   - INCLUSIVE bound (`>=`): claims sharing a timestamp with the watermark are re-seen, never
+ *     skipped; `filterUnseen` makes re-seeing free.
+ *   - Forward-only, enforced in SQL: a slow overlapping tick can never rewind a faster one.
+ *     (Cursors are blocksync's fixed-format ISO timestamps — text order IS time order.)
+ *   - Advanced only AFTER instances started, so a failure re-scans the page instead of skipping it.
+ *   - Pathological tie (a full page sharing one timestamp) can't advance the cursor: we warn loudly
+ *     (`cron.cursor.stuck`) instead of spinning silently. SCAN_LIMIT bounds how many same-second
+ *     claims that takes.
+ *
+ * Rubrics are NOT this lane's business: the workflow's facts step resolves and pins the
+ * collection's `#rub` per claim (workflows/claim-ingest.ts). The cron never reads or writes `rubricId`.
+ */
+import type { Database } from "../db/client.js";
+import type { OracleEnv } from "../env.js";
+import { type BlocksyncDeps } from "../lib/ixo/blocksync/index.js";
+import type { Logger } from "../lib/logger.js";
+import type { Metrics } from "../lib/metrics.js";
+/**
+ * Claims fetched per collection per tick — also the createBatch ceiling (100/call) AND D1's cap on
+ * bound parameters per statement (100), which `filterUnseen`'s `IN (…)` spends one per claim.
+ */
+export declare const SCAN_LIMIT = 100;
+export interface CronCtx {
+    deps: BlocksyncDeps;
+    db: Database;
+    env: OracleEnv;
+    logger: Logger;
+    metrics: Metrics;
+}
+/** Discover new claims in one collection and start their ingest workflows. Returns the count. */
+export declare function processCollection(ctx: CronCtx, collection: {
+    collectionId: string;
+    cursor: string | null;
+}): Promise<number>;
+//# sourceMappingURL=collection.d.ts.map

package/dist/cron/collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../../src/cron/collection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAExC,OAAO,EAAE,KAAK,aAAa,EAAqC,MAAM,sBAAsB,CAAC;AAC7F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;;GAGG;AACH,eAAO,MAAM,UAAU,MAAM,CAAC;AAE9B,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,aAAa,CAAC;IACpB,EAAE,EAAE,QAAQ,CAAC;IACb,GAAG,EAAE,SAAS,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,iGAAiG;AACjG,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,OAAO,EACZ,UAAU,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1D,OAAO,CAAC,MAAM,CAAC,CAgBjB"}

package/dist/cron/collection.js

@@ -0,0 +1,118 @@
+/**
+ * Processing ONE watched collection per cron tick (README "How a claim flows" step 2):
+ *
+ *   fetchNewClaims → filterUnseen → startInstances → advanceCursor → stampScanned
+ *
+ * The cron only DISCOVERS claims and STARTS workflow instances — the workflow owns its own
+ * `claim_ingests` row for its whole lifecycle (see workflows/claim-ingest.ts), so there is no
+ * ledger writing, releasing, or healing here. Duplicate protection is two-layer and simple:
+ *   - `filterUnseen`: a claim with a ledger row was already started (the workflow durably writes
+ *     its row as its first step) — skip it.
+ *   - `createBatch` is idempotent: an instance id that already exists (within retention) is
+ *     skipped by the runtime, so even two overlapping ticks racing the same page converge on
+ *     exactly one instance per claim.
+ *
+ * Cursor rules (unchanged from the engine's proven design):
+ *   - INCLUSIVE bound (`>=`): claims sharing a timestamp with the watermark are re-seen, never
+ *     skipped; `filterUnseen` makes re-seeing free.
+ *   - Forward-only, enforced in SQL: a slow overlapping tick can never rewind a faster one.
+ *     (Cursors are blocksync's fixed-format ISO timestamps — text order IS time order.)
+ *   - Advanced only AFTER instances started, so a failure re-scans the page instead of skipping it.
+ *   - Pathological tie (a full page sharing one timestamp) can't advance the cursor: we warn loudly
+ *     (`cron.cursor.stuck`) instead of spinning silently. SCAN_LIMIT bounds how many same-second
+ *     claims that takes.
+ *
+ * Rubrics are NOT this lane's business: the workflow's facts step resolves and pins the
+ * collection's `#rub` per claim (workflows/claim-ingest.ts). The cron never reads or writes `rubricId`.
+ */
+import { and, eq, inArray, isNull, lt, or } from "drizzle-orm";
+import { claimIngests, watchedCollections } from "../db/schema.js";
+import { instanceIdFor } from "../lib/ids.js";
+import { getClaimsSince } from "../lib/ixo/blocksync/index.js";
+import { toTraceparent } from "../lib/tracing.js";
+/**
+ * Claims fetched per collection per tick — also the createBatch ceiling (100/call) AND D1's cap on
+ * bound parameters per statement (100), which `filterUnseen`'s `IN (…)` spends one per claim.
+ */
+export const SCAN_LIMIT = 100;
+/** Discover new claims in one collection and start their ingest workflows. Returns the count. */
+export async function processCollection(ctx, collection) {
+    const { collectionId, cursor } = collection;
+    try {
+        const claims = await getClaimsSince(ctx.deps, collectionId, cursor, SCAN_LIMIT);
+        if (claims.length === 0)
+            return 0;
+        const unseen = await filterUnseen(ctx.db, claims);
+        if (unseen.length > 0)
+            await startInstances(ctx, collectionId, unseen);
+        await advanceCursor(ctx, collectionId, cursor, claims, unseen.length);
+        return unseen.length;
+    }
+    finally {
+        // Rotation clock: stamped on every visit — found claims, found nothing, or failed — so a busy
+        // or broken collection rotates to the back of the queue instead of starving the rest.
+        await stampScanned(ctx.db, collectionId).catch(() => { });
+    }
+}
+/** Drop claims that already have a ledger row — their workflow has already started (or finished). */
+async function filterUnseen(db, claims) {
+    const seen = await db
+        .select({ claimId: claimIngests.claimId })
+        .from(claimIngests)
+        .where(inArray(claimIngests.claimId, claims.map((c) => c.claimId)));
+    const seenIds = new Set(seen.map((r) => r.claimId));
+    return claims.filter((c) => !seenIds.has(c.claimId));
+}
+/** One createBatch call for the page (idempotent — existing instance ids are skipped). */
+async function startInstances(ctx, collectionId, claims) {
+    const batch = await Promise.all(claims.map(async (c) => ({
+        id: await instanceIdFor(c.claimId),
+        // The tick's trace, so the claim's run logs under the scan that found it.
+        params: { claimId: c.claimId, collectionId, traceparent: toTraceparent(ctx.deps.trace) },
+    })));
+    const created = await ctx.env.CLAIM_WORKFLOW.createBatch(batch);
+    // Skipped = a retained instance already exists for a claim with NO ledger row — happens when the
+    // ledger was cleared (e.g. an operator re-ingest) within the instance-retention window. No run
+    // starts, so no row will appear; say so LOUDLY. Converge by restarting that instance from the
+    // dashboard (or wait out retention). Never silent — an operator must be able to see why a
+    // re-ingested claim didn't come back.
+    const createdIds = new Set(created.map((i) => i.id));
+    const skipped = batch.filter((b) => !createdIds.has(b.id));
+    if (skipped.length > 0) {
+        ctx.logger.warn("cron.claims.skipped-retained-instance", {
+            collectionId,
+            claimIds: skipped.map((b) => b.params.claimId),
+            hint: "instance exists within retention — restart it from the dashboard to re-ingest",
+        });
+    }
+    ctx.metrics.count("cron.claim.started", claims.length);
+    ctx.logger.info("cron.claims.started", {
+        collectionId,
+        count: claims.length,
+        claimIds: claims.map((c) => c.claimId),
+    });
+}
+/** Move the watermark to the newest dated claim in the page — forward-only, enforced in SQL. */
+async function advanceCursor(ctx, collectionId, cursor, claims, started) {
+    // Newest non-NULL submissionDate (NULLs sort last in ASC and must not park the cursor).
+    const advanceTo = claims.filter((c) => c.submissionDate).at(-1)?.submissionDate ?? null;
+    if (advanceTo && advanceTo !== cursor) {
+        await ctx.db
+            .update(watchedCollections)
+            .set({ cursor: advanceTo })
+            .where(and(eq(watchedCollections.collectionId, collectionId), or(isNull(watchedCollections.cursor), lt(watchedCollections.cursor, advanceTo))));
+        return;
+    }
+    // Full page, nothing new, watermark can't move: a same-timestamp tie wider than SCAN_LIMIT.
+    if (claims.length === SCAN_LIMIT && started === 0) {
+        ctx.logger.warn("cron.cursor.stuck", { collectionId, cursor, tie: claims.length });
+    }
+}
+/** Stamp the rotation clock (see schema `lastScannedAt`). */
+async function stampScanned(db, collectionId) {
+    await db
+        .update(watchedCollections)
+        .set({ lastScannedAt: new Date() })
+        .where(eq(watchedCollections.collectionId, collectionId));
+}
+//# sourceMappingURL=collection.js.map

package/dist/cron/collection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collection.js","sourceRoot":"","sources":["../../src/cron/collection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAsB,cAAc,EAAqB,MAAM,sBAAsB,CAAC;AAG7F,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,CAAC;AAU9B,iGAAiG;AACjG,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAY,EACZ,UAA2D;IAE3D,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAChF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAClD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,cAAc,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAEvE,MAAM,aAAa,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;YAAS,CAAC;QACT,8FAA8F;QAC9F,sFAAsF;QACtF,MAAM,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,KAAK,UAAU,YAAY,CAAC,EAAY,EAAE,MAAsB;IAC9D,MAAM,IAAI,GAAG,MAAM,EAAE;SAClB,MAAM,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC;SACzC,IAAI,CAAC,YAAY,CAAC;SAClB,KAAK,CACJ,OAAO,CACL,YAAY,CAAC,OAAO,EACpB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;IACJ,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,cAAc,CAC3B,GAAY,EACZ,YAAoB,EACpB,MAAsB;IAEtB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACvB,EAAE,EAAE,MAAM,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC;QAClC,0EAA0E;QAC1E,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;KACzF,CAAC,CAAC,CACJ,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAEhE,iGAAiG;IACjG,+FAA+F;IAC/F,8FAA8F;IAC9F,0FAA0F;IAC1F,sCAAsC;IACtC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE;YACvD,YAAY;YACZ,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAC9C,IAAI,EAAE,+EAA+E;SACtF,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;QACrC,YAAY;QACZ,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;KACvC,CAAC,CAAC;AACL,CAAC;AAED,gGAAgG;AAChG,KAAK,UAAU,aAAa,CAC1B,GAAY,EACZ,YAAoB,EACpB,MAAqB,EACrB,MAAsB,EACtB,OAAe;IAEf,wFAAwF;IACxF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,IAAI,IAAI,CAAC;IAExF,IAAI,SAAS,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACtC,MAAM,GAAG,CAAC,EAAE;aACT,MAAM,CAAC,kBAAkB,CAAC;aAC1B,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;aAC1B,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,CAAC,EACjD,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAChF,CACF,CAAC;QACJ,OAAO;IACT,CAAC;IAED,4FAA4F;IAC5F,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,6DAA6D;AAC7D,KAAK,UAAU,YAAY,CAAC,EAAY,EAAE,YAAoB;IAC5D,MAAM,EAAE;SACL,MAAM,CAAC,kBAAkB,CAAC;SAC1B,GAAG,CAAC,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;SAClC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC;AAC9D,CAAC"}

package/dist/cron/funding.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Un-pausing after the chain's "insufficient funds: payment withdrawal failed" (lib/ixo/
+ * chain-write.ts): the collection's admin account could not pay what the message draws, so the
+ * act lane parked the decision `blocked` and paused the collection `admin-unfunded`. The fix is
+ * the OWNER's (top up the admin account — the paused notice names it and the amount), so this
+ * tick reads that account's spendable balance and, once it covers the largest single parked
+ * delivery per denom (`largestParkedDraw`: the evaluation leg PLUS that verdict's payment — chain
+ * v8 fails an approve whole unless the admin covers both, so the evaluation fee alone would
+ * resume, re-fail and re-pause in a loop), resumes the collection and RE-DELIVERS every decision
+ * parked behind the empty account — the ones that exhausted the act step's retries as `pending`
+ * too. Never resumes an operator's pause (`pausedReason: null`) or a reason that isn't ours to
+ * clear. The twin of `billing.ts`.
+ */
+import type { CronCtx } from "./collection.js";
+export declare function resumeAdminFunds(ctx: CronCtx): Promise<number>;
+//# sourceMappingURL=funding.d.ts.map

package/dist/cron/funding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"funding.d.ts","sourceRoot":"","sources":["../../src/cron/funding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAYH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAa5C,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAwIpE"}