@stellar-agent-services/provider-kit 0.0.0-stage → 0.1.1

package/dist/jobs/reconcile.js

@@ -0,0 +1,82 @@
+// Reconciliation: deciding payment_uncertain jobs from on-chain data [J13].
+//
+// A job is payment_uncertain when its settlement was attempted and no usable answer came
+// (payments/settlement.ts, jobs/recovery.ts). It is never charged again. The chain says what happened
+// (payments/onchain.ts), and the job moves on as §5.6 allows:
+//
+//   settled      upfront: queued, with the receipt, and a worker runs it.
+//                on-success: succeeded, with the receipt. The output was stored before settling [J14].
+//   not settled  failed with payment_failed, charged: false.
+//   undecided    stays payment_uncertain, and the next run looks again.
+//
+// The provider runs this after every recovery sweep when it has a Soroban RPC, and operators run it
+// with `provider-kit reconcile`. Each update is compare-and-set from payment_uncertain, so a late
+// settlement answer, another process or a second run deciding the same job is harmless: they all read
+// the same chain, and the first decision stands.
+import { jobError } from "@stellar-agent-services/protocol";
+import { findPayment } from "../payments/onchain.js";
+const DEFAULT_LIMIT = 500;
+/** Decides the store's payment_uncertain jobs that the chain can decide. */
+export async function reconcile(store, options) {
+    const jobs = await store.list({
+        statuses: ["payment_uncertain"],
+        createdBefore: new Date(Date.now() + 1000).toISOString(),
+        limit: options.limit ?? DEFAULT_LIMIT,
+    });
+    if (jobs.length === 0)
+        return [];
+    // One read of the RPC's ledger range serves every job of the run.
+    let health;
+    try {
+        health = await options.chain.getHealth();
+    }
+    catch (error) {
+        const reason = `the chain could not be read: ${error.message}`;
+        return jobs.map((job) => ({ jobId: job.jobId, service: job.service, outcome: "undecided", status: job.status, reason }));
+    }
+    const results = [];
+    for (const job of jobs)
+        results.push(await reconcileJob(store, job, options, health));
+    return results;
+}
+async function reconcileJob(store, job, options, health) {
+    // Every ledger the payment could be in is gone from this RPC's history: nothing to ask it, so a
+    // backlog of such jobs costs no calls. One with more history (provider-kit reconcile --rpc) may tell.
+    const lookup = job.claim.expirationLedger < health.oldestLedger
+        ? { outcome: "undecided", reason: `the payment's ledgers are older than the RPC's history (it starts at ledger ${health.oldestLedger}); use an RPC with more history` }
+        : await findPayment(job.claim, job.settlement?.transaction, options, health);
+    const result = (status, extra = {}) => ({
+        jobId: job.jobId,
+        service: job.service,
+        outcome: lookup.outcome,
+        status,
+        ...extra,
+    });
+    let patch;
+    let extra;
+    switch (lookup.outcome) {
+        case "undecided":
+            return result(job.status, { reason: lookup.reason });
+        case "settled":
+            extra = { transaction: lookup.receipt.transaction };
+            if (job.charge === "upfront")
+                patch = { status: "queued", payment: lookup.receipt };
+            // The runner stores the output before it settles, so this means a broken store: never report a paid job as unpaid.
+            else if (job.output === undefined)
+                return { ...result(job.status, extra), outcome: "undecided", reason: "the payment settled, but the job's output is missing" };
+            else
+                patch = { status: "succeeded", payment: lookup.receipt };
+            break;
+        case "not_settled":
+            extra = { reason: lookup.reason };
+            patch = { status: "failed", error: jobError("payment_failed", `the payment was not settled (${lookup.reason}); nothing was charged`) };
+            break;
+    }
+    if (options.dryRun)
+        return result(patch.status, extra);
+    const updated = await store.update(job.jobId, "payment_uncertain", patch);
+    // Decided meanwhile, by a late settlement answer or another process: that decision stands.
+    const status = updated?.status ?? (await store.get(job.jobId))?.status ?? job.status;
+    return result(status, extra);
+}
+//# sourceMappingURL=reconcile.js.map

package/dist/jobs/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../../src/jobs/reconcile.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,yFAAyF;AACzF,sGAAsG;AACtG,8DAA8D;AAC9D,EAAE;AACF,0EAA0E;AAC1E,uGAAuG;AACvG,6DAA6D;AAC7D,wEAAwE;AACxE,EAAE;AACF,oGAAoG;AACpG,kGAAkG;AAClG,sGAAsG;AACtG,iDAAiD;AAEjD,OAAO,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AAE5D,OAAO,EAAE,WAAW,EAA0C,MAAM,wBAAwB,CAAC;AAsB7F,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAe,EAAE,OAAyB;IACxE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC;QAC5B,QAAQ,EAAE,CAAC,mBAAmB,CAAC;QAC/B,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;QACxD,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,aAAa;KACtC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,kEAAkE;IAClE,IAAI,MAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,gCAAiC,KAAe,CAAC,OAAO,EAAE,CAAC;QAC1E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3H,CAAC;IACD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,IAAI;QAAE,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACtF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAe,EAAE,GAAc,EAAE,OAAyB,EAAE,MAAiC;IACvH,gGAAgG;IAChG,sGAAsG;IACtG,MAAM,MAAM,GACV,GAAG,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,YAAY;QAC9C,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,+EAA+E,MAAM,CAAC,YAAY,iCAAiC,EAAE;QACvK,CAAC,CAAC,MAAM,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,CAAC,MAAoB,EAAE,QAAoD,EAAE,EAAc,EAAE,CAAC,CAAC;QAC5G,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM;QACN,GAAG,KAAK;KACT,CAAC,CAAC;IAEH,IAAI,KAAe,CAAC;IACpB,IAAI,KAAiD,CAAC;IACtD,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,KAAK,WAAW;YACd,OAAO,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,KAAK,SAAS;YACZ,KAAK,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACpD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;gBAAE,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;YACpF,mHAAmH;iBAC9G,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,sDAAsD,EAAE,CAAC;;gBAC5J,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9D,MAAM;QACR,KAAK,aAAa;YAChB,KAAK,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,gBAAgB,EAAE,gCAAgC,MAAM,CAAC,MAAM,wBAAwB,CAAC,EAAE,CAAC;YACvI,MAAM;IACV,CAAC;IAED,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,mBAAmB,EAAE,KAAK,CAAC,CAAC;IAC1E,2FAA2F;IAC3F,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC;IACrF,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC/B,CAAC"}

package/dist/jobs/recovery.d.ts

@@ -0,0 +1,19 @@
+import type { LedgerClock } from "../payments/chain.js";
+import type { JobStore } from "./store.js";
+/** How far past its last deadline a job must be before a sweep decides that its process is gone. */
+export declare const ORPHAN_MARGIN_SECONDS = 60;
+export interface SweepOptions {
+    maxTimeoutSeconds: number;
+    /** How long a settle call may take (payments/facilitator.ts). */
+    settleTimeoutSeconds: number;
+    retentionSeconds: number;
+    /** Tells that a payment expired from its ledger, before the wall-clock cutoff. */
+    clock?: LedgerClock | undefined;
+}
+export declare function sweep(store: JobStore, options: SweepOptions, now?: number): Promise<void>;
+/**
+ * Paid jobs, by service, of services not in `offered`: queued or running when the provider stopped
+ * offering them. They wait, so restoring the service runs them, and check-status keeps answering.
+ */
+export declare function strandedJobs(store: JobStore, offered: ReadonlyMap<string, unknown>): Promise<Map<string, number>>;
+//# sourceMappingURL=recovery.d.ts.map

package/dist/jobs/recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../src/jobs/recovery.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAa,MAAM,YAAY,CAAC;AAEtD,oGAAoG;AACpG,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAGxC,MAAM,WAAW,YAAY;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iEAAiE;IACjE,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACjC;AAED,wBAAsB,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,SAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CA2BnG;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMvH"}

package/dist/jobs/recovery.js

@@ -0,0 +1,65 @@
+// Recovery sweeps: decide what a crash, a lost worker or the passage of time left behind. Run at
+// startup and then periodically, by every process sharing a store (the steps are compare-and-set, so
+// running them twice is harmless).
+//
+// - A settlement that never answered (the process died, or settle outlived every deadline) becomes
+//   payment_uncertain, to be decided from on-chain data (jobs/reconcile.ts) [J13]. Nothing is charged
+//   again.
+// - An on-success job that never started settling before its payment expired can never be charged:
+//   it fails with `timeout`, uncharged [S2], [Q4]. Expired means the ledger clock is past the payment's
+//   expiration ledger, or, whether or not the clock answers, maxTimeoutSeconds have passed since the
+//   claim.
+// - An upfront job whose worker died is not handled here: workers lease it again (store.lease()).
+// - Final jobs past jobs.retentionSeconds are deleted [J15].
+import { jobError } from "@stellar-agent-services/protocol";
+/** How far past its last deadline a job must be before a sweep decides that its process is gone. */
+export const ORPHAN_MARGIN_SECONDS = 60;
+const BATCH = 500;
+export async function sweep(store, options, now = Date.now()) {
+    const settleCutoff = now - (options.settleTimeoutSeconds + ORPHAN_MARGIN_SECONDS) * 1000;
+    for (const job of await all(store, ["pending", "running"], settleCutoff)) {
+        // A running upfront job was settled before it queued; only on-success jobs settle while running.
+        if (job.status === "running" && job.charge !== "on-success")
+            continue;
+        if (job.settlingSince === undefined || Date.parse(job.settlingSince) >= settleCutoff)
+            continue;
+        console.error(`[provider-kit] job ${job.jobId}: its settlement never answered; it is payment_uncertain until reconciled`);
+        await store.update(job.jobId, job.status, {
+            status: "payment_uncertain",
+            settlement: { reason: "the settlement did not answer before the provider stopped" },
+        });
+    }
+    // An on-success job settles before its payment expires or not at all. Past its expiration ledger the
+    // signature is void, so even a live process can no longer charge for the job.
+    const paymentCutoff = now - (options.maxTimeoutSeconds + ORPHAN_MARGIN_SECONDS) * 1000;
+    const latestLedger = await options.clock?.latest().catch(() => undefined);
+    for (const job of await all(store, ["queued", "running"], latestLedger === undefined ? paymentCutoff : now + 1000)) {
+        if (job.charge !== "on-success" || job.settlingSince !== undefined)
+            continue;
+        const expired = Date.parse(job.createdAt) < paymentCutoff || (latestLedger !== undefined && latestLedger > job.claim.expirationLedger);
+        if (!expired)
+            continue;
+        const detail = "the job did not finish before its payment expired; nothing was charged";
+        if (job.status === "queued" && (await store.update(job.jobId, "queued", { status: "running" })) === undefined)
+            continue;
+        await store.update(job.jobId, "running", { status: "failed", error: jobError("timeout", detail) });
+    }
+    await store.deleteFinishedBefore(new Date(now - options.retentionSeconds * 1000).toISOString());
+}
+/**
+ * Paid jobs, by service, of services not in `offered`: queued or running when the provider stopped
+ * offering them. They wait, so restoring the service runs them, and check-status keeps answering.
+ */
+export async function strandedJobs(store, offered) {
+    const stranded = new Map();
+    for (const job of await all(store, ["queued", "running"], Date.now() + 1000)) {
+        if (!offered.has(job.service))
+            stranded.set(job.service, (stranded.get(job.service) ?? 0) + 1);
+    }
+    return stranded;
+}
+/** Up to BATCH jobs per status group and sweep: the next sweep takes the rest. */
+function all(store, statuses, before) {
+    return store.list({ statuses, createdBefore: new Date(before).toISOString(), limit: BATCH });
+}
+//# sourceMappingURL=recovery.js.map

package/dist/jobs/recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../src/jobs/recovery.ts"],"names":[],"mappings":"AAAA,iGAAiG;AACjG,qGAAqG;AACrG,mCAAmC;AACnC,EAAE;AACF,mGAAmG;AACnG,sGAAsG;AACtG,WAAW;AACX,mGAAmG;AACnG,wGAAwG;AACxG,qGAAqG;AACrG,WAAW;AACX,kGAAkG;AAClG,6DAA6D;AAE7D,OAAO,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AAI5D,oGAAoG;AACpG,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACxC,MAAM,KAAK,GAAG,GAAG,CAAC;AAWlB,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,KAAe,EAAE,OAAqB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IAClF,MAAM,YAAY,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,oBAAoB,GAAG,qBAAqB,CAAC,GAAG,IAAI,CAAC;IACzF,KAAK,MAAM,GAAG,IAAI,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC;QACzE,iGAAiG;QACjG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY;YAAE,SAAS;QACtE,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,YAAY;YAAE,SAAS;QAC/F,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,KAAK,2EAA2E,CAAC,CAAC;QAC1H,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;YACxC,MAAM,EAAE,mBAAmB;YAC3B,UAAU,EAAE,EAAE,MAAM,EAAE,2DAA2D,EAAE;SACpF,CAAC,CAAC;IACL,CAAC;IAED,qGAAqG;IACrG,8EAA8E;IAC9E,MAAM,aAAa,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,iBAAiB,GAAG,qBAAqB,CAAC,GAAG,IAAI,CAAC;IACvF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC1E,KAAK,MAAM,GAAG,IAAI,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC;QACnH,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS;YAAE,SAAS;QAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,aAAa,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvI,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,MAAM,GAAG,wEAAwE,CAAC;QACxF,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,SAAS;YAAE,SAAS;QACxH,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,oBAAoB,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAClG,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAe,EAAE,OAAqC;IACvF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAC7E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjG,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,kFAAkF;AAClF,SAAS,GAAG,CAAC,KAAe,EAAE,QAA+B,EAAE,MAAc;IAC3E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/F,CAAC"}

package/dist/jobs/runner.d.ts

@@ -0,0 +1,38 @@
+import { type JsonObject } from "@stellar-agent-services/protocol";
+import type { PaymentPayload, PaymentRequirements, SettleResponse } from "@x402/core/types";
+import type { Service } from "../define-service.js";
+import type { LedgerClock } from "../payments/chain.js";
+import type { Facilitator } from "../payments/facilitator.js";
+import type { CheckedPayment } from "../payments/payload-checks.js";
+import { type Scheduler } from "./scheduler.js";
+import type { JobStore, StoredJob } from "./store.js";
+/** Starts of one upfront job before it fails: a handler that keeps killing its worker is not retried forever. */
+export declare const MAX_ATTEMPTS = 3;
+export interface PaidRun {
+    store: JobStore;
+    facilitator: Facilitator;
+    scheduler: Scheduler;
+    service: Service;
+    input: JsonObject;
+    payload: PaymentPayload;
+    requirements: PaymentRequirements;
+    payment: CheckedPayment;
+    /** How long settlement may take before the job is payment_uncertain. */
+    settleTimeoutMs: number;
+    /** Times an on-success job by its payment's expiration ledger. */
+    clock?: LedgerClock | undefined;
+}
+/** on-success: resolves when the job is final or payment_uncertain, with the settle response if settled. */
+export declare function runOnSuccess(job: StoredJob, run: PaidRun): Promise<SettleResponse | undefined>;
+/** upfront: settles, then queues the job for a worker. Resolves once it is queued, failed or payment_uncertain. */
+export declare function settleUpfront(job: StoredJob, run: PaidRun): Promise<SettleResponse | undefined>;
+export interface WorkerContext {
+    store: JobStore;
+    services: ReadonlyMap<string, Service>;
+    scheduler: Scheduler;
+    owner: string;
+    leaseSeconds: number;
+}
+/** Runs a leased upfront job to its end, renewing the lease while the handler works. */
+export declare function runLeased(job: StoredJob, context: WorkerContext): Promise<void>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/jobs/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/jobs/runner.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAiC,KAAK,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAClG,OAAO,KAAK,EAAE,cAAc,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAEpE,OAAO,EAAyB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEtD,iHAAiH;AACjH,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,QAAQ,CAAC;IAChB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,cAAc,CAAC;IACxB,YAAY,EAAE,mBAAmB,CAAC;IAClC,OAAO,EAAE,cAAc,CAAC;IACxB,wEAAwE;IACxE,eAAe,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACjC;AAED,4GAA4G;AAC5G,wBAAsB,YAAY,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAgCpG;AAYD,mHAAmH;AACnH,wBAAgB,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAE/F;AAyDD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,CAAC;IAChB,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wFAAwF;AACxF,wBAAsB,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAsCrF"}

package/dist/jobs/runner.js

@@ -0,0 +1,194 @@
+// Runs claimed jobs under their charge policy (docs/provider-standard.md §3.2, §5.6).
+//
+// - "on-success": claimed as queued -> its turn (scheduler) -> running -> handler -> output checked ->
+//   output stored, settlement started -> settle -> succeeded | failed | payment_uncertain. A handler
+//   failure or an invalid output settles nothing [S1]. A job that can't start, or finish, before its
+//   payment expires fails with `timeout`, also uncharged [S2].
+// - "upfront": claimed as pending -> settle -> queued (or failed | payment_uncertain). A worker in any
+//   process sharing the store then leases it: running -> handler -> succeeded | failed. Paid before it
+//   waits, so its place survives a restart [Q4]: a job whose worker died is leased again, up to
+//   MAX_ATTEMPTS starts, so the handler may run more than once for one job.
+// Every state change is stored before the next step, so a lost response is always recoverable, and a
+// process that dies mid-settlement leaves a trail recovery.ts can decide from.
+//
+// Settlement has a deadline (settleTimeoutMs). Past it the job is payment_uncertain, but the answer
+// still counts if it comes: a late or raced result is recorded from payment_uncertain, which the spec
+// allows to become queued, succeeded or failed (§5.6). A charged payment is never dropped.
+//
+// An on-success job's deadline is when its payment expires, less room to settle: the close of its
+// expiration ledger by the ledger clock (payments/chain.ts), so a payment the client resent long after
+// signing [Q8] is not worked on past its life. The claim time plus maxTimeoutSeconds bounds it too,
+// and is the deadline when the clock can't answer.
+import { jobError } from "@stellar-agent-services/protocol";
+import { settle } from "../payments/settlement.js";
+import { SETTLE_MARGIN_SECONDS } from "./scheduler.js";
+/** Starts of one upfront job before it fails: a handler that keeps killing its worker is not retried forever. */
+export const MAX_ATTEMPTS = 3;
+/** on-success: resolves when the job is final or payment_uncertain, with the settle response if settled. */
+export async function runOnSuccess(job, run) {
+    const { store, scheduler, service } = run;
+    const deadline = await paymentDeadline(job, run);
+    try {
+        const started = await scheduler.waitForTurn(job, deadline);
+        if (started === undefined) {
+            // Its turn never came, or came just as it gave up: either way it is failed from running.
+            await store.update(job.jobId, "queued", { status: "running" });
+            await fail(store, job.jobId, "timeout", "the job could not start before its payment expired; nothing was charged");
+            return undefined;
+        }
+        const budgetMs = Math.min(deadline - Date.now(), service.timeoutSeconds * 1000);
+        if (budgetMs <= 0) {
+            await fail(store, job.jobId, "timeout", "the job could not start before its payment expired; nothing was charged");
+            return undefined;
+        }
+        const startedAt = Date.now();
+        const work = await doWork(service, job.jobId, run.input, budgetMs);
+        scheduler.record(service.name, Date.now() - startedAt);
+        if ("failure" in work) {
+            await fail(store, job.jobId, work.failure, work.detail);
+            return undefined;
+        }
+        // Stored before settling: if this process dies mid-settlement, recovery finds the output and
+        // marks the job payment_uncertain rather than losing a payment that may have landed [J14].
+        const settling = await store.update(job.jobId, "running", { status: "running", output: work.output, settlingSince: new Date().toISOString() });
+        if (settling === undefined)
+            return undefined;
+        return await settleAndRecord(job, run, "running", (receipt) => ({ status: "succeeded", payment: receipt }));
+    }
+    finally {
+        scheduler.wake(service.name);
+    }
+}
+/**
+ * When an on-success job's work must be done, leaving room to settle before its payment expires [S2]:
+ * the close of its expiration ledger, but no later than maxTimeoutSeconds after the claim.
+ */
+async function paymentDeadline(job, run) {
+    const byClaim = Date.parse(job.createdAt) + Math.max(1, run.requirements.maxTimeoutSeconds - SETTLE_MARGIN_SECONDS) * 1000;
+    const expires = await run.clock?.closesAt(job.claim.expirationLedger).catch(() => undefined);
+    return expires === undefined ? byClaim : Math.min(byClaim, expires - SETTLE_MARGIN_SECONDS * 1000);
+}
+/** upfront: settles, then queues the job for a worker. Resolves once it is queued, failed or payment_uncertain. */
+export function settleUpfront(job, run) {
+    return settleAndRecord(job, run, "pending", (receipt) => ({ status: "queued", payment: receipt }));
+}
+async function settleAndRecord(job, run, from, settled) {
+    const { store, facilitator, payload, requirements, payment, settleTimeoutMs } = run;
+    const attempt = settle(facilitator, payload, requirements, payment.payer);
+    let timer;
+    const late = new Promise((resolve) => (timer = setTimeout(resolve, settleTimeoutMs, undefined)));
+    const settlement = await Promise.race([attempt, late]);
+    clearTimeout(timer);
+    if (settlement === undefined) {
+        const seconds = Math.round(settleTimeoutMs / 1000);
+        await record(job, run, from, { outcome: "uncertain", reason: `the settlement did not answer in ${seconds} s` }, settled);
+        void attempt.then((answer) => (answer.outcome === "uncertain" ? undefined : record(job, run, "payment_uncertain", answer, settled)));
+        return undefined;
+    }
+    return record(job, run, from, settlement, settled);
+}
+/**
+ * Stores a settlement's outcome. If the job is no longer in `from` because recovery already marked it
+ * payment_uncertain, the outcome is recorded from there instead [J13].
+ */
+async function record(job, run, from, settlement, settled) {
+    const { store } = run;
+    const update = async (patch) => (await store.update(job.jobId, from, patch)) ?? (from === "payment_uncertain" ? undefined : await store.update(job.jobId, "payment_uncertain", patch));
+    switch (settlement.outcome) {
+        case "settled": {
+            const stored = await update(settled(settlement.receipt));
+            if (stored?.status === "queued")
+                run.scheduler.wake(run.service.name);
+            return settlement.response;
+        }
+        case "failed":
+            await update(finished({ status: "failed", error: jobError("payment_failed", `the payment was not settled (${settlement.reason}); nothing was charged`) }));
+            return undefined;
+        case "uncertain": {
+            // Decided later from on-chain data, never by charging again [J13]. The output stays withheld [J14].
+            const detail = settlement.transaction === undefined ? { reason: settlement.reason } : { reason: settlement.reason, transaction: settlement.transaction };
+            console.error(`[provider-kit] job ${job.jobId}: settlement uncertain (${settlement.reason}); run reconcile`);
+            await update({ status: "payment_uncertain", settlement: detail });
+            return undefined;
+        }
+    }
+}
+/** Runs a leased upfront job to its end, renewing the lease while the handler works. */
+export async function runLeased(job, context) {
+    const { store, owner, leaseSeconds } = context;
+    const finish = (patch) => store.update(job.jobId, "running", patch, owner);
+    const service = context.services.get(job.service);
+    if (service === undefined) {
+        // Workers lease only the services they run, so this is a bug: leave the paid job for its lease to expire.
+        console.error(`[provider-kit] job ${job.jobId}: leased for "${job.service}", which this process does not run`);
+        return;
+    }
+    if (job.input === undefined) {
+        await finish(finished({ status: "failed", error: jobError("service_failed", "the job's input was lost") }));
+        return;
+    }
+    if ((job.attempts ?? 1) > MAX_ATTEMPTS) {
+        await finish(finished({ status: "failed", error: jobError("service_failed", `the job was interrupted ${MAX_ATTEMPTS} times`) }));
+        return;
+    }
+    const budgetMs = service.timeoutSeconds * 1000;
+    const lost = new AbortController();
+    const renew = setInterval(() => {
+        store.renew(job.jobId, owner, leaseSeconds).then((held) => held || lost.abort(new Error("the lease was lost")), (error) => console.error(`[provider-kit] job ${job.jobId}: the lease could not be renewed:`, error));
+    }, (leaseSeconds * 1000) / 3);
+    renew.unref();
+    try {
+        const startedAt = Date.now();
+        const work = await doWork(service, job.jobId, job.input, budgetMs, lost.signal);
+        context.scheduler.record(service.name, Date.now() - startedAt);
+        // Another worker took the job over: its result counts, this one is dropped.
+        if (lost.signal.aborted)
+            return;
+        if ("failure" in work)
+            await finish(finished({ status: "failed", error: jobError(work.failure, work.detail) }));
+        else
+            await finish(finished({ status: "succeeded", output: work.output }));
+    }
+    finally {
+        clearInterval(renew);
+    }
+}
+/** Runs the handler for at most budgetMs, or until `stop` aborts. The handler's signal aborts with it. */
+async function doWork(service, jobId, input, budgetMs, stop) {
+    const timeout = new AbortController();
+    // A plain timer rather than AbortSignal.timeout(), so tests can drive it with fake timers.
+    const timer = setTimeout(() => timeout.abort(new DOMException("the work took too long", "TimeoutError")), budgetMs);
+    const signal = stop === undefined ? timeout.signal : AbortSignal.any([timeout.signal, stop]);
+    let output;
+    try {
+        output = await Promise.race([
+            service.handler(structuredClone(input), { signal, jobId }),
+            new Promise((_, reject) => {
+                if (signal.aborted)
+                    reject(signal.reason);
+                signal.addEventListener("abort", () => reject(signal.reason), { once: true });
+            }),
+        ]);
+    }
+    catch (error) {
+        if (signal.aborted)
+            return { failure: "timeout", detail: `the work took longer than ${Math.round(budgetMs / 1000)} s` };
+        // The handler's message may contain anything, so it is not passed on [E2].
+        console.error(`[provider-kit] ${service.name} failed:`, error);
+        return { failure: "service_failed", detail: "the service handler failed" };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    if (!service.validateOutput(output))
+        return { failure: "output_invalid", detail: "the output did not match outputSchema" };
+    return { output };
+}
+/** running -> failed. */
+function fail(store, jobId, code, detail) {
+    return store.update(jobId, "running", finished({ status: "failed", error: jobError(code, detail) }));
+}
+function finished(patch) {
+    return { ...patch, finishedAt: new Date().toISOString() };
+}
+//# sourceMappingURL=runner.js.map

package/dist/jobs/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/jobs/runner.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,EAAE;AACF,uGAAuG;AACvG,qGAAqG;AACrG,qGAAqG;AACrG,+DAA+D;AAC/D,uGAAuG;AACvG,uGAAuG;AACvG,gGAAgG;AAChG,4EAA4E;AAC5E,qGAAqG;AACrG,+EAA+E;AAC/E,EAAE;AACF,oGAAoG;AACpG,sGAAsG;AACtG,2FAA2F;AAC3F,EAAE;AACF,kGAAkG;AAClG,uGAAuG;AACvG,oGAAoG;AACpG,mDAAmD;AAEnD,OAAO,EAAE,QAAQ,EAAwC,MAAM,kCAAkC,CAAC;AAMlG,OAAO,EAAE,MAAM,EAAmB,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAkB,MAAM,gBAAgB,CAAC;AAGvE,iHAAiH;AACjH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAiB9B,4GAA4G;AAC5G,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAc,EAAE,GAAY;IAC7D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC3D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,yFAAyF;YACzF,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,yEAAyE,CAAC,CAAC;YACnH,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;QAChF,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,yEAAyE,CAAC,CAAC;YACnH,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACnE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;QACvD,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC/I,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC7C,OAAO,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;YAAS,CAAC;QACT,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe,CAAC,GAAc,EAAE,GAAY;IACzD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,iBAAiB,GAAG,qBAAqB,CAAC,GAAG,IAAI,CAAC;IAC3H,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7F,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,GAAG,qBAAqB,GAAG,IAAI,CAAC,CAAC;AACrG,CAAC;AAED,mHAAmH;AACnH,MAAM,UAAU,aAAa,CAAC,GAAc,EAAE,GAAY;IACxD,OAAO,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AACrG,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,GAAc,EACd,GAAY,EACZ,IAA2B,EAC3B,OAAuI;IAEvI,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,GAAG,CAAC;IACpF,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,KAAiC,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5G,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACvD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;QACnD,MAAM,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,oCAAoC,OAAO,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACzH,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACrI,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,MAAM,CACnB,GAAc,EACd,GAAY,EACZ,IAAiD,EACjD,UAAsB,EACtB,OAAuI;IAEvI,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC;IACtB,MAAM,MAAM,GAAG,KAAK,EAAE,KAAwC,EAAE,EAAE,CAChE,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,mBAAmB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,mBAAmB,EAAE,KAAK,CAAC,CAAC,CAAC;IACzJ,QAAQ,UAAU,CAAC,OAAO,EAAE,CAAC;QAC3B,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ;gBAAE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACtE,OAAO,UAAU,CAAC,QAAQ,CAAC;QAC7B,CAAC;QACD,KAAK,QAAQ;YACX,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,gBAAgB,EAAE,gCAAgC,UAAU,CAAC,MAAM,wBAAwB,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3J,OAAO,SAAS,CAAC;QACnB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,oGAAoG;YACpG,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,CAAC;YACzJ,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,KAAK,2BAA2B,UAAU,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC7G,MAAM,MAAM,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YAClE,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC;AAUD,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAc,EAAE,OAAsB;IACpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAC/C,MAAM,MAAM,GAAG,CAAC,KAAwC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9G,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,0GAA0G;QAC1G,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,KAAK,iBAAiB,GAAG,CAAC,OAAO,oCAAoC,CAAC,CAAC;QAC/G,OAAO;IACT,CAAC;IACD,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,gBAAgB,EAAE,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5G,OAAO;IACT,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,gBAAgB,EAAE,2BAA2B,YAAY,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjI,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,IAAI,CAC9C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,EAC7D,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,KAAK,mCAAmC,EAAE,KAAK,CAAC,CACpG,CAAC;IACJ,CAAC,EAAE,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChF,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;QAC/D,4EAA4E;QAC5E,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAChC,IAAI,SAAS,IAAI,IAAI;YAAE,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;;YAC3G,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5E,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAID,0GAA0G;AAC1G,KAAK,UAAU,MAAM,CAAC,OAAgB,EAAE,KAAa,EAAE,KAAiB,EAAE,QAAgB,EAAE,IAAkB;IAC5G,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,2FAA2F;IAC3F,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,wBAAwB,EAAE,cAAc,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACpH,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7F,IAAI,MAAkB,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAC1B,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAC1D,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,IAAI,MAAM,CAAC,OAAO;oBAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAChF,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,6BAA6B,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACxH,2EAA2E;QAC3E,OAAO,CAAC,KAAK,CAAC,kBAAkB,OAAO,CAAC,IAAI,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/D,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAC7E,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,uCAAuC,EAAE,CAAC;IAC3H,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC;AAED,yBAAyB;AACzB,SAAS,IAAI,CAAC,KAAe,EAAE,KAAa,EAAE,IAAoB,EAAE,MAAc;IAChF,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,QAAQ,CAAuB,KAAY;IAClD,OAAO,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;AAC5D,CAAC"}

package/dist/jobs/scheduler.d.ts

@@ -0,0 +1,66 @@
+import type { Charge, QueuePosition } from "@stellar-agent-services/protocol";
+import type { Service } from "../define-service.js";
+import type { JobStore, StoredJob } from "./store.js";
+export interface LaneConfig {
+    service: string;
+    charge: Charge;
+    concurrency: number;
+    maxQueued: number;
+    /** Longest a new job may expect to wait before it is turned away [Q2]. */
+    maxWaitSeconds: number;
+    /** Seeds the average job duration until jobs finish. */
+    expectedSeconds: number;
+}
+export type Admission = {
+    accepted: true;
+    reservation: Reservation;
+} | {
+    accepted: false;
+    retryAfterSeconds: number;
+};
+/** A place held for a request between admission and its claim. */
+export interface Reservation {
+    /** Safe to call more than once. */
+    release(): void;
+}
+export interface SchedulerOptions {
+    store: JobStore;
+    lanes: readonly LaneConfig[];
+    /** Runs a leased upfront job to its end. Without it, this process runs no upfront jobs (HTTP only). */
+    work?: (job: StoredJob) => Promise<void>;
+    /** This process's worker id, written on its leases. */
+    owner: string;
+    leaseSeconds: number;
+    /** How often to look for work other processes queued or left behind. */
+    pollMs: number;
+}
+export interface Scheduler {
+    admit(service: string): Promise<Admission>;
+    /** Runs fn under the lane's admission lock. The claim runs here, so admission never counts it twice. */
+    exclusive<T>(service: string, fn: () => Promise<T>): Promise<T>;
+    /**
+     * Waits until it is a queued on-success job's turn and starts it (queued → running). Resolves with
+     * the running job, or undefined if the turn did not come before `deadline` (epoch ms).
+     */
+    waitForTurn(job: StoredJob, deadline: number): Promise<StoredJob | undefined>;
+    /** Something changed in the lane (a job queued or finished): start whatever can start now. */
+    wake(service: string): void;
+    /** Resolves the next time a lane wakes or one of this process's jobs finishes: a cue to look again. */
+    changed(): Promise<void>;
+    /** Where a queued job stands [Q5], or undefined once it is no longer queued. */
+    position(job: StoredJob): Promise<QueuePosition | undefined>;
+    /** Feeds a finished job's duration into the wait estimates. */
+    record(service: string, durationMs: number): void;
+    /** Starts the poll that picks up work queued or left behind by other processes. */
+    start(): void;
+    stop(): void;
+}
+/** Room left, after an "on-success" job's work, to settle before the signed payment expires [S2]. */
+export declare const SETTLE_MARGIN_SECONDS = 10;
+/**
+ * The lane for a service. An "on-success" job is settled after its work, before the payment expires
+ * [S2], so it may wait only as long as leaves room for the work and the settlement.
+ */
+export declare function laneFor(service: Service, maxTimeoutSeconds: number): LaneConfig;
+export declare function scheduler(options: SchedulerOptions): Scheduler;
+//# sourceMappingURL=scheduler.d.ts.map

package/dist/jobs/scheduler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/jobs/scheduler.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEtD,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,EAAE,IAAI,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,QAAQ,EAAE,KAAK,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtH,kEAAkE;AAClE,MAAM,WAAW,WAAW;IAC1B,mCAAmC;IACnC,OAAO,IAAI,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,QAAQ,CAAC;IAChB,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7B,uGAAuG;IACvG,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3C,wGAAwG;IACxG,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAChE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IAC9E,8FAA8F;IAC9F,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,uGAAuG;IACvG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;IAC7D,+DAA+D;IAC/D,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,mFAAmF;IACnF,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,IAAI,CAAC;CACd;AASD,qGAAqG;AACrG,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAExC;;;GAGG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,GAAG,UAAU,CAQ/E;AAmBD,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,SAAS,CAyJ9D"}