@elisym/cli 0.29.0 → 0.31.0

package/dist/index.js

@@ -2,7 +2,7 @@
 import { ReadableStream } from 'node:stream/web';
 import { readFileSync, existsSync, readdirSync, statSync, renameSync, chmodSync, mkdirSync, writeFileSync, appendFileSync, unlinkSync, rmdirSync, realpathSync } from 'node:fs';
 import { dirname, join, resolve, basename, extname, relative, sep } from 'node:path';
-import { SolanaPaymentStrategy, validateAgentName, RELAYS, ElisymIdentity, splAssetsForNetwork, formatSol, formatAssetAmount, signerFromSecretKeyBase58, resolveUsdcAsset, ElisymClient, KIND_EXTERNAL_IDENTITIES, POLICY_D_TAG_PREFIX, KIND_LONG_FORM_ARTICLE, jobRequestKind, DEFAULT_KIND_OFFSET, toDTag, DEFAULTS, createBlossomTransport, generateSolanaWallet, verifyAgentIdentities, getProtocolConfig, getProtocolProgramId, calculateProtocolFee, LIMITS, makeCensor, DEFAULT_REDACT_PATHS, POLICY_T_TAG, SESSION_ID_REGEX, createSlidingWindowLimiter, utf8ByteLength, readAcceptedTransports, resolveDelegationAsset, MAX_PROOF_TTL_SECS, PROOF_CLOCK_SKEW_SECS, verifyDelegationAuthProof, deriveOwnerDelegationAta, getDelegation, buildDelegatedTransfer, buildSignedPull, sendConfirmToTerminal, parseDelegatedPayment, confirmPullToTerminal, isDefinitelyUnpaid, decodeJobPayload, BoundedSet, KIND_JOB_FEEDBACK, encodeSecretKeyBase58, GITHUB_USERNAME_REGEX, X_USERNAME_REGEX, normalizeNip05Identifier, splitNip05Identifier, NATIVE_SOL, GIST_ID_REGEX, TWEET_ID_REGEX } from '@elisym/sdk';
+import { DEFAULTS, SolanaPaymentStrategy, validateAgentName, RELAYS, ElisymIdentity, splAssetsForNetwork, formatSol, formatAssetAmount, signerFromSecretKeyBase58, resolveUsdcAsset, ElisymClient, KIND_EXTERNAL_IDENTITIES, POLICY_D_TAG_PREFIX, KIND_LONG_FORM_ARTICLE, jobRequestKind, DEFAULT_KIND_OFFSET, toDTag, createBlossomTransport, generateSolanaWallet, verifyAgentIdentities, getProtocolConfig, getProtocolProgramId, calculateProtocolFee, LIMITS, makeCensor, DEFAULT_REDACT_PATHS, POLICY_T_TAG, SESSION_ID_REGEX, createSlidingWindowLimiter, utf8ByteLength, readAcceptedTransports, resolveDelegationAsset, MAX_PROOF_TTL_SECS, PROOF_CLOCK_SKEW_SECS, verifyDelegationAuthProof, deriveOwnerDelegationAta, getDelegation, buildDelegatedTransfer, buildSignedPull, sendConfirmToTerminal, parseDelegatedPayment, confirmPullToTerminal, isDefinitelyUnpaid, decodeJobPayload, BoundedSet, KIND_JOB_FEEDBACK, encodeSecretKeyBase58, GITHUB_USERNAME_REGEX, X_USERNAME_REGEX, normalizeNip05Identifier, splitNip05Identifier, NATIVE_SOL, GIST_ID_REGEX, TWEET_ID_REGEX } from '@elisym/sdk';
 import { ElisymYamlSchema, resolveInHome, resolveInProject, createAgentDir, writeYamlInitial, writeExampleSkillTemplate, writeSecrets, listAgents, loadAgent, writeYaml, agentPaths, readMediaCache, loadPoliciesFromDir, ensureGitignoreHasIrohEntry, ensureGitignoreHasX402Entries, ensureGitignoreHasSessionsEntry, ensureGitignoreHasDelegationNoncesEntry, readAgentPublic, lookupCachedUrl, newCacheEntry, writeMediaCache } from '@elisym/sdk/agent-store';
 import { isAddress, createSolanaRpc, address, signature } from '@solana/kit';
 import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools';
@@ -1499,6 +1499,8 @@ var init_init = __esm({
 var MAX_CONCURRENT_JOBS = 10;
 var RECOVERY_MAX_RETRIES = 5;
 var RECOVERY_INTERVAL_SECS = 60;
+var MAX_PAID_AGE_MS = 24 * 60 * 60 * 1e3;
+var LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
 var WATCHDOG_PROBE_INTERVAL_MS = 5 * 60 * 1e3;
 var WATCHDOG_PROBE_TIMEOUT_MS = 1e4;
 var WATCHDOG_SELF_PING_INTERVAL_MS = 10 * 60 * 1e3;
@@ -3283,8 +3285,15 @@ var VALID_TRANSITIONS = {
   delivered: [],
   failed: []
 };
+var MAX_DOUBLE_SETTLE_WARNINGS = 20;
 var JobLedger = class {
   entries = /* @__PURE__ */ new Map();
+  /**
+   * settlement signature -> the job that consumed it. Derived state, rebuilt
+   * from `entries` on every {@link load} - the entries themselves are the
+   * durable record.
+   */
+  paymentSignatureOwners = /* @__PURE__ */ new Map();
   path;
   /**
    * @param ledgerPath absolute path to the ledger file
@@ -3299,8 +3308,23 @@ var JobLedger = class {
     try {
       const raw = readFileSync(this.path, "utf-8");
       const data = JSON.parse(raw);
+      let unusable = 0;
       for (const [id, entry] of Object.entries(data)) {
-        this.entries.set(id, entry);
+        if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
+          unusable += 1;
+          continue;
+        }
+        const candidate = entry;
+        if (typeof candidate.job_id !== "string" || candidate.job_id.length === 0) {
+          unusable += 1;
+          continue;
+        }
+        this.entries.set(id, candidate);
+      }
+      if (unusable > 0) {
+        console.warn(
+          `  ! Ledger load warning: skipped ${unusable} unusable ${unusable === 1 ? "entry" : "entries"} in ${this.path} - not an object, or carrying no job id. They cannot be routed or recovered, and the next write will not preserve them: copy the file before restarting if its history matters.`
+        );
       }
     } catch (e) {
       if (e?.code !== "ENOENT") {
@@ -3313,6 +3337,7 @@ var JobLedger = class {
         }
       }
     }
+    this.indexPaymentSignatures();
   }
   flush() {
     const dir = dirname(this.path);
@@ -3320,8 +3345,8 @@ var JobLedger = class {
     const obj = Object.fromEntries(this.entries);
     const tmp = this.path + ".tmp";
     writeFileSync(tmp, JSON.stringify(obj, null, 2), { mode: LEDGER_FILE_MODE });
+    chmodSync(tmp, LEDGER_FILE_MODE);
     renameSync(tmp, this.path);
-    chmodSync(this.path, LEDGER_FILE_MODE);
   }
   recordPaid(entry) {
     if (this.entries.has(entry.job_id)) {
@@ -3376,6 +3401,121 @@ var JobLedger = class {
       this.flush();
     }
   }
+  /**
+   * Build the settlement-signature index from the loaded entries. A signature
+   * recorded on two entries is the on-disk fingerprint of a double settle -
+   * shout about it rather than silently picking a winner, but once per SIGNATURE
+   * and capped overall: a corrupt ledger repeating one signature across
+   * thousands of entries buried every other startup line under ~30 000
+   * identical warnings. The suppressed count is reported at the end.
+   */
+  indexPaymentSignatures() {
+    const warnedSignatures = /* @__PURE__ */ new Set();
+    let suppressed = 0;
+    for (const entry of this.entries.values()) {
+      const paymentSignature = entry.payment_signature;
+      if (typeof paymentSignature !== "string" || paymentSignature.length === 0) {
+        continue;
+      }
+      const owner = this.paymentSignatureOwners.get(paymentSignature);
+      if (owner !== void 0 && owner !== entry.job_id) {
+        if (warnedSignatures.has(paymentSignature)) {
+          continue;
+        }
+        warnedSignatures.add(paymentSignature);
+        if (warnedSignatures.size > MAX_DOUBLE_SETTLE_WARNINGS) {
+          suppressed += 1;
+          continue;
+        }
+        console.warn(
+          `  ! DOUBLE SETTLE in the ledger: on-chain settlement ${paymentSignature} is recorded for BOTH job ${owner} and job ${entry.job_id}. One transaction must settle only one job - audit both before trusting this agent's payment history.`
+        );
+        continue;
+      }
+      this.paymentSignatureOwners.set(paymentSignature, entry.job_id);
+    }
+    if (suppressed > 0) {
+      console.warn(
+        `  ! DOUBLE SETTLE: ${suppressed} further duplicated settlement signature(s) not listed. This ledger is corrupt - audit it in full.`
+      );
+    }
+  }
+  /** The job that consumed `paymentSignature`, or undefined when unclaimed. */
+  paymentSignatureOwner(paymentSignature) {
+    return this.paymentSignatureOwners.get(paymentSignature);
+  }
+  /**
+   * Bind an on-chain settlement signature to exactly one job - the whole of
+   * "one transaction settles one job" for the FLAT paid path. It exists because
+   * the SDK verifier is stateless by contract (see
+   * `PaymentStrategy.verifyPayment`), so one transfer carrying N job references
+   * verifies for all N and only the provider can pick the single job it settles.
+   *
+   * The exactly-once property comes from this method being SYNCHRONOUS: index
+   * read and write-back happen in one uninterrupted turn of the event loop, so
+   * N concurrent jobs presenting the same signature serialize and exactly one
+   * wins - the same shape as {@link UsedNonceStore}'s `has` -> `markUsed` pair.
+   * Never make this async.
+   *
+   * SCOPE: one ledger file, i.e. one agent directory, in one process. Two
+   * `elisym start` processes sharing a directory are deliberately not
+   * serialized against each other (no cross-process lock) - run one per agent
+   * directory. Separate directories are separate providers with separate
+   * wallets, so they have nothing to de-duplicate in common.
+   *
+   * A re-claim by the SAME job always succeeds: live re-confirmation and crash
+   * recovery re-verifying its own payment must both keep working.
+   *
+   * A job that claims a SECOND, DIFFERENT signature RELEASES the first one's
+   * index mark, because the entry carries exactly one `payment_signature` and
+   * the index must not outlive the entry field that justifies it - the prune
+   * path relies on "every mark corresponds to a persisted `payment_signature`".
+   * Releasing it is safe, not generous: a second claim is only reachable while
+   * the job is still unconfirmed (`net_amount` unset, which is the only state
+   * that re-runs verification), so the released signature settled nothing and
+   * is exactly as unowned as it was before this job ever looked at it.
+   *
+   * That release is guarded on OWNERSHIP, and so is its rollback. On a ledger
+   * that already double-settles (two entries carrying one signature -
+   * `indexPaymentSignatures` warns and keeps the first as owner), this entry's
+   * `payment_signature` can name a mark another job holds. Releasing it
+   * unguarded would free a settlement the rightful owner still needs, and
+   * re-assigning it unguarded on a failed flush would hand that owner's mark to
+   * this job - refusing the owner its own settlement until the next restart.
+   */
+  claimPaymentSignature(paymentSignature, jobId) {
+    const entry = this.entries.get(jobId);
+    if (entry === void 0) {
+      return "unknown-job";
+    }
+    const owner = this.paymentSignatureOwners.get(paymentSignature);
+    if (owner !== void 0 && owner !== jobId) {
+      return "consumed-by-other";
+    }
+    const previousSignature = entry.payment_signature;
+    const previousOwner = owner;
+    const releasedSignature = typeof previousSignature === "string" && previousSignature !== paymentSignature && this.paymentSignatureOwners.get(previousSignature) === jobId ? previousSignature : void 0;
+    this.paymentSignatureOwners.set(paymentSignature, jobId);
+    if (releasedSignature !== void 0) {
+      this.paymentSignatureOwners.delete(releasedSignature);
+    }
+    entry.payment_signature = paymentSignature;
+    try {
+      this.flush();
+    } catch {
+      entry.payment_signature = previousSignature;
+      if (previousOwner === void 0) {
+        this.paymentSignatureOwners.delete(paymentSignature);
+      } else {
+        this.paymentSignatureOwners.set(paymentSignature, previousOwner);
+      }
+      if (releasedSignature !== void 0) {
+        this.paymentSignatureOwners.set(releasedSignature, jobId);
+      }
+      return "not-persisted";
+    }
+    return "claimed";
+  }
   /** Attempt a state transition. Returns the entry if valid, undefined otherwise. */
   transition(jobId, to) {
     const entry = this.entries.get(jobId);
@@ -3448,10 +3588,6 @@ var JobLedger = class {
   allEntries() {
     return [...this.entries.values()];
   }
-  /** Remove old delivered/failed entries (default: 7 days). */
-  gc(maxAgeSecs = 7 * 24 * 60 * 60) {
-    this.pruneOldEntries(maxAgeSecs * 1e3);
-  }
   /**
    * Drop terminal entries (`delivered` / `failed`) whose `created_at`
    * predates `now - retentionMs`. Stuck non-terminal entries are never
@@ -3462,18 +3598,48 @@ var JobLedger = class {
    */
   pruneOldEntries(retentionMs) {
     const cutoff = Math.floor(Date.now() / 1e3) - Math.floor(retentionMs / 1e3);
+    const prunedSignatures = /* @__PURE__ */ new Set();
     let deleted = 0;
     for (const [id, entry] of this.entries) {
       if ((entry.status === "delivered" || entry.status === "failed") && entry.created_at < cutoff) {
         this.entries.delete(id);
+        if (typeof entry.payment_signature === "string" && entry.payment_signature.length > 0) {
+          prunedSignatures.add(entry.payment_signature);
+        }
         deleted += 1;
       }
     }
+    if (prunedSignatures.size > 0) {
+      this.releasePrunedPaymentSignatures(prunedSignatures);
+    }
     if (deleted > 0) {
       this.flush();
     }
     return deleted;
   }
+  /**
+   * Release the index marks of PRUNED entries, so the index cannot outgrow the
+   * ledger. Every mark corresponds to a persisted `payment_signature` (a claim
+   * whose flush failed rolls both back), so the entries are the whole story.
+   * Safe: `LEDGER_RETENTION_MS` is far past Solana's ~2-3 day history horizon,
+   * so a pruned signature can no longer verify for any job. A signature still
+   * carried by a SURVIVING entry (the fingerprint of a double settle) is kept
+   * whoever owns it - pruning one side must never hand the transaction to a
+   * third job.
+   */
+  releasePrunedPaymentSignatures(prunedSignatures) {
+    const survivingSignatures = /* @__PURE__ */ new Set();
+    for (const entry of this.entries.values()) {
+      if (typeof entry.payment_signature === "string") {
+        survivingSignatures.add(entry.payment_signature);
+      }
+    }
+    for (const paymentSignature of prunedSignatures) {
+      if (!survivingSignatures.has(paymentSignature)) {
+        this.paymentSignatureOwners.delete(paymentSignature);
+      }
+    }
+  }
 };
 var NONCE_STORE_MAX_ENTRIES = 1e4;
 var UsedNonceStore = class {
@@ -3700,6 +3866,664 @@ var MIME_BY_EXT = {
 function mimeFromPath(path) {
   return MIME_BY_EXT[extname(path).toLowerCase()] ?? "application/octet-stream";
 }
+var REFERENCE_SCAN_WINDOW = DEFAULTS.VERIFY_SIGNATURE_LIMIT;
+var REFERENCE_SCAN_LIST_ATTEMPTS = 3;
+var REFERENCE_SCAN_LIST_RETRY_DELAY_MS = 400;
+var REFERENCE_SCAN_VERIFY_RETRIES = REFERENCE_SCAN_LIST_ATTEMPTS;
+var OWN_SETTLEMENT_VERIFY_RETRIES = REFERENCE_SCAN_VERIFY_RETRIES + 2;
+var REFERENCE_SCAN_DEADLINE_MS = 3e4;
+var CLUSTER_GENESIS_HASHES = {
+  mainnet: "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d",
+  devnet: "EtWTRABZaYq6iMfeYKouRu166VU2xqa1wcaWoxPkrZBG"
+};
+var ADDRESS_HISTORY_PROBE_ADDRESS = "11111111111111111111111111111111";
+var RECOVERY_DEFER_BACKOFF_TICKS = [1, 2, 5, 15, 30, 60];
+var RECOVERY_DEFER_JITTER_TICKS = 0.5;
+var TERMINAL_CONFIRMATION_MIN_RUNG = 3;
+var RECOVERY_SCAN_BUDGET_DIVISOR = 2;
+function recoveryScanBudgetPerTick(maxConcurrentJobs) {
+  return Math.max(1, Math.floor(maxConcurrentJobs / RECOVERY_SCAN_BUDGET_DIVISOR));
+}
+function recoveryDeferDelayMs(attempts, recoveryIntervalSecs, jitter) {
+  const rung = Math.min(attempts, RECOVERY_DEFER_BACKOFF_TICKS.length) - 1;
+  const ticks = RECOVERY_DEFER_BACKOFF_TICKS[rung] ?? 1;
+  const intervalMs = recoveryIntervalSecs * 1e3;
+  const jitterMs = jitter * RECOVERY_DEFER_JITTER_TICKS * intervalMs;
+  return ticks * intervalMs + jitterMs;
+}
+function waitMs(ms) {
+  return new Promise((resolve4) => {
+    setTimeout(resolve4, ms);
+  });
+}
+function needsPaymentScan(entry, delegated) {
+  return entry.status === "paid" && !entry.net_amount && entry.payment_request !== void 0 && !delegated;
+}
+function claimRefusalSentence(claim) {
+  switch (claim) {
+    case "consumed-by-other":
+      return "the transaction carrying this job reference was already consumed by another job, so it is not attributable here";
+    case "not-persisted":
+      return "the claim on that settlement could not be written to disk, so the payment was refused rather than risk delivering twice for one transfer";
+    case "unknown-job":
+      return "this job has no ledger entry, so the claim had nowhere durable to live";
+  }
+}
+var UNNAMED_SETTLEMENT_SENTENCE = "the payment verified without a settlement signature, so it cannot be bound to one job";
+var RecoveryDeferrals = class {
+  /**
+   * `recoveryIntervalSecs` is read through a function because the backoff
+   * ladder is counted in TICKS and the runtime owns the tick length.
+   * `random` returns `[0, 1)`; injected so the jitter is testable.
+   */
+  constructor(recoveryIntervalSecs, random = Math.random) {
+    this.recoveryIntervalSecs = recoveryIntervalSecs;
+    this.random = random;
+  }
+  deferrals = /* @__PURE__ */ new Map();
+  /** How many entries are currently held back. */
+  get size() {
+    return this.deferrals.size;
+  }
+  /** This entry's deferral state, or undefined when it is not held back. */
+  peek(jobId) {
+    return this.deferrals.get(jobId);
+  }
+  /**
+   * Record that a recovery tick ended INCONCLUSIVE for this entry and push its
+   * next attempt out by the backoff ladder.
+   *
+   * A deferral is cheap to decide and expensive to repeat: a full reference
+   * scan (an RPC listing plus a transaction fetch per candidate) holding a
+   * `p-limit` slot and a queue slot shared with live intake. Repeating that
+   * every 60s for 24h is how a handful of stuck entries turns into "Server
+   * overloaded" for paying customers. Retries are still NOT burned - a deferral
+   * is never evidence against the customer - the wait simply lengthens.
+   *
+   * `sawNoPayment` carries forward the one observation that can ever become
+   * terminal, and is cleared by any other outcome - that is what "two
+   * CONSECUTIVE clean empty scans" means.
+   */
+  note(jobId, sawNoPayment = false) {
+    const previous = this.deferrals.get(jobId);
+    const attempts = (previous?.attempts ?? 0) + 1;
+    const delayMs = recoveryDeferDelayMs(
+      sawNoPayment ? Math.max(attempts, TERMINAL_CONFIRMATION_MIN_RUNG) : attempts,
+      this.recoveryIntervalSecs(),
+      this.random() * 2 - 1
+    );
+    this.deferrals.set(jobId, {
+      attempts,
+      nextAttemptAt: Date.now() + delayMs,
+      firstDeferredAt: previous?.firstDeferredAt ?? Date.now(),
+      sawNoPayment
+    });
+  }
+  /**
+   * Record a scan that found an empty reference INSIDE the payment window this
+   * provider itself advertised.
+   *
+   * Deliberately off the backoff ladder. The ladder counts inconclusive looks -
+   * moments where the chain, the RPC or our own disk let us down - and "the
+   * customer still has eight of their ten minutes left" is none of those: it is
+   * the expected state of an ordinary job that nobody has paid yet. Counting it
+   * cost twice over. A customer whose wallet confirms four minutes in was not
+   * looked at again until minute nine, because the ladder had already climbed to
+   * its 5-tick rung while they were still entitled to pay. And a job nobody ever
+   * pays reached its expiry already parked on a 15- or 30-tick rung, so the two
+   * consecutive scans that close it landed the better part of an hour later -
+   * holding a `paid` entry, and its recovery slot, for all of it.
+   *
+   * So: hold the entry back for one tick, leave `attempts` where it is, and let
+   * the ladder start climbing only once the window is actually over. The cost is
+   * one `getSignaturesForAddress` per tick per unpaid job for the length of the
+   * window (ten minutes by default) - an empty scan fetches no transactions -
+   * and the per-tick scan budget still bounds how many run at once.
+   *
+   * `sawNoPayment` is cleared for the same reason it is cleared on any other
+   * inconclusive outcome: this scan must never count toward the two consecutive
+   * sightings that fail a job, because it was taken before the customer's
+   * deadline.
+   */
+  noteAwaitingWindow(jobId) {
+    const previous = this.deferrals.get(jobId);
+    const delayMs = recoveryDeferDelayMs(1, this.recoveryIntervalSecs(), this.random() * 2 - 1);
+    this.deferrals.set(jobId, {
+      attempts: previous?.attempts ?? 0,
+      nextAttemptAt: Date.now() + delayMs,
+      firstDeferredAt: previous?.firstDeferredAt ?? Date.now(),
+      sawNoPayment: false
+    });
+  }
+  /**
+   * Park an entry whose payment re-scan did not fit in this tick's scan budget.
+   *
+   * NOT a deferral: nothing was attempted, so the ladder does not advance and
+   * the `sawNoPayment` observation is neither set nor cleared. The wait is
+   * deliberately SUB-TICK, so a parked entry is due again by the next tick
+   * rather than pushed minutes into the future for a queue that was merely
+   * busy. What spreads a restart's backlog across ticks is the per-tick budget
+   * itself (see {@link recoveryScanBudgetPerTick}), not this wait; the jitter
+   * only stops a whole parked batch from becoming due on the same millisecond.
+   */
+  parkForCapacity(jobId) {
+    const previous = this.deferrals.get(jobId);
+    const jitteredMs = this.random() * this.recoveryIntervalSecs() * 1e3;
+    this.deferrals.set(jobId, {
+      attempts: previous?.attempts ?? 0,
+      nextAttemptAt: Date.now() + jitteredMs,
+      firstDeferredAt: previous?.firstDeferredAt ?? Date.now(),
+      sawNoPayment: previous?.sawNoPayment ?? false
+    });
+  }
+  /**
+   * Forget this entry's deferral state entirely. Called when the entry stops
+   * being deferred at all - its payment confirmed - so the backlog summary's
+   * "oldest deferred" cannot keep ageing on an entry nobody is holding back.
+   */
+  clear(jobId) {
+    this.deferrals.delete(jobId);
+  }
+  /** Whether the LAST attempt on this entry saw a complete, empty reference history. */
+  sawNoPayment(jobId) {
+    return this.deferrals.get(jobId)?.sawNoPayment === true;
+  }
+  /** Whether this entry is still inside its deferral backoff window. */
+  isDeferred(jobId) {
+    const deferral = this.deferrals.get(jobId);
+    return deferral !== void 0 && Date.now() < deferral.nextAttemptAt;
+  }
+  /**
+   * Drop the state of entries that have left the pending set (delivered,
+   * failed, pruned), keeping this map bounded by the pending set rather than by
+   * the ledger's whole history.
+   */
+  sweep(stillPending) {
+    if (this.deferrals.size === 0) {
+      return;
+    }
+    for (const jobId of [...this.deferrals.keys()]) {
+      if (!stillPending.has(jobId)) {
+        this.deferrals.delete(jobId);
+      }
+    }
+  }
+  /**
+   * One line per tick summing up the backlog, or `null` when there is nothing
+   * to say. A pinned entry otherwise produces the same per-entry line as a
+   * flaky RPC, roughly thirty times a day, and an operator has no way to tell
+   * "one job is stuck" from "the chain is unreachable". The oldest age is the
+   * number that matters: it says how close the backlog is to the 24h cutoff.
+   */
+  summaryLine() {
+    if (this.deferrals.size === 0) {
+      return null;
+    }
+    let oldestFirstDeferredAt = Number.POSITIVE_INFINITY;
+    for (const deferral of this.deferrals.values()) {
+      oldestFirstDeferredAt = Math.min(oldestFirstDeferredAt, deferral.firstDeferredAt);
+    }
+    const oldestMinutes = Math.floor((Date.now() - oldestFirstDeferredAt) / 6e4);
+    const noun = this.deferrals.size === 1 ? "job is" : "jobs are";
+    return `Recovery: ${this.deferrals.size} ${noun} deferred awaiting payment confirmation (oldest deferred ${oldestMinutes}m ago; the 24h cutoff closes them).`;
+  }
+};
+var PaymentRecovery = class {
+  constructor(ledger, network, fetchProtocolConfig, strategy) {
+    this.ledger = ledger;
+    this.network = network;
+    this.fetchProtocolConfig = fetchProtocolConfig;
+    this.strategy = strategy;
+  }
+  /**
+   * Whether the RPC endpoint has earned the right to end a paying customer's
+   * job. Process-wide and cached, because it is a property of the URL this
+   * process was started with, not of any one scan.
+   *
+   * Only the two STABLE answers are cached. `wrong-cluster` cannot change
+   * without a restart, and `sound` is not worth re-proving on every verdict.
+   * Everything else (an unreachable endpoint, a throttled probe, a node that
+   * refuses address history) stays `unchecked` and is re-probed next tick: an
+   * endpoint that is merely down must not be condemned permanently, and one
+   * that never answers simply never unlocks the verdict.
+   */
+  endpointSoundness = "unchecked";
+  /**
+   * The probe currently in flight, so several entries reaching a verdict on the
+   * same tick share ONE pair of RPC calls instead of each making its own before
+   * the first has had a chance to cache its answer.
+   */
+  endpointProbe;
+  /** Reason tags already shouted about, so the loud line is loud exactly once each. */
+  warnedEndpointReasons = /* @__PURE__ */ new Set();
+  /** Shout about an endpoint we will not end jobs on - once per distinct reason. */
+  warnUntrustedEndpoint(reason, log, detail) {
+    if (this.warnedEndpointReasons.has(reason)) {
+      return;
+    }
+    this.warnedEndpointReasons.add(reason);
+    log(
+      `  ! WARNING: the Solana RPC endpoint cannot be trusted to decide that a customer did not pay: ${detail} Paid jobs whose payment cannot be found will NOT be closed as unpaid - they stay recoverable until the 24h cutoff and then fail as "the agent did not recover". Check SOLANA_RPC_URL and that it serves the ${this.network} cluster with transaction history enabled.`
+    );
+  }
+  /**
+   * Whether this endpoint may be believed when it says a reference is empty.
+   *
+   * The terminal no-payment verdict is the one place the provider tells a paying
+   * customer their money bought nothing, and every input to it comes from a
+   * single RPC client named by an environment variable nothing checks. Two
+   * consecutive empty listings from the WRONG CLUSTER, or from a node that does
+   * not serve address history at all, are two copies of the same meaningless
+   * answer - and would fail every paying customer of that agent.
+   *
+   * So before the verdict is allowed at all: the endpoint must report the
+   * genesis hash of the cluster this agent is configured for, and must answer a
+   * `getSignaturesForAddress` query. Neither costs anything after the first
+   * success - the answer is cached for the process.
+   *
+   * Fails SAFE: anything short of both checks passing means "keep deferring",
+   * which ends at the 24h cutoff as "the agent did not recover" rather than as a
+   * false accusation. An operator running against a local validator (whose
+   * genesis is its own) therefore never gets the fast verdict; that is the
+   * intended trade.
+   */
+  endpointMayIssueTerminalVerdict(rpc, log) {
+    if (this.endpointSoundness === "sound") {
+      return Promise.resolve(true);
+    }
+    if (this.endpointSoundness === "wrong-cluster") {
+      return Promise.resolve(false);
+    }
+    if (this.endpointProbe === void 0) {
+      const probe = this.probeEndpointSoundness(rpc, log);
+      this.endpointProbe = probe;
+      const release = () => {
+        if (this.endpointProbe === probe) {
+          this.endpointProbe = void 0;
+        }
+      };
+      void probe.then(release, release);
+    }
+    return this.endpointProbe;
+  }
+  /** The two RPC calls behind {@link endpointMayIssueTerminalVerdict}. */
+  async probeEndpointSoundness(rpc, log) {
+    try {
+      const expectedGenesisHash = CLUSTER_GENESIS_HASHES[this.network];
+      let genesisHash;
+      try {
+        genesisHash = await rpc.getGenesisHash().send();
+      } catch (e) {
+        this.warnUntrustedEndpoint(
+          "unreachable",
+          log,
+          `it did not answer getGenesisHash (${e?.message ?? "unknown error"}).`
+        );
+        return false;
+      }
+      if (genesisHash !== expectedGenesisHash) {
+        this.endpointSoundness = "wrong-cluster";
+        this.warnUntrustedEndpoint(
+          "wrong-cluster",
+          log,
+          `it reports genesis ${genesisHash}, but this agent is configured for ${this.network}, whose genesis is ${expectedGenesisHash}.`
+        );
+        return false;
+      }
+      try {
+        await rpc.getSignaturesForAddress(address(ADDRESS_HISTORY_PROBE_ADDRESS), {
+          limit: 1,
+          commitment: "confirmed"
+        }).send();
+      } catch (e) {
+        this.warnUntrustedEndpoint(
+          "no-history",
+          log,
+          `it is on the right cluster but did not answer an address-history query (${e?.message ?? "unknown error"}).`
+        );
+        return false;
+      }
+      this.endpointSoundness = "sound";
+      return true;
+    } catch (e) {
+      this.warnUntrustedEndpoint(
+        "unreachable",
+        log,
+        `the endpoint check itself failed (${e?.message ?? "unknown error"}).`
+      );
+      return false;
+    }
+  }
+  /**
+   * Bind a verified settlement signature to this job, exactly once - see
+   * `JobLedger.claimPaymentSignature`. Anything but `claimed` means the job is
+   * NOT paid here and now; none of the three refusals is on its own evidence
+   * that the customer failed to pay. Diagnostics log the FULL signature, both
+   * job ids and the customer pubkey - all public values, and an operator
+   * chasing a disputed payment should not have to match truncated prefixes.
+   */
+  claimSettlementSignature(txSignature, job, log) {
+    const owner = this.ledger.paymentSignatureOwner(txSignature);
+    const outcome = this.ledger.claimPaymentSignature(txSignature, job.jobId);
+    if (outcome === "consumed-by-other") {
+      log(
+        `[${job.jobId.slice(0, 8)}] REFUSED: settlement ${txSignature} was already consumed by job ${owner ?? "unknown"}. One transaction settles one job, so it is not attributable to job ${job.jobId} (customer ${job.customerId}).`
+      );
+    } else if (outcome === "not-persisted") {
+      log(
+        `[${job.jobId.slice(0, 8)}] REFUSED: could not WRITE the claim on settlement ${txSignature} for job ${job.jobId} (customer ${job.customerId}) - refusing the payment rather than risk delivering twice for one transfer. Check disk space and permissions at the agent directory; the job retries on the next recovery tick.`
+      );
+    } else if (outcome === "unknown-job") {
+      log(
+        `[${job.jobId.slice(0, 8)}] REFUSED: job ${job.jobId} (customer ${job.customerId}) has no ledger entry, so the claim on settlement ${txSignature} has nowhere durable to live. This is a provider bug, not a disk problem - every paid job is recorded before payment collection. Report it with this log line.`
+      );
+    }
+    return outcome;
+  }
+  /**
+   * List the transactions that could possibly be a payment for this request:
+   * one `getSignaturesForAddress` window against the reference key, newest
+   * first, minus the transactions that FAILED on chain. A failed transaction
+   * moved no money, so it is neither a payment nor evidence that anyone touched
+   * this reference - counting it would let ~5000 lamports of deliberately
+   * failing transaction (or an honest customer whose transfer simply failed)
+   * turn a job that should close in a minute into a 24h wait.
+   *
+   * Listed at `confirmed`, the level the SDK verifier reads transactions at.
+   * The RPC default `finalized` would hide a confirmed-but-not-yet-finalized
+   * payment from the one check that decides whether anything is there.
+   *
+   * RETRIED (`REFERENCE_SCAN_LIST_ATTEMPTS`): the terminal "nobody paid" verdict
+   * rests entirely on this one call, and the public RPC is known to throttle and
+   * lag this exact index. A single un-retried shot deciding whether a customer
+   * loses their money is not a trade worth making; a false deferral costs
+   * latency, a false "unpaid" destroys money. Retrying stops at the scan
+   * `deadline`: past it the scan is going to return inconclusive anyway, and
+   * the attempts left would only hold a shared slot.
+   *
+   * `windowFull` reports whether the RAW listing came back at the window size.
+   * A full window means the history is TRUNCATED - a payment can be hiding
+   * behind the newer transactions - so the caller must not read "nothing
+   * verified" as "nobody paid". Measured before the failed-transaction filter,
+   * because the limit applies to the raw page.
+   *
+   * An error is never "the customer did not pay" - the caller keeps the job
+   * recoverable.
+   */
+  async listReferenceCandidates(reference, rpc, deadline) {
+    let lastMessage = "unknown error";
+    let attempts = 0;
+    for (let attempt = 0; attempt < REFERENCE_SCAN_LIST_ATTEMPTS; attempt++) {
+      attempts = attempt + 1;
+      try {
+        const listed = await rpc.getSignaturesForAddress(reference, {
+          limit: REFERENCE_SCAN_WINDOW,
+          commitment: "confirmed"
+        }).send();
+        return {
+          signatures: listed.filter((candidate) => !candidate.err).map((candidate) => candidate.signature),
+          windowFull: listed.length >= REFERENCE_SCAN_WINDOW
+        };
+      } catch (e) {
+        lastMessage = e?.message ?? "unknown error";
+        if (Date.now() >= deadline) {
+          break;
+        }
+        if (attempt < REFERENCE_SCAN_LIST_ATTEMPTS - 1) {
+          await waitMs(REFERENCE_SCAN_LIST_RETRY_DELAY_MS);
+        }
+      }
+    }
+    return {
+      error: `could not reach the chain to list the reference after ${attempts} of ${REFERENCE_SCAN_LIST_ATTEMPTS} attempts: ${lastMessage}`
+    };
+  }
+  /**
+   * Provider-side reference scan: list the reference's transactions and verify
+   * the first one this job may actually own.
+   *
+   * The SDK's own reference path returns the newest VERIFYING transaction in
+   * its window and cannot be asked for the next one. Fine for a customer
+   * confirming their own payment; fatal for a provider, because a stranger who
+   * attaches this job's reference to their own newer transfer makes the genuine
+   * payment invisible to that path and the job dies at the 24h cutoff with the
+   * money already taken. Walking the window and skipping the signatures the
+   * ledger gave to OTHER jobs - which only the provider can know - is what
+   * reaches such a payment.
+   *
+   * EXACTLY WHAT THAT RECOVERS, and no more: a payment masked by transactions
+   * THIS ledger has already consumed, lying INSIDE one window. A mask built from
+   * transactions the ledger knows nothing about is not skipped, only walked past
+   * (each is fetched and rejected, which makes the scan inconclusive rather than
+   * terminal); and a mask longer than the window pushes the payment off the page
+   * entirely, where nothing here can see it. Both of those stay recoverable
+   * until the 24h cutoff rather than being wrongly closed, which is the property
+   * that actually protects the customer's money.
+   *
+   * BUDGET: `REFERENCE_SCAN_VERIFY_RETRIES` per candidate and the caller's
+   * `deadline` for the whole scan, after which it returns inconclusive. Without
+   * both, a rate-limited RPC holds a `p-limit` slot shared with live intake for
+   * the SDK default of 10 retries x 3s x a full window - over ten minutes for
+   * one deferred entry.
+   *
+   * TERMINAL ("none") REQUIRES ALL THREE, and the caller adds two more:
+   *   1. the listing SUCCEEDED and came back SHORT of the window, so this is the
+   *      reference's whole history rather than a truncated page. A flood that
+   *      fills the window could be hiding the payment behind it, which is
+   *      exactly how an attacker would manufacture a "nobody paid" verdict;
+   *   2. NO candidate failed to verify. A candidate that did not verify is not
+   *      evidence: the SDK reports "this is not a payment for this request" and
+   *      "I could not reach the chain for this transaction" as the same
+   *      `{verified: false, error}`, and an unreachable RPC must never be
+   *      evidence of non-payment;
+   *   3. no candidate was SKIPPED for belonging to another job. A skip means
+   *      something did touch this reference and we chose not to look at it, so
+   *      the scan saw less than the whole truth.
+   * The caller then requires the payment request's own expiry to have passed and
+   * a second consecutive sighting. Anything else is inconclusive: a false
+   * deferral costs latency, a false "unpaid" destroys the customer's money.
+   *
+   * What "none" therefore means is narrow and honest: after the failed-on-chain
+   * transactions are dropped, the reference's entire history is EMPTY - there
+   * was nothing at all to look at.
+   */
+  async verifyByReferenceScan(reference, rpc, jobId, deadline, verifySignature) {
+    const listed = await this.listReferenceCandidates(reference, rpc, deadline);
+    if ("error" in listed) {
+      return { outcome: "inconclusive", error: listed.error };
+    }
+    let skippedConsumed = false;
+    let unverifiableCandidate;
+    for (const candidate of listed.signatures) {
+      if (Date.now() > deadline) {
+        return { outcome: "inconclusive", error: "the reference scan ran out of time" };
+      }
+      const owner = this.ledger.paymentSignatureOwner(candidate);
+      if (owner !== void 0 && owner !== jobId) {
+        skippedConsumed = true;
+        continue;
+      }
+      const verified = await verifySignature(candidate, REFERENCE_SCAN_VERIFY_RETRIES);
+      if (verified.verified) {
+        return { outcome: "verified", txSignature: candidate };
+      }
+      unverifiableCandidate ??= verified.error ?? "no reason given";
+    }
+    if (listed.windowFull) {
+      return {
+        outcome: "inconclusive",
+        error: `the reference carries at least ${REFERENCE_SCAN_WINDOW} transactions, so its history is truncated and a payment could be hidden behind the flood`
+      };
+    }
+    if (skippedConsumed) {
+      return {
+        outcome: "inconclusive",
+        error: "the reference carries a transaction another job already settled, so this scan did not see the whole picture"
+      };
+    }
+    if (unverifiableCandidate !== void 0) {
+      return {
+        outcome: "inconclusive",
+        error: `the reference carries a transaction that did not verify as a payment for this job (${unverifiableCandidate}) - indistinguishable, in what the verifier reports, from one we simply failed to read`
+      };
+    }
+    return { outcome: "none" };
+  }
+  /**
+   * Re-verify an on-chain payment during crash recovery.
+   *
+   * Two outcomes are terminal, and they mean opposite things. `corrupt-state` is
+   * OUR state being unusable - no amount of waiting repairs it, so the caller
+   * closes the job at once rather than hold it for 24h. `no-payment` is a scan
+   * that saw the reference's WHOLE history and found nothing capable of being a
+   * payment; even that only fails the job once the request's own expiry has
+   * passed AND on a second consecutive sighting (the caller's check), because
+   * one empty listing from a throttling RPC is not worth a customer's money.
+   * Everything else defers: recovery cannot tell a non-paying customer apart
+   * from a shutdown abort, a flaky RPC, or a stranger's transaction carrying
+   * this job's (publicly readable) reference. The 24h `MAX_PAID_AGE_MS` cutoff
+   * bounds the wait and closes the job as "the agent did not recover", never as
+   * a false "the customer did not pay".
+   *
+   * Evidence order matters: a job that already CLAIMED a settlement is
+   * re-verified against that exact signature first, and an empty scan can never
+   * outrank it - the ledger is better evidence than an RPC that has aged the
+   * transaction out of its history window.
+   *
+   * Limitation: Solana transaction data expires after ~2-3 days (recent blockhash
+   * window). If the agent was down longer, a confirmed payment may not be found
+   * on-chain. For mainnet: use monitoring, avoid extended downtime, or configure
+   * an archive RPC via SOLANA_RPC_URL.
+   */
+  async reVerifyPayment(entry, paymentRequestJson, priceSubunits, log, signal) {
+    const shortId = entry.job_id.slice(0, 8);
+    let request;
+    let reference;
+    try {
+      request = JSON.parse(paymentRequestJson);
+      reference = address(request.reference);
+    } catch (e) {
+      log(
+        `[${shortId}] Recovery: the persisted payment request is unusable (${e?.message ?? "unknown error"}) - this is provider-side state, not a chain or customer problem.`
+      );
+      return "corrupt-state";
+    }
+    const deadline = Date.now() + REFERENCE_SCAN_DEADLINE_MS;
+    try {
+      const rpc = createSolanaRpc(getRpcUrl(this.network));
+      const protocolConfig = await this.fetchProtocolConfig();
+      const verify = async (txSignature2, retries) => {
+        const verification = this.strategy.verifyPayment(rpc, request, protocolConfig, {
+          txSignature: txSignature2,
+          retries
+        });
+        if (!signal) {
+          return verification;
+        }
+        let abortHandler;
+        const abortPromise = new Promise((_, reject) => {
+          abortHandler = () => {
+            const err = new Error("The operation was aborted");
+            err.name = "AbortError";
+            reject(err);
+          };
+          if (signal.aborted) {
+            abortHandler();
+            return;
+          }
+          signal.addEventListener("abort", abortHandler, { once: true });
+        });
+        try {
+          return await Promise.race([verification, abortPromise]);
+        } finally {
+          if (abortHandler) {
+            signal.removeEventListener("abort", abortHandler);
+          }
+        }
+      };
+      const ownSignature = entry.payment_signature;
+      let txSignature;
+      if (ownSignature !== void 0) {
+        const own = await verify(ownSignature, OWN_SETTLEMENT_VERIFY_RETRIES);
+        if (own.verified) {
+          txSignature = ownSignature;
+        } else {
+          log(
+            `[${shortId}] Recovery: the settlement this job already claimed (${ownSignature}) no longer verifies (${own.error ?? "unknown"}); falling back to the reference scan.`
+          );
+        }
+      }
+      if (txSignature === void 0) {
+        const scan = await this.verifyByReferenceScan(
+          reference,
+          rpc,
+          entry.job_id,
+          deadline,
+          verify
+        );
+        if (scan.outcome === "none") {
+          if (ownSignature !== void 0) {
+            log(
+              `[${shortId}] Recovery: the reference scan is empty, but this job already owns settlement ${ownSignature} - treating the empty scan as an RPC history gap, not as non-payment. Deferring.`
+            );
+            return "deferred";
+          }
+          const notBefore = terminalVerdictNotBefore(request, entry.created_at);
+          if (Date.now() < notBefore) {
+            const secondsLeft = Math.ceil((notBefore - Date.now()) / 1e3);
+            log(
+              `[${shortId}] Recovery: the reference is still empty, but the payment request this provider issued does not expire for another ${secondsLeft}s - a customer is entitled to the whole window. Waiting out the window.`
+            );
+            return "awaiting-window";
+          }
+          if (!await this.endpointMayIssueTerminalVerdict(rpc, log)) {
+            log(
+              `[${shortId}] Recovery: the reference scan is empty, but this RPC endpoint has not proved it is the ${this.network} cluster and serves address history - refusing to call that non-payment. Deferring.`
+            );
+            return "deferred";
+          }
+          log(
+            `[${shortId}] Recovery: a complete scan of this reference, after the payment request expired, found no payment for this job.`
+          );
+          return "no-payment";
+        }
+        if (scan.outcome === "inconclusive") {
+          log(
+            `[${shortId}] Recovery: payment could not be confirmed (${scan.error}); deferring to the next tick.`
+          );
+          return "deferred";
+        }
+        txSignature = scan.txSignature;
+      }
+      const claim = this.claimSettlementSignature(
+        txSignature,
+        { jobId: entry.job_id, customerId: entry.customer_id },
+        log
+      );
+      if (claim !== "claimed") {
+        log(
+          `[${shortId}] Recovery: ${claimRefusalSentence(claim)}; deferring. The job stays paid and the next tick looks again.`
+        );
+        return "deferred";
+      }
+      const fee = calculateProtocolFee(priceSubunits, protocolConfig.feeBps);
+      const netAmount = priceSubunits - fee;
+      this.ledger.updatePayment(entry.job_id, netAmount);
+      log(`[${shortId}] Recovery: payment re-verified (${netAmount} subunits)`);
+      return "verified";
+    } catch (e) {
+      log(`[${shortId}] Recovery: payment re-verification error: ${e.message}; deferring.`);
+      return "deferred";
+    }
+  }
+};
+function terminalVerdictNotBefore(request, entryCreatedAt) {
+  const requestCreatedAt = Number(request.created_at);
+  const expirySecs = Number(request.expiry_secs);
+  const createdAt = Number.isFinite(requestCreatedAt) && requestCreatedAt > 0 ? requestCreatedAt : entryCreatedAt;
+  const window = Number.isFinite(expirySecs) && expirySecs > 0 ? expirySecs : DEFAULTS.PAYMENT_EXPIRY_SECS;
+  return (createdAt + window) * 1e3;
+}
 var SESSIONS_DIR_NAME = ".sessions";
 var SESSION_MAX_CONCURRENT_JOBS = 2;
 var SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1e3;
@@ -4468,8 +5292,6 @@ var X402PermanentError = class extends Error {
 // src/runtime.ts
 var payment = new SolanaPaymentStrategy();
 var LEDGER_GC_INTERVAL_MS = 60 * 60 * 1e3;
-var LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
-var MAX_PAID_AGE_MS = 24 * 60 * 60 * 1e3;
 var SIG_PATH_TIMEOUT_MS = 60 * 1e3;
 var SESSION_SUMMARIZE_SYSTEM = "Summarize the following conversation between a user and an assistant so the conversation can continue with your summary standing in for the older turns. Preserve facts, decisions, names, numbers, constraints, and open questions. Reply with the summary only.";
 function buildUserRecord(data, fileName) {
@@ -4538,9 +5360,13 @@ var SeedFailedError = class extends Error {
     this.name = "SeedFailedError";
   }
 };
+var RECOVERY_NO_PAYMENT_CUSTOMER_MESSAGE = "Job permanently failed: no payment for this job was found on-chain after the payment request expired.";
+var RECOVERY_UNVERIFIABLE_CUSTOMER_MESSAGE = "Job permanently failed: the provider could not verify payment for this job. If you paid, contact the provider with your transaction signature.";
+var RECOVERY_NO_SKILL_CUSTOMER_MESSAGE = "Job permanently failed: this provider no longer offers a skill for this job. If you paid, contact the provider with your transaction signature.";
+var RECOVERY_INPUT_UNAVAILABLE_CUSTOMER_MESSAGE = "Job permanently failed: the input file for this job could not be retrieved after an agent restart - contact the provider to resolve.";
 var PaymentTimeoutError = class extends Error {
   constructor() {
-    super("Payment verification timed out; awaiting late confirmation.");
+    super("Payment timeout: no payment received before the deadline.");
     this.name = "PaymentTimeoutError";
   }
 };
@@ -4580,6 +5406,17 @@ function customerSafeMessage(error) {
   }
   return "Internal processing error";
 }
+function parseRawEventDelegated(rawEventJson) {
+  if (rawEventJson === void 0) {
+    return false;
+  }
+  try {
+    const raw = JSON.parse(rawEventJson);
+    return Array.isArray(raw.tags) && raw.tags.some((tag) => Array.isArray(tag) && tag[0] === "payment" && tag[1] === "delegated");
+  } catch {
+    return false;
+  }
+}
 function bodyLooksLikeBilling3(body) {
   const lower = body.toLowerCase();
   return BILLING_BODY_MARKERS3.some((marker) => lower.includes(marker));
@@ -4627,6 +5464,12 @@ var AgentRuntime = class {
     this.nonceStore = nonceStore;
     this.limit = pLimit(config.maxConcurrentJobs);
     this.maxQueueSize = config.maxQueueSize ?? config.maxConcurrentJobs * 10;
+    this.paymentRecovery = new PaymentRecovery(
+      ledger,
+      config.network,
+      () => this.fetchProtocolConfig(),
+      payment
+    );
   }
   limit;
   inFlight = /* @__PURE__ */ new Set();
@@ -4637,6 +5480,27 @@ var AgentRuntime = class {
   recoveryInterval = null;
   gcInterval = null;
   stopped = false;
+  /**
+   * Backoff bookkeeping for `paid` entries whose payment recovery could not
+   * conclude. Public so an operator-facing surface (and the tests) can read the
+   * backlog without reaching into the runtime's internals.
+   */
+  recoveryDeferrals = new RecoveryDeferrals(() => this.config.recoveryIntervalSecs);
+  /** Settlement binding and payment re-verification - see `payment-recovery.ts`. */
+  paymentRecovery;
+  /**
+   * Size of the deferral backlog at the end of the previous tick, so the
+   * transition back to zero is logged exactly once. Without it the summary goes
+   * quiet on the tick the backlog clears and an operator never sees it clear -
+   * the one line they were waiting for.
+   */
+  lastDeferralBacklog = 0;
+  /**
+   * Memoised `raw_event_json` delegated discriminator, keyed on the ledger entry
+   * OBJECT - see {@link rawEventIsDelegated}. Weak so it is bounded by the
+   * entries the ledger is holding, not by everything this process has ever seen.
+   */
+  rawEventDelegatedCache = /* @__PURE__ */ new WeakMap();
   /** Per-customer sliding-window rate limiter for free skills. */
   freeCustomerLimiter = createSlidingWindowLimiter({
     windowMs: RATE_LIMIT_WINDOW_MS,
@@ -5632,24 +6496,80 @@ var AgentRuntime = class {
       }
     }
   }
+  /**
+   * Whether this tick would spend an RPC reference scan on the entry. The
+   * delegated discriminator is the runtime's (delegated entries reconcile
+   * through the pull path, which never reads the reference); the rest of the
+   * predicate is `needsPaymentScan`.
+   */
+  needsPaymentReVerification(entry) {
+    return needsPaymentScan(entry, this.isDelegatedEntry(entry));
+  }
+  /**
+   * Close a recovered job AND tell the customer why.
+   *
+   * Recovery's verdicts are the last word on a job the customer already paid
+   * for (or believes they did), so a silent `markFailed` leaves them watching a
+   * job that will never answer. The 24h cutoff has always sent a notice; every
+   * other terminal recovery verdict sends one too.
+   */
+  async failRecoveredJob(entry, job, message) {
+    this.ledger.markFailed(entry.job_id);
+    await this.transport.sendFeedback(job, { type: "error", message }).catch(() => {
+    });
+  }
+  /**
+   * One line per tick that sums up the deferral backlog, including the tick it
+   * finally clears - an operator watching a stuck agent needs to see the end of
+   * it, and a summary that simply stops appearing is indistinguishable from an
+   * agent that stopped logging.
+   */
+  logDeferralSummary(log) {
+    const backlog = this.recoveryDeferrals.size;
+    const summary = this.recoveryDeferrals.summaryLine();
+    if (summary !== null) {
+      log(summary);
+    } else if (this.lastDeferralBacklog > 0) {
+      log("Recovery: no jobs are deferred awaiting payment confirmation any more.");
+    }
+    this.lastDeferralBacklog = backlog;
+  }
   /**
    * Delegated discriminator for recovery routing. The ledger flag is primary;
    * the persisted raw event's top-level `payment` tag is the fallback for an
    * entry whose flag write was lost.
+   *
+   * The flag is re-read every call - `markDelegated` can set it on an entry a
+   * recovery tick has already looked at - while the FALLBACK is memoised, since
+   * `raw_event_json` never changes for a given entry object.
    */
   isDelegatedEntry(entry) {
     if (entry.delegated === true) {
       return true;
     }
-    if (entry.raw_event_json === void 0) {
-      return false;
-    }
-    try {
-      const raw = JSON.parse(entry.raw_event_json);
-      return Array.isArray(raw.tags) && raw.tags.some((tag) => Array.isArray(tag) && tag[0] === "payment" && tag[1] === "delegated");
-    } catch {
-      return false;
+    return this.rawEventIsDelegated(entry);
+  }
+  /**
+   * The `raw_event_json` half of {@link isDelegatedEntry}, memoised per entry
+   * OBJECT.
+   *
+   * The per-tick scan budget asks this of every non-deferred pending entry, and
+   * an entry over budget is parked and asked again next tick, so an uncached
+   * version re-parses every pending job's whole payload once a minute for as
+   * long as the backlog lasts - a full JSON parse of the customer's input (up to
+   * the 64KB inline cap) per entry per tick, to read one tag. The persisted
+   * event is immutable for the life of an entry object, so one parse is the
+   * whole answer; a ledger reload produces fresh objects and re-parses, which is
+   * correct.
+   */
+  rawEventIsDelegated(entry) {
+    const memoised = this.rawEventDelegatedCache.get(entry);
+    if (memoised !== void 0) {
+      return memoised;
     }
+    const delegated = parseRawEventDelegated(entry.raw_event_json);
+    this.rawEventDelegatedCache.set(entry, delegated);
+    return delegated;
   }
   /**
    * Reconcile a delegated `paid` entry to its terminal - the FIRST action in
@@ -6085,6 +7005,7 @@ var AgentRuntime = class {
     const sigPathTimeoutMs = Math.min(deadlineMs, SIG_PATH_TIMEOUT_MS);
     let sigOutcome;
     let refOutcome;
+    let paymentRefusal;
     let result;
     try {
       result = await new Promise((resolve4, reject) => {
@@ -6112,6 +7033,25 @@ var AgentRuntime = class {
             resolve4(lastResult);
           }
         };
+        const accept = (verified, pathLabel, askedSignature) => {
+          if (settled) {
+            return;
+          }
+          const txSignature = askedSignature ?? verified.txSignature;
+          if (txSignature === void 0) {
+            paymentRefusal = { consumedByOther: false, sentence: UNNAMED_SETTLEMENT_SENTENCE };
+            lose({ verified: false }, `${pathLabel}: ${UNNAMED_SETTLEMENT_SENTENCE}`);
+            return;
+          }
+          const claim = this.paymentRecovery.claimSettlementSignature(txSignature, job, log);
+          if (claim !== "claimed") {
+            const sentence = claimRefusalSentence(claim);
+            paymentRefusal = { consumedByOther: claim === "consumed-by-other", sentence };
+            lose({ verified: false }, `${pathLabel}: settlement refused - ${sentence}`);
+            return;
+          }
+          win(verified);
+        };
         this.transport.waitForPaymentSignature(job.jobId, job.customerId, verifyAbort.signal, sigPathTimeoutMs).then(async (sig) => {
           if (settled) {
             return;
@@ -6127,7 +7067,7 @@ var AgentRuntime = class {
               txSignature: sig
             });
             if (verified.verified) {
-              win(verified);
+              accept(verified, "sig path", sig);
             } else {
               const reason = verified.error ?? "unknown";
               sigOutcome = { gotSignature: true, error: reason };
@@ -6145,7 +7085,7 @@ var AgentRuntime = class {
         });
         payment.verifyPayment(rpc, request, protocolConfig).then((verified) => {
           if (verified.verified) {
-            win(verified);
+            accept(verified, "ref path");
           } else {
             const reason = verified.error ?? "unknown";
             refOutcome = { error: reason };
@@ -6196,23 +7136,28 @@ var AgentRuntime = class {
     if (result.verified) {
       return { netAmount, paymentRequest: requestJson };
     }
-    const refDefinitiveNoPay = refOutcome?.error === "No matching transaction found for reference key";
-    const sigGotSignature = sigOutcome?.gotSignature === true;
-    const customerAbandoned = !sigGotSignature && refDefinitiveNoPay;
-    if (customerAbandoned) {
+    if (paymentRefusal?.consumedByOther === true) {
       log(
-        `[${job.jobId.slice(0, 8)}] Payment not received; on-chain scan found no matching transaction - job abandoned by customer.`
+        `[${job.jobId.slice(0, 8)}] Payment not accepted: ${paymentRefusal.sentence}. The job stays recoverable - recovery keeps looking for a transfer of its own.`
+      );
+    } else if (paymentRefusal !== void 0) {
+      log(
+        `[${job.jobId.slice(0, 8)}] Payment verified but NOT accepted: ${paymentRefusal.sentence}. The job stays recoverable and the next recovery tick retries it - see the REFUSED line above for what to fix.`
       );
     } else {
+      let sigReport;
+      if (sigOutcome === void 0) {
+        sigReport = "signature path: no result";
+      } else if (!sigOutcome.gotSignature) {
+        sigReport = sigOutcome.error ? `signature path: no payment-completed feedback (${sigOutcome.error})` : "signature path: no payment-completed feedback";
+      } else {
+        sigReport = `signature path: the customer asserted a signature that did not verify (${sigOutcome.error ?? "unknown"})`;
+      }
+      const refReport = refOutcome === void 0 ? "no result" : refOutcome.error ?? "unknown";
       log(
-        `[${job.jobId.slice(0, 8)}] WARNING: Payment verification timed out. Customer may have paid on-chain. Check address ${this.config.solanaAddress} manually.`
+        `[${job.jobId.slice(0, 8)}] WARNING: Payment verification timed out - reference path: ${refReport}; ${sigReport}. The customer may still have paid on-chain; recovery makes the final call. Check address ${this.config.solanaAddress} manually if it does not.`
       );
     }
-    await this.transport.sendFeedback(job, { type: "error", message: "payment timeout" }).catch(() => {
-    });
-    if (customerAbandoned) {
-      throw new Error("Payment timeout");
-    }
     throw new PaymentTimeoutError();
   }
   /**
@@ -6255,16 +7200,18 @@ var AgentRuntime = class {
   }
   async recoverPendingJobs() {
     const pending = this.ledger.pendingJobs().filter((e) => !this.inFlight.has(e.job_id));
+    this.recoveryDeferrals.sweep(new Set(this.ledger.pendingJobs().map((entry) => entry.job_id)));
     if (this.sessionStore !== void 0) {
       this.sessionStore.syncRecoveryRegistrations(
         this.collectRecoverySessionRefs(pending),
         (jobId) => this.ledger.getStatus(jobId) === "paid"
       );
     }
+    const log = this.callbacks.onLog ?? console.log;
+    this.logDeferralSummary(log);
     if (pending.length === 0) {
       return;
     }
-    const log = this.callbacks.onLog ?? console.log;
     log(`Recovering ${pending.length} pending jobs...`);
     if (this.healthMonitor) {
       const snap = this.healthMonitor.snapshot();
@@ -6288,6 +7235,8 @@ var AgentRuntime = class {
         }
       }
     }
+    const scanBudget = recoveryScanBudgetPerTick(this.config.maxConcurrentJobs);
+    let scansStartedThisTick = 0;
     for (const entry of pending) {
       const ageMs = (Math.floor(Date.now() / 1e3) - entry.created_at) * 1e3;
       const expired = ageMs > MAX_PAID_AGE_MS;
@@ -6320,6 +7269,16 @@ var AgentRuntime = class {
       if (!entry.raw_event_json) {
         continue;
       }
+      if (this.recoveryDeferrals.isDeferred(entry.job_id)) {
+        continue;
+      }
+      if (this.needsPaymentReVerification(entry)) {
+        if (scansStartedThisTick >= scanBudget) {
+          this.recoveryDeferrals.parkForCapacity(entry.job_id);
+          continue;
+        }
+        scansStartedThisTick += 1;
+      }
       if (this.pending >= this.maxQueueSize) {
         break;
       }
@@ -6386,9 +7345,54 @@ var AgentRuntime = class {
         const skill = this.skills.route(entry.tags);
         if (!skill) {
           log(`[${entry.job_id.slice(0, 8)}] Recovery: no skill for tags, marking failed`);
-          this.ledger.markFailed(entry.job_id);
+          await this.failRecoveredJob(entry, fakeJob, RECOVERY_NO_SKILL_CUSTOMER_MESSAGE);
           return;
         }
+        if (skill.priceSubunits > 0 && !entry.net_amount) {
+          if (entry.payment_request) {
+            const reVerification = await this.paymentRecovery.reVerifyPayment(
+              entry,
+              entry.payment_request,
+              skill.priceSubunits,
+              log,
+              recoveryAbort.signal
+            );
+            if (reVerification === "deferred") {
+              this.recoveryDeferrals.note(entry.job_id);
+              return;
+            }
+            if (reVerification === "awaiting-window") {
+              this.recoveryDeferrals.noteAwaitingWindow(entry.job_id);
+              return;
+            }
+            if (reVerification === "corrupt-state") {
+              log(
+                `[${entry.job_id.slice(0, 8)}] Recovery: the persisted payment request is unusable, so this job can never be verified - marking failed. This is a provider-side state problem; audit the ledger entry.`
+              );
+              await this.failRecoveredJob(entry, fakeJob, RECOVERY_UNVERIFIABLE_CUSTOMER_MESSAGE);
+              return;
+            }
+            if (reVerification === "no-payment") {
+              if (!this.recoveryDeferrals.sawNoPayment(entry.job_id)) {
+                log(
+                  `[${entry.job_id.slice(0, 8)}] Recovery: the reference's whole history is empty, but a single listing is not enough to close a paid job - confirming on the next attempt.`
+                );
+                this.recoveryDeferrals.note(entry.job_id, true);
+                return;
+              }
+              log(
+                `[${entry.job_id.slice(0, 8)}] Recovery: two consecutive complete scans found no payment on this reference - marking failed.`
+              );
+              await this.failRecoveredJob(entry, fakeJob, RECOVERY_NO_PAYMENT_CUSTOMER_MESSAGE);
+              return;
+            }
+            this.recoveryDeferrals.clear(entry.job_id);
+          } else {
+            log(`[${entry.job_id.slice(0, 8)}] Recovery: payment not confirmed, marking failed`);
+            await this.failRecoveredJob(entry, fakeJob, RECOVERY_UNVERIFIABLE_CUSTOMER_MESSAGE);
+            return;
+          }
+        }
         const healthPair = resolveHealthPair(skill);
         if (this.healthMonitor && healthPair) {
           try {
@@ -6423,24 +7427,6 @@ var AgentRuntime = class {
           }
         }
         this.ledger.incrementRetry(entry.job_id);
-        if (skill.priceSubunits > 0 && !entry.net_amount) {
-          if (entry.payment_request) {
-            const verified = await this.reVerifyPayment(
-              entry,
-              skill.priceSubunits,
-              log,
-              recoveryAbort.signal
-            );
-            if (!verified) {
-              this.ledger.markFailed(entry.job_id);
-              return;
-            }
-          } else {
-            log(`[${entry.job_id.slice(0, 8)}] Recovery: payment not confirmed, marking failed`);
-            this.ledger.markFailed(entry.job_id);
-            return;
-          }
-        }
         let recoveryInputFile;
         let recoverySession;
         try {
@@ -6458,7 +7444,7 @@ var AgentRuntime = class {
           );
         } catch {
           log(`[${entry.job_id.slice(0, 8)}] Recovery: input file unavailable, marking failed`);
-          this.ledger.markFailed(entry.job_id);
+          await this.failRecoveredJob(entry, fakeJob, RECOVERY_INPUT_UNAVAILABLE_CUSTOMER_MESSAGE);
           return;
         }
         const recoveryBudgetMs = this.resolveExecutionBudgetMs(skill);
@@ -6540,61 +7526,6 @@ var AgentRuntime = class {
       this.jobAbortControllers.delete(recoveryAbort);
     }
   }
-  /**
-   * Re-verify an on-chain payment during crash recovery.
-   *
-   * Limitation: Solana transaction data expires after ~2-3 days (recent blockhash window).
-   * If the agent was down longer, a confirmed payment may not be found on-chain and the
-   * job will be marked failed. For mainnet: use monitoring, avoid extended downtime, or
-   * configure an archive RPC via SOLANA_RPC_URL.
-   */
-  async reVerifyPayment(entry, priceSubunits, log, signal) {
-    try {
-      const request = JSON.parse(entry.payment_request);
-      const rpc = createSolanaRpc(getRpcUrl(this.config.network));
-      const protocolConfig = await this.fetchProtocolConfig();
-      let result;
-      if (signal) {
-        let abortHandler;
-        const abortPromise = new Promise((_, reject) => {
-          abortHandler = () => {
-            const err = new Error("The operation was aborted");
-            err.name = "AbortError";
-            reject(err);
-          };
-          if (signal.aborted) {
-            abortHandler();
-            return;
-          }
-          signal.addEventListener("abort", abortHandler, { once: true });
-        });
-        try {
-          result = await Promise.race([
-            payment.verifyPayment(rpc, request, protocolConfig),
-            abortPromise
-          ]);
-        } finally {
-          if (abortHandler) {
-            signal.removeEventListener("abort", abortHandler);
-          }
-        }
-      } else {
-        result = await payment.verifyPayment(rpc, request, protocolConfig);
-      }
-      if (result.verified) {
-        const fee = calculateProtocolFee(priceSubunits, protocolConfig.feeBps);
-        const netAmount = priceSubunits - fee;
-        this.ledger.updatePayment(entry.job_id, netAmount, entry.payment_request);
-        log(`[${entry.job_id.slice(0, 8)}] Recovery: payment re-verified (${netAmount} subunits)`);
-        return true;
-      }
-      log(`[${entry.job_id.slice(0, 8)}] Recovery: payment not found on-chain, marking failed`);
-      return false;
-    } catch (e) {
-      log(`[${entry.job_id.slice(0, 8)}] Recovery: payment re-verification error: ${e.message}`);
-      return false;
-    }
-  }
 };
 var SkillRegistry = class {
   skills = [];