@xyo-network/xl1-protocol-sdk 1.29.8 → 1.30.1

package/dist/neutral/index.mjs

@@ -4051,7 +4051,6 @@ import {
   asSignedHydratedBlockWithStorageMeta,
   asXL1BlockNumber as asXL1BlockNumber8,
   BlockViewerMoniker as BlockViewerMoniker2,
-  DataLakeViewerMoniker,
   FinalizationViewerMoniker
 } from "@xyo-network/xl1-protocol-lib";
 
@@ -4092,10 +4091,9 @@ var MIN_HEAD_POLL_INTERVAL_MS = 5e3;
 var SimpleBlockViewer = class extends AbstractCreatableProvider {
   moniker = SimpleBlockViewer.defaultMoniker;
   _store;
-  dataLakeViewer;
   finalizationViewer;
   payloadCache = new LruCacheMap({ max: 1e4 });
-  signedHydratedBlockWithDataLakePayloadsCache = new LruCacheMap({ max: 2e3, ttl: 1e3 * 60 * 60 });
+  signedHydratedBlockWithHashMetaCache = new LruCacheMap({ max: 2e3, ttl: 1e3 * 60 * 60 });
   _headPollHash;
   _headPollInProgress = false;
   _headPollTimer = null;
@@ -4131,17 +4129,16 @@ var SimpleBlockViewer = class extends AbstractCreatableProvider {
   }
   async blockByHash(hash) {
     return await this.spanAsync("blockByHash", async () => {
-      const cachedBlock = this.signedHydratedBlockWithDataLakePayloadsCache.get(hash);
+      const cachedBlock = this.signedHydratedBlockWithHashMetaCache.get(hash);
       if (cachedBlock) {
         return cachedBlock;
       }
       const cache = this.hydratedBlockCache;
       const block = await cache.get(hash);
-      const [result] = block ? await addDataLakePayloads(block, this.dataLakeViewer) : [null, []];
-      if (result) {
-        this.signedHydratedBlockWithDataLakePayloadsCache.set(hash, result);
+      if (block) {
+        this.signedHydratedBlockWithHashMetaCache.set(hash, block);
       }
-      return result;
+      return block ?? null;
     }, { ...this.context, timeBudgetLimit: 100 });
   }
   async blockByNumber(blockNumber) {
@@ -4194,13 +4191,11 @@ var SimpleBlockViewer = class extends AbstractCreatableProvider {
   }
   async createHandler() {
     await super.createHandler();
-    this.dataLakeViewer = await this.locator.tryGetInstance(DataLakeViewerMoniker);
     this.finalizationViewer = await this.locator.getInstance(FinalizationViewerMoniker);
     this._store = { chainMap: this.params.finalizedArchivist };
   }
   async currentBlock() {
-    const [result] = await addDataLakePayloads(await this.finalizationViewer.head(), this.dataLakeViewer);
-    return result;
+    return await this.finalizationViewer.head();
   }
   async currentBlockHash() {
     return await this.finalizationViewer.headHash();
@@ -4218,7 +4213,7 @@ var SimpleBlockViewer = class extends AbstractCreatableProvider {
     const cachedHashes = new Set(cachedPayloads.map((p) => p._hash));
     remainingHashes = remainingHashes.filter((h) => !cachedHashes.has(h));
     const finalizedPayloads = remainingHashes.length > 0 ? await this.finalizedArchivist.get(remainingHashes) : [];
-    const [resultPayloads] = await addDataLakePayloadsToPayloads(hashes, [...cachedPayloads, ...finalizedPayloads.filter(exists4)], this.dataLakeViewer);
+    const resultPayloads = [...cachedPayloads, ...finalizedPayloads.filter(exists4)];
     resultPayloads.map((payload) => {
       this.payloadCache.set(payload._hash, payload);
     });
@@ -4257,9 +4252,7 @@ var SimpleBlockViewer = class extends AbstractCreatableProvider {
     await super.stopHandler();
   }
   async blockByNumberWithContext(chainContext, blockNumber) {
-    const block = asSignedHydratedBlockWithHashMeta(await hydratedBlockByNumber(chainContext, blockNumber)) ?? null;
-    const [result] = block ? await addDataLakePayloads(block, this.dataLakeViewer) : [null, []];
-    return result;
+    return asSignedHydratedBlockWithHashMeta(await hydratedBlockByNumber(chainContext, blockNumber)) ?? null;
   }
   async pollHead(emitOnChange) {
     if (this._headPollInProgress) return;
@@ -4629,14 +4622,14 @@ RestDataLakeRunner = __decorateClass([
 
 // src/simple/datalake/RestDataLakeViewer.ts
 import {
-  DataLakeViewerMoniker as DataLakeViewerMoniker2
+  DataLakeViewerMoniker
 } from "@xyo-network/xl1-protocol-lib";
 var RestDataLakeViewer = class extends AbstractRestDataLake {
   moniker = RestDataLakeViewer.defaultMoniker;
 };
-__publicField(RestDataLakeViewer, "defaultMoniker", DataLakeViewerMoniker2);
+__publicField(RestDataLakeViewer, "defaultMoniker", DataLakeViewerMoniker);
 __publicField(RestDataLakeViewer, "dependencies", []);
-__publicField(RestDataLakeViewer, "monikers", [DataLakeViewerMoniker2]);
+__publicField(RestDataLakeViewer, "monikers", [DataLakeViewerMoniker]);
 RestDataLakeViewer = __decorateClass([
   creatableProvider()
 ], RestDataLakeViewer);
@@ -4714,14 +4707,14 @@ SimpleDataLakeRunner = __decorateClass([
 
 // src/simple/datalake/SimpleDataLakeViewer.ts
 import {
-  DataLakeViewerMoniker as DataLakeViewerMoniker3
+  DataLakeViewerMoniker as DataLakeViewerMoniker2
 } from "@xyo-network/xl1-protocol-lib";
 var SimpleDataLakeViewer = class extends AbstractSimpleDataLake {
   moniker = SimpleDataLakeViewer.defaultMoniker;
 };
-__publicField(SimpleDataLakeViewer, "defaultMoniker", DataLakeViewerMoniker3);
+__publicField(SimpleDataLakeViewer, "defaultMoniker", DataLakeViewerMoniker2);
 __publicField(SimpleDataLakeViewer, "dependencies", []);
-__publicField(SimpleDataLakeViewer, "monikers", [DataLakeViewerMoniker3]);
+__publicField(SimpleDataLakeViewer, "monikers", [DataLakeViewerMoniker2]);
 SimpleDataLakeViewer = __decorateClass([
   creatableProvider()
 ], SimpleDataLakeViewer);
@@ -4980,18 +4973,26 @@ import {
   isSignedHydratedBlockWithHashMeta,
   isSignedHydratedTransactionWithHashMeta,
   MempoolRunnerMoniker,
+  MempoolViewerMoniker,
   TransactionRejectionSchema,
   TransactionValidationViewerMoniker
 } from "@xyo-network/xl1-protocol-lib";
 import { Mutex } from "async-mutex";
 var DEFAULT_SYNC_INTERVAL = 3e4;
 var DEFAULT_SYNC_LIMIT = 100;
+var ENFORCE_CAP_BATCH_SIZE = 1e3;
+function isDemotionAware(viewer) {
+  if (!viewer) return false;
+  const candidate = viewer;
+  return typeof candidate.forget === "function" && typeof candidate.getEvictionPriorityOrder === "function";
+}
 var SimpleMempoolRunner = class extends AbstractCreatableProvider {
   moniker = SimpleMempoolRunner.defaultMoniker;
   _blockValidationViewer;
   _chainContractViewer;
   _deadLetterQueueRunner;
   _finalizationViewer;
+  _mempoolViewer;
   _transactionValidationViewer;
   _syncMutex = new Mutex();
   _syncTimerId = null;
@@ -5010,6 +5011,9 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
   get maxExpAhead() {
     return this.params.maxExpAhead ?? DEFAULT_MAX_EXP_AHEAD;
   }
+  get maxPendingTransactions() {
+    return this.params.maxPendingTransactions ?? 0;
+  }
   get pendingBlocksArchivist() {
     return this.params.pendingBlocksArchivist;
   }
@@ -5042,6 +5046,7 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
     this._finalizationViewer = await this.locator.getInstance(FinalizationViewerMoniker4);
     this._transactionValidationViewer = await this.locator.getInstance(TransactionValidationViewerMoniker);
     this._deadLetterQueueRunner = await this.locator.tryGetInstance(DeadLetterQueueRunnerMoniker);
+    this._mempoolViewer = await this.locator.tryGetInstance(MempoolViewerMoniker);
   }
   async prunePendingBlocks({
     batchSize = 10,
@@ -5156,6 +5161,7 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
       pruned += pruneHashes.length;
       total += batch.length;
       await this.pendingTransactionsArchivist.delete(pruneHashes);
+      this.forgetBundleHashes(pruneHashes);
       const pruneSet = new Set(pruneHashes);
       const lastSurvivor = batch.findLast((p) => !pruneSet.has(p._hash));
       cursor = lastSurvivor?._sequence ?? cursor;
@@ -5165,8 +5171,7 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
         order: "desc"
       });
     }
-    this.logger?.debug(`prunePendingTransactions completed: pruned=${pruned}, totalChecked=${total}`);
-    return [pruned, total];
+    return this.finalizePruneTransactionsResult(pruned, total);
   }
   async submitBlocks(blocks) {
     const bundles = await Promise.all(blocks.map(async ([bw, payloads]) => {
@@ -5212,6 +5217,7 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
     }
     const bundles = hashedTransactions.map((tx) => hydratedTransactionToPayloadBundle(tx));
     const inserted = await this.pendingTransactionsArchivist.insert(bundles);
+    await this.enforceCap();
     return inserted.map((p) => p._hash);
   }
   async startHandler() {
@@ -5230,6 +5236,50 @@ var SimpleMempoolRunner = class extends AbstractCreatableProvider {
       this._syncTimerId = null;
     }
   }
+  async collectAllPendingTransactionBundles() {
+    const all = [];
+    let cursor;
+    while (true) {
+      const batch = await this.pendingTransactionsArchivist.next({
+        limit: ENFORCE_CAP_BATCH_SIZE,
+        cursor,
+        order: "asc"
+      });
+      if (batch.length === 0) break;
+      all.push(...batch);
+      cursor = batch.at(-1)?._sequence;
+      if (batch.length < ENFORCE_CAP_BATCH_SIZE) break;
+    }
+    return all;
+  }
+  async enforceCap() {
+    const cap = this.maxPendingTransactions;
+    if (cap <= 0) return;
+    const all = await this.collectAllPendingTransactionBundles();
+    if (all.length <= cap) return;
+    const excess = all.length - cap;
+    const bundleHashesOldestFirst = all.map((p) => p._hash);
+    const orderedForEviction = this.orderForEviction(bundleHashesOldestFirst);
+    const toEvict = orderedForEviction.slice(0, excess);
+    await this.pendingTransactionsArchivist.delete(toEvict);
+    this.forgetBundleHashes(toEvict);
+    this.logger?.debug(`enforceCap evicted ${toEvict.length} bundles (pool=${all.length}, cap=${cap})`);
+  }
+  async finalizePruneTransactionsResult(pruned, total) {
+    this.logger?.debug(`prunePendingTransactions completed: pruned=${pruned}, totalChecked=${total}`);
+    await this.enforceCap();
+    return [pruned, total];
+  }
+  forgetBundleHashes(bundleHashes) {
+    if (bundleHashes.length === 0) return;
+    if (isDemotionAware(this._mempoolViewer)) this._mempoolViewer.forget(bundleHashes);
+  }
+  orderForEviction(bundleHashesOldestFirst) {
+    if (isDemotionAware(this._mempoolViewer)) {
+      return this._mempoolViewer.getEvictionPriorityOrder(bundleHashesOldestFirst);
+    }
+    return bundleHashesOldestFirst;
+  }
   async routeRejectedTransaction(transaction, errors) {
     if (!this._deadLetterQueueRunner) return;
     const rejectionErrors = errors.map((e) => ({
@@ -5342,13 +5392,22 @@ import {
 } from "@xylabs/sdk-js";
 import { isHashMeta as isHashMeta2, isPayloadBundle as isPayloadBundle2 } from "@xyo-network/sdk-js";
 import {
-  MempoolViewerMoniker,
+  MempoolViewerMoniker as MempoolViewerMoniker2,
   WindowedBlockViewerMoniker
 } from "@xyo-network/xl1-protocol-lib";
 var DEFAULT_MEMPOOL_SELECTION_RATIO = 0.66;
+var DEFAULT_DEMOTION_THRESHOLD = 3;
+var DEFAULT_HANDOUT_STATS_TTL_BLOCKS = 1e3;
 var SimpleMempoolViewer = class extends AbstractCreatableProvider {
   moniker = SimpleMempoolViewer.defaultMoniker;
+  _handoutStats = /* @__PURE__ */ new Map();
   _windowedBlockViewer;
+  get demotionThreshold() {
+    return this.params.demotionThreshold ?? DEFAULT_DEMOTION_THRESHOLD;
+  }
+  get handoutStatsTtlBlocks() {
+    return this.params.handoutStatsTtlBlocks ?? DEFAULT_HANDOUT_STATS_TTL_BLOCKS;
+  }
   get pendingBlocksArchivist() {
     return this.params.pendingBlocksArchivist;
   }
@@ -5362,6 +5421,32 @@ var SimpleMempoolViewer = class extends AbstractCreatableProvider {
     await super.createHandler();
     this._windowedBlockViewer = await this.locator.getInstance(WindowedBlockViewerMoniker);
   }
+  /** Drop handout stats for the given bundle hashes. Called when a bundle has been evicted or otherwise removed from the pool. */
+  forget(bundleHashes) {
+    for (const hash of bundleHashes) this._handoutStats.delete(hash);
+  }
+  /** Return the subset of the given bundle hashes that are currently considered demoted. */
+  getDemotedBundleHashes(bundleHashes) {
+    return bundleHashes.filter((hash) => this.isDemoted(hash));
+  }
+  /**
+   * Return the bundle hashes in the order they should be evicted under size pressure.
+   * Demoted entries come first, sorted by handouts descending; the remainder is left as-is
+   * so the caller can append by FIFO sequence order.
+   */
+  getEvictionPriorityOrder(bundleHashes) {
+    const demoted = [];
+    const nonDemoted = [];
+    for (const hash of bundleHashes) {
+      if (this.isDemoted(hash)) demoted.push(hash);
+      else nonDemoted.push(hash);
+    }
+    demoted.sort((a, b) => (this._handoutStats.get(b)?.handouts ?? 0) - (this._handoutStats.get(a)?.handouts ?? 0));
+    return [...demoted, ...nonDemoted];
+  }
+  getHandoutStats(bundleHash) {
+    return this._handoutStats.get(bundleHash);
+  }
   async pendingBlocks({ cursor: providedCursor } = {}) {
     let cursor = void 0;
     if (isHash2(providedCursor)) {
@@ -5404,6 +5489,7 @@ var SimpleMempoolViewer = class extends AbstractCreatableProvider {
       })
     )).filter(exists9);
     const currentBlock = await this.windowedBlockViewer.currentBlock();
+    const currentBlockNumber = currentBlock[0].block;
     const evaluated = await Promise.all(
       hydratedWithBundle.map(async ({ bundle: bundle3, tx }) => ({
         bundle: bundle3,
@@ -5417,25 +5503,43 @@ var SimpleMempoolViewer = class extends AbstractCreatableProvider {
     await Promise.all(
       deletionCandidates.map(async ({ bundle: bundle3, tx }) => {
         await this.deleteBundledTransaction(bundle3);
+        this._handoutStats.delete(bundle3._hash);
         this.logger?.debug(`Purged completed/expired bundled transaction: ${bundle3._hash}/${tx[0]._hash}`);
       })
     );
-    const inclusionCandidates = (await Promise.all(validTransactions.map((x) => x.tx).map(async (tx) => {
-      if (await this.isInclusionCandidate(tx, currentBlock, false)) return tx;
+    this.gcHandoutStats(currentBlockNumber);
+    const inclusionCandidates = (await Promise.all(validTransactions.map(async ({ bundle: bundle3, tx }) => {
+      if (await this.isInclusionCandidate(tx, currentBlock, false)) return { bundle: bundle3, tx };
     }))).filter(exists9);
     const selectionRatio = this.params.mempoolSelectionRatio ?? DEFAULT_MEMPOOL_SELECTION_RATIO;
-    const maxByRatio = Math.ceil(inclusionCandidates.length * selectionRatio);
-    const effectiveLimit = Math.min(limit, maxByRatio);
-    const randomInclusionCandidates = deduplicateBySigner(
-      inclusionCandidates.filter(() => Math.random() < selectionRatio)
-    ).slice(0, effectiveLimit);
-    const result = randomInclusionCandidates.length > 0 ? randomInclusionCandidates : deduplicateBySigner(inclusionCandidates).slice(0, 1);
-    this.logger?.debug(`Inclusion candidates: ${inclusionCandidates.length}`);
-    return result;
+    const nonDemoted = inclusionCandidates.filter(({ bundle: bundle3 }) => !this.isDemoted(bundle3._hash));
+    const demoted = inclusionCandidates.filter(({ bundle: bundle3 }) => this.isDemoted(bundle3._hash));
+    const primary = this.selectWithRatio(nonDemoted, limit, selectionRatio);
+    const topupNeeded = limit - primary.length;
+    const topup = topupNeeded > 0 ? this.selectWithRatio(demoted, topupNeeded, selectionRatio) : [];
+    let combined = [...primary, ...topup];
+    if (combined.length === 0 && inclusionCandidates.length > 0) {
+      combined = deduplicateWithBundleBySigner(inclusionCandidates).slice(0, 1);
+    }
+    for (const { bundle: bundle3 } of combined) {
+      this.recordHandout(bundle3._hash, currentBlockNumber);
+    }
+    this.logger?.debug(`Inclusion candidates: ${inclusionCandidates.length} (nonDemoted=${nonDemoted.length}, demoted=${demoted.length}); returning ${combined.length}`);
+    return combined.map(({ tx }) => tx);
+  }
+  isDemoted(bundleHash) {
+    const stats = this._handoutStats.get(bundleHash);
+    return stats !== void 0 && stats.handouts >= this.demotionThreshold;
   }
   async deleteBundledTransaction(bundle3) {
     await this.pendingTransactionsArchivist.delete([bundle3._hash]);
   }
+  gcHandoutStats(currentBlockNumber) {
+    const ttl = this.handoutStatsTtlBlocks;
+    for (const [hash, stats] of this._handoutStats) {
+      if (currentBlockNumber - stats.firstHandoutAt > ttl) this._handoutStats.delete(hash);
+    }
+  }
   /**
    * Evaluates a transaction to determine if it should be purged from the mempool.
    * @param tx The transaction to evaluate
@@ -5468,16 +5572,31 @@ var SimpleMempoolViewer = class extends AbstractCreatableProvider {
     if (checkForDeletable && await this.isDeletable(tx, currentBlock)) return false;
     return true;
   }
+  recordHandout(bundleHash, currentBlockNumber) {
+    const existing = this._handoutStats.get(bundleHash);
+    if (existing) {
+      existing.handouts += 1;
+    } else {
+      this._handoutStats.set(bundleHash, { handouts: 1, firstHandoutAt: currentBlockNumber });
+    }
+  }
+  selectWithRatio(group, take, selectionRatio) {
+    if (take <= 0 || group.length === 0) return [];
+    const maxByRatio = Math.ceil(group.length * selectionRatio);
+    const effectiveLimit = Math.min(take, maxByRatio);
+    const randomSelected = group.filter(() => Math.random() < selectionRatio);
+    return deduplicateWithBundleBySigner(randomSelected).slice(0, effectiveLimit);
+  }
 };
-__publicField(SimpleMempoolViewer, "defaultMoniker", MempoolViewerMoniker);
+__publicField(SimpleMempoolViewer, "defaultMoniker", MempoolViewerMoniker2);
 __publicField(SimpleMempoolViewer, "dependencies", [WindowedBlockViewerMoniker]);
-__publicField(SimpleMempoolViewer, "monikers", [MempoolViewerMoniker]);
+__publicField(SimpleMempoolViewer, "monikers", [MempoolViewerMoniker2]);
 SimpleMempoolViewer = __decorateClass([
   creatableProvider()
 ], SimpleMempoolViewer);
-function deduplicateBySigner(txs) {
+function deduplicateWithBundleBySigner(items) {
   const seen = /* @__PURE__ */ new Set();
-  return txs.filter((tx) => {
+  return items.filter(({ tx }) => {
     const key = tx[0].addresses.toSorted().join(",");
     if (seen.has(key)) return false;
     seen.add(key);