@xyo-network/chain-services 1.5.11 → 1.5.13

package/dist/neutral/index.mjs

@@ -394,8 +394,8 @@ var XyoBlockProducer = class _XyoBlockProducer extends BaseService {
   get pendingTransactionsService() {
     return assertEx3(this.params.pendingTransactionsService, () => "Missing pendingTransactionsService");
   }
-  get rejectedTransactionsArchivist() {
-    return assertEx3(this.params.rejectedTransactionsArchivist, () => "No rejected transactions archivist");
+  get rejectedBundledTransactionsArchivist() {
+    return assertEx3(this.params.rejectedBundledTransactionsArchivist, () => "No rejected bundled transactions archivist");
   }
   get rewardAddress() {
     return assertEx3(this.params.rewardAddress, () => "No reward address provided");
@@ -502,7 +502,7 @@ var XyoBlockProducer = class _XyoBlockProducer extends BaseService {
       if (errors.length > 0) {
         this.logger?.warn(`Validation of produced block failed: ${errors.at(0)?.message}`);
         const rejectedTransactions = block[1];
-        await this.rejectedTransactionsArchivist.insert(rejectedTransactions);
+        await this.rejectedBundledTransactionsArchivist.insert(rejectedTransactions);
       } else {
         return block;
       }
@@ -651,8 +651,8 @@ var XyoValidator = class extends BaseService {
   get electionService() {
     return assertEx6(this.params.electionService, () => "electionService is required");
   }
-  get pendingTransactionsArchivist() {
-    return assertEx6(this.params.pendingTransactionsArchivist, () => "pendingTransactions is required");
+  get pendingBundledTransactionsArchivist() {
+    return assertEx6(this.params.pendingBundledTransactionsArchivist, () => "pendingBundledTransactions is required");
   }
   get rewardService() {
     return assertEx6(this.params.rewardService, () => "rewardService is required");
@@ -729,14 +729,16 @@ XyoElectionService = _ts_decorate6([
 
 // src/PendingTransactions/PendingTransactions.ts
 import { ValueType } from "@opentelemetry/api";
-import { filterAs, filterAsync } from "@xylabs/array";
+import { filterAsync } from "@xylabs/array";
 import { assertEx as assertEx8 } from "@xylabs/assert";
 import { exists as exists2 } from "@xylabs/exists";
 import { forget } from "@xylabs/forget";
 import { MemoryArchivist } from "@xyo-network/archivist-memory";
+import { globalAttributes } from "@xyo-network/chain-telemetry";
 import { validateTransaction } from "@xyo-network/chain-validation";
-import { asOptionalTransactionBoundWitnessWithStorageMeta } from "@xyo-network/xl1-protocol";
-import { flattenHydratedTransactions, tryHydrateTransaction } from "@xyo-network/xl1-protocol-sdk";
+import { PayloadBuilder as PayloadBuilder4 } from "@xyo-network/payload-builder";
+import { asOptionalTransactionBoundWitnessWithStorageMeta, isTransactionBoundWitnessWithStorageMeta } from "@xyo-network/xl1-protocol";
+import { flattenHydratedTransaction } from "@xyo-network/xl1-protocol-sdk";
 import { Mutex as Mutex2 } from "async-mutex";
 function _ts_decorate7(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -745,6 +747,18 @@ function _ts_decorate7(decorators, target, key, desc) {
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
 __name(_ts_decorate7, "_ts_decorate");
+async function bundledPayloadsToHydratedTransaction(payload) {
+  const withStorageMeta = await PayloadBuilder4.addStorageMeta(payload.payloads);
+  const tx = asOptionalTransactionBoundWitnessWithStorageMeta(withStorageMeta.find((p) => p._hash === payload.root));
+  if (tx) {
+    return [
+      tx,
+      withStorageMeta.map((p) => p._hash === payload.root ? void 0 : p).filter(exists2)
+    ];
+  }
+}
+__name(bundledPayloadsToHydratedTransaction, "bundledPayloadsToHydratedTransaction");
+globalAttributes.setAttribute("XyoPendingTransactionsService:status", "unknown");
 var XyoPendingTransactionsService = class _XyoPendingTransactionsService extends BaseService {
   static {
     __name(this, "XyoPendingTransactionsService");
@@ -776,11 +790,12 @@ var XyoPendingTransactionsService = class _XyoPendingTransactionsService extends
   * A local Archivist optimized for fast retrieval that stores only validated
   * pending transactions
   */
-  _curatedPendingTransactionsArchivist;
+  _curatedPendingBundledTransactionsArchivist;
   /**
   * The last count of total pending transactions
   */
   _pendingTransactionsCount = 0;
+  _removablePendingTransactionHashes = /* @__PURE__ */ new Set();
   /**
   * A mutex to ensure that the curated pending transactions archivist is
   * updated in a thread-safe manner
@@ -792,73 +807,80 @@ var XyoPendingTransactionsService = class _XyoPendingTransactionsService extends
   get chainIdentification() {
     return assertEx8(this.params.chainIdentification, () => "No chain id");
   }
-  get pendingTransactionsArchivist() {
-    return assertEx8(this.params.pendingTransactionsArchivist, () => "No pending transactions archivist");
+  get pendingBundledTransactionsArchivist() {
+    return assertEx8(this.params.pendingBundledTransactionsArchivist, () => "No pending bundled transactions archivist");
+  }
+  get pendingBundledTransactionsLocalArchivist() {
+    return assertEx8(this._curatedPendingBundledTransactionsArchivist, () => "No pending bundled transactions curated archivist");
   }
   get pendingTransactionsCount() {
     forget(this.countPendingTransactions());
     return this._pendingTransactionsCount;
   }
-  get pendingTransactionsLocalArchivist() {
-    return assertEx8(this._curatedPendingTransactionsArchivist, () => "No pending transactions curated archivist");
-  }
-  get rejectedTransactionsArchivist() {
-    return assertEx8(this.params.rejectedTransactionsArchivist, () => "No rejected transactions archivist");
+  get rejectedBundledTransactionsArchivist() {
+    return assertEx8(this.params.rejectedBundledTransactionsArchivist, () => "No rejected bundled transactions archivist");
   }
   async createHandler() {
     await super.createHandler();
-    this._curatedPendingTransactionsArchivist = await MemoryArchivist.create({
+    this._curatedPendingBundledTransactionsArchivist = await MemoryArchivist.create({
       account: "random"
     });
-    this.pendingTransactionsArchivist.on("inserted", async ({ payloads }) => {
+    this.pendingBundledTransactionsArchivist.on("inserted", async ({ payloads }) => {
       await this.insertNewTransactions(payloads);
     });
-    this.chainArchivist.on("inserted", async ({ payloads }) => {
-      await this.removeFinalizedTransactions(payloads);
+    this.chainArchivist.on("inserted", ({ payloads }) => {
+      this.removeFinalizedTransactions(payloads);
     });
-    this.rejectedTransactionsArchivist.on("inserted", async ({ payloads }) => {
-      await this.removeRejectedTransactions(payloads);
+    this.rejectedBundledTransactionsArchivist.on("inserted", ({ payloads }) => {
+      this.removeRejectedBundledTransactions(payloads);
     });
-    const pendingTransactionsGauge = this.meter?.createObservableGauge("xyo_pending_transactions_count", {
+    const pendingTransactionsCounter = this.meter?.createObservableUpDownCounter("xyo_pending_transactions_counter", {
       description: "The current number of pending transactions",
-      valueType: ValueType.INT
+      valueType: ValueType.INT,
+      unit: "1"
     });
-    pendingTransactionsGauge?.addCallback((observer) => {
+    pendingTransactionsCounter?.addCallback((observer) => {
       observer.observe(this.pendingTransactionsCount);
     });
+    globalAttributes.setAttribute("XyoPendingTransactionsService:status", "created");
   }
   async getPendingTransactions(head, limit) {
     return await this.spanAsync("getPendingTransactions", async () => {
       return await this._updateCuratedPendingTransactionsArchivistMutex.runExclusive(async () => {
         const foundPendingTransactions = [];
+        const foundPendingTransactionsToDeleteHashes = [];
         let cursor;
         while (foundPendingTransactions.length < limit) {
-          const payloads = await this.pendingTransactionsLocalArchivist.next({
+          const payloads = await this.pendingBundledTransactionsLocalArchivist.next({
             limit: 100,
             order: "asc",
             cursor
           });
           if (payloads.length === 0) break;
           cursor = payloads.at(-1)?._sequence;
-          const transactions = payloads.map((payload) => asOptionalTransactionBoundWitnessWithStorageMeta(payload)).filter(exists2);
+          const deletedTransactionBundles = payloads.filter((tx) => this._removablePendingTransactionHashes.has(tx.root));
+          foundPendingTransactionsToDeleteHashes.push(...deletedTransactionBundles.map((tx) => tx._hash).filter(exists2));
+          const undeletedTransactionBundles = payloads.filter((tx) => !this._removablePendingTransactionHashes.has(tx.root));
+          const transactions = (await Promise.all(undeletedTransactionBundles.map((p) => bundledPayloadsToHydratedTransaction(p)))).filter(exists2);
           foundPendingTransactions.push(...transactions);
         }
-        const hydratedTransactions = await Promise.all(foundPendingTransactions.map((tx) => tryHydrateTransaction(this.pendingTransactionsLocalArchivist, tx._hash)));
-        return hydratedTransactions.filter(exists2);
+        for (const hash of foundPendingTransactionsToDeleteHashes) {
+          this._removablePendingTransactionHashes.delete(hash);
+        }
+        await this.pendingBundledTransactionsLocalArchivist.delete(foundPendingTransactionsToDeleteHashes);
+        return foundPendingTransactions;
       }, _XyoPendingTransactionsService.MutexPriority.ReadTransactions);
     });
   }
   async countPendingTransactions() {
     if (this._countPendingTransactionsMutex.isLocked()) return;
     await this._countPendingTransactionsMutex.runExclusive(async () => {
-      const payloads = await this._curatedPendingTransactionsArchivist?.all() ?? [];
-      const pendingTransactions = filterAs(payloads, asOptionalTransactionBoundWitnessWithStorageMeta);
-      this._pendingTransactionsCount = pendingTransactions.length;
+      const payloads = await this._curatedPendingBundledTransactionsArchivist?.all() ?? [];
+      this._pendingTransactionsCount = payloads.length;
     });
   }
-  async filterAlreadyFinalizedTransactions(payloads) {
-    const incomingTransactions = filterAs(payloads, asOptionalTransactionBoundWitnessWithStorageMeta);
-    const incomingTransactionHashes = incomingTransactions.map((payload) => payload._hash);
+  async filterAlreadyFinalizedTransactions(incomingTransactions) {
+    const incomingTransactionHashes = incomingTransactions.map((payload) => payload.root);
     const finalizedTransactions = await this.chainArchivist.get(incomingTransactionHashes);
     const finalizedTransactionHashes = new Set(finalizedTransactions.map((item) => item._hash));
     const nonFinalizedTransactions = incomingTransactions.filter((item) => !finalizedTransactionHashes.has(item._hash));
@@ -869,8 +891,8 @@ var XyoPendingTransactionsService = class _XyoPendingTransactionsService extends
     return await this.spanAsync("InsertNewTransactions", async () => {
       return await this._updateCuratedPendingTransactionsArchivistMutex.runExclusive(async () => {
         const unprocessedTransactions = await this.filterAlreadyFinalizedTransactions(payloads);
-        const hydratedUnprocessedTransactions = (await Promise.all(unprocessedTransactions.map((tx) => {
-          return tryHydrateTransaction(this.pendingTransactionsArchivist, tx._hash);
+        const hydratedUnprocessedTransactions = (await Promise.all(unprocessedTransactions.map(async (tx) => {
+          return await bundledPayloadsToHydratedTransaction(tx);
         }))).filter(exists2);
         const validTransactions = await filterAsync(hydratedUnprocessedTransactions, async (tx) => {
           const errors = await validateTransaction(tx, this.chainIdentification.id);
@@ -880,32 +902,47 @@ var XyoPendingTransactionsService = class _XyoPendingTransactionsService extends
           return errors.length > 0 ? false : true;
         });
         if (validTransactions.length > 0) {
-          await this.pendingTransactionsLocalArchivist.insert(flattenHydratedTransactions(validTransactions));
+          await this.pendingBundledTransactionsLocalArchivist.insert(validTransactions.map((tx) => {
+            return {
+              root: tx[0]._hash,
+              payloads: flattenHydratedTransaction(tx)
+            };
+          }));
         }
       }, _XyoPendingTransactionsService.MutexPriority.InsertNewTransactions);
     });
   }
-  async removeFinalizedTransactions(payloads) {
-    return await this.removeTransactions(payloads, "RemoveFinalizedTransactions");
-  }
-  async removeRejectedTransactions(payloads) {
-    return await this.removeTransactions(payloads, "RemoveRejectedTransactions");
-  }
-  async removeTransactions(payloads, priority) {
+  async removeBundledTransactions(bundledPayloads, priority) {
     return await this.spanAsync(priority, async () => {
       return await this._updateCuratedPendingTransactionsArchivistMutex.runExclusive(async () => {
-        const transactions = filterAs(payloads, asOptionalTransactionBoundWitnessWithStorageMeta);
-        await this.pendingTransactionsLocalArchivist.delete(transactions.map((tx) => tx._hash));
+        const bundledTransactions = (await Promise.all(bundledPayloads.map(async (payload) => {
+          const withStorageMeta = await PayloadBuilder4.addStorageMeta(payload.payloads);
+          const tx = asOptionalTransactionBoundWitnessWithStorageMeta(withStorageMeta.find((p) => p._hash === payload.root));
+          return tx !== void 0 && payload._hash !== void 0 ? tx : void 0;
+        }))).filter(exists2);
+        await this.pendingBundledTransactionsLocalArchivist.delete(bundledTransactions.map((btx) => btx._hash));
       }, _XyoPendingTransactionsService.MutexPriority[priority]);
     });
   }
+  removeFinalizedTransactions(payloads) {
+    const finalizedTransactionHashes = payloads.filter(isTransactionBoundWitnessWithStorageMeta).map((p) => p._hash);
+    for (const hash of finalizedTransactionHashes) {
+      this._removablePendingTransactionHashes.add(hash);
+    }
+  }
+  removeRejectedBundledTransactions(payloads) {
+    const rejectedTransactionHashes = payloads.filter(isTransactionBoundWitnessWithStorageMeta).map((p) => p._hash);
+    for (const hash of rejectedTransactionHashes) {
+      this._removablePendingTransactionHashes.add(hash);
+    }
+  }
 };
 XyoPendingTransactionsService = _ts_decorate7([
   creatableService()
 ], XyoPendingTransactionsService);
 
 // src/StakeIntent/lib/getBlockSignedStakeDeclarations.ts
-import { filterAs as filterAs2 } from "@xylabs/array";
+import { filterAs } from "@xylabs/array";
 import { exists as exists3 } from "@xylabs/exists";
 import { asOptionalBoundWitness } from "@xyo-network/boundwitness-model";
 import { payloadSchemasContains } from "@xyo-network/boundwitness-validator";
@@ -913,13 +950,13 @@ import { BoundWitnessWrapper } from "@xyo-network/boundwitness-wrapper";
 import { asOptionalChainStakeIntent, ChainStakeIntentSchema as ChainStakeIntentSchema2 } from "@xyo-network/xl1-protocol";
 var getBlockSignedStakeDeclarations = /* @__PURE__ */ __name(async (block, archivist, intent) => {
   const blockData = await archivist.get(block.payload_hashes);
-  const bwsFromBlock = filterAs2(blockData, asOptionalBoundWitness);
+  const bwsFromBlock = filterAs(blockData, asOptionalBoundWitness);
   const bwsFromBlockWithDeclarations = bwsFromBlock.filter((bw) => payloadSchemasContains(bw, ChainStakeIntentSchema2));
   const validBlockBwsWithDeclarations = await filterToValidSignedBoundWitnesses(bwsFromBlockWithDeclarations);
   return (await Promise.all(validBlockBwsWithDeclarations.map(async (bw) => {
     const stakeIntentHashes = validBlockBwsWithDeclarations.flatMap(mapBoundWitnessToStakeIntentHashes).filter(exists3);
     const payloads = await archivist.get(stakeIntentHashes);
-    const stakeIntents = filterAs2(payloads, asOptionalChainStakeIntent).filter((p) => p.intent === intent).filter((p) => bw.addresses.includes(p.from));
+    const stakeIntents = filterAs(payloads, asOptionalChainStakeIntent).filter((p) => p.intent === intent).filter((p) => bw.addresses.includes(p.from));
     return stakeIntents;
   }))).flat();
 }, "getBlockSignedStakeDeclarations");
@@ -932,12 +969,12 @@ var mapBoundWitnessToStakeIntentHashes = /* @__PURE__ */ __name((bw) => {
 }, "mapBoundWitnessToStakeIntentHashes");
 
 // src/StakeIntent/XyoStakeIntentService.ts
-import { filterAs as filterAs3 } from "@xylabs/array";
+import { filterAs as filterAs2 } from "@xylabs/array";
 import { assertEx as assertEx9 } from "@xylabs/assert";
 import { asAddress } from "@xylabs/hex";
 import { analyzeChain as analyzeChain3, ChainStakeIntentAnalyzer, isChainSummaryStakeIntent } from "@xyo-network/chain-analyze";
 import { DEFAULT_FIND_FIRST_MATCHING_NEXT_OPTIONS, findFirstMatching, IntervalMap } from "@xyo-network/chain-utils";
-import { PayloadBuilder as PayloadBuilder4 } from "@xyo-network/payload-builder";
+import { PayloadBuilder as PayloadBuilder5 } from "@xyo-network/payload-builder";
 import { asOptionalBlockBoundWitness, asOptionalBlockBoundWitnessWithStorageMeta, asOptionalChainIndexingServiceStateWithStorageMeta, asOptionalChainStakeIntent as asOptionalChainStakeIntent2, ChainIndexingServiceStateSchema, isChainIndexingServiceState } from "@xyo-network/xl1-protocol";
 import { Mutex as Mutex3 } from "async-mutex";
 import { LRUCache } from "lru-cache";
@@ -993,7 +1030,7 @@ var XyoStakeIntentService = class extends BaseService {
   }
   async createHandler() {
     const head = await this.chainIterator.head();
-    const headHash = await PayloadBuilder4.hash(head);
+    const headHash = await PayloadBuilder5.hash(head);
     if (head?.block === void 0) return;
     await this.recoverState(headHash);
     await this.updateIndex(true);
@@ -1048,7 +1085,7 @@ var XyoStakeIntentService = class extends BaseService {
   }
   async persistState(current) {
     const state = this._producers.serialize();
-    const payload = new PayloadBuilder4({
+    const payload = new PayloadBuilder5({
       schema: ChainIndexingServiceStateSchema
     }).fields({
       endBlockHash: current,
@@ -1099,11 +1136,11 @@ var XyoStakeIntentService = class extends BaseService {
     await this._updateMutex.runExclusive(async () => {
       return await this.spanAsync("updateIndex", async () => {
         const currentHead = await this.chainIterator.head();
-        const currentHeadHash = await PayloadBuilder4.hash(currentHead);
+        const currentHeadHash = await PayloadBuilder5.hash(currentHead);
         const result = await analyzeChain3(this.chainArchivist, [
           new ChainStakeIntentAnalyzer("producer")
         ], currentHeadHash, this._lastIndexedBlockHash);
-        const signedDeclarations = filterAs3(result.find(isChainSummaryStakeIntent)?.intents ?? [], asOptionalChainStakeIntent2);
+        const signedDeclarations = filterAs2(result.find(isChainSummaryStakeIntent)?.intents ?? [], asOptionalChainStakeIntent2);
         if (currentHead.block === void 0) return;
         const currentHeadBlockNum = currentHead.block;
         if (displayProgress) this.logger?.info(`Updating index through 0x${currentHeadBlockNum}`);
@@ -1224,7 +1261,7 @@ var EvmChainService = class extends Base {
 import { assertEx as assertEx10 } from "@xylabs/assert";
 import { BaseEmitter as BaseEmitter2 } from "@xylabs/events";
 import { isDefined as isDefined2, isNull } from "@xylabs/typeof";
-import { PayloadBuilder as PayloadBuilder5 } from "@xyo-network/payload-builder";
+import { PayloadBuilder as PayloadBuilder6 } from "@xyo-network/payload-builder";
 import { asBlockBoundWitness as asBlockBoundWitness2, asOptionalBlockBoundWitness as asOptionalBlockBoundWitness2, isBlockBoundWitness } from "@xyo-network/xl1-protocol";
 import { LRUCache as LRUCache2 } from "lru-cache";
 var XyoChainBlockNumberIterator = class _XyoChainBlockNumberIterator extends BaseEmitter2 {
@@ -1261,7 +1298,7 @@ var XyoChainBlockNumberIterator = class _XyoChainBlockNumberIterator extends Bas
     const cached = this._blocksByBlockNumber.get(block);
     if (cached) return cached;
     const startingBlock = this._head;
-    const currentBlockHash = await PayloadBuilder5.hash(startingBlock);
+    const currentBlockHash = await PayloadBuilder6.hash(startingBlock);
     let currentBlock = (await this._chainArchivist.get([
       currentBlockHash
     ])).at(0);
@@ -1304,7 +1341,7 @@ var XyoChainBlockNumberIterator = class _XyoChainBlockNumberIterator extends Bas
         ]);
         currentBlock = asOptionalBlockBoundWitness2(nextBlock[0]);
       } else {
-        const hash = PayloadBuilder5.hash(currentBlock);
+        const hash = PayloadBuilder6.hash(currentBlock);
         assertEx10(asBlockBoundWitness2(currentBlock), () => `Expected hash to be a block bound witness [${hash}]`);
       }
     }