@aztec/pxe 0.0.1-commit.54489865 → 0.0.1-commit.592b9384

package/src/storage/private_event_store/private_event_store.ts

@@ -1,6 +1,5 @@
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
-import { toArray } from '@aztec/foundation/iterable';
 import { createLogger } from '@aztec/foundation/log';
 import { Semaphore } from '@aztec/foundation/queue';
 import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
@@ -81,43 +80,45 @@ export class PrivateEventStore implements StagedStore {
     metadata: PrivateEventMetadata,
     jobId: string,
   ) {
-    return this.#withJobLock(jobId, async () => {
-      const { contractAddress, scope, txHash, l2BlockNumber, l2BlockHash, txIndexInBlock, eventIndexInTx } = metadata;
-      const eventId = siloedEventCommitment.toString();
-
-      this.logger.verbose('storing private event log (job stage)', {
-        eventId,
-        contractAddress,
-        scope,
-        msgContent,
-        l2BlockNumber,
-      });
-
-      const existing = await this.#readEvent(eventId, jobId);
+    return this.#withJobLock(jobId, () =>
+      this.#store.transactionAsync(async () => {
+        const { contractAddress, scope, txHash, l2BlockNumber, l2BlockHash, txIndexInBlock, eventIndexInTx } = metadata;
+        const eventId = siloedEventCommitment.toString();
 
-      if (existing) {
-        // If we already stored this event, we still want to make sure to track it for the given scope
-        existing.addScope(scope.toString());
-        this.#writeEvent(eventId, existing, jobId);
-      } else {
-        this.#writeEvent(
+        this.logger.verbose('storing private event log (job stage)', {
           eventId,
-          new StoredPrivateEvent(
-            randomness,
-            msgContent,
-            l2BlockNumber,
-            l2BlockHash,
-            txHash,
-            txIndexInBlock,
-            eventIndexInTx,
-            contractAddress,
-            eventSelector,
-            new Set([scope.toString()]),
-          ),
-          jobId,
-        );
-      }
-    });
+          contractAddress,
+          scope,
+          msgContent,
+          l2BlockNumber,
+        });
+
+        const existing = await this.#readEvent(eventId, jobId);
+
+        if (existing) {
+          // If we already stored this event, we still want to make sure to track it for the given scope
+          existing.addScope(scope.toString());
+          this.#writeEvent(eventId, existing, jobId);
+        } else {
+          this.#writeEvent(
+            eventId,
+            new StoredPrivateEvent(
+              randomness,
+              msgContent,
+              l2BlockNumber,
+              l2BlockHash,
+              txHash,
+              txIndexInBlock,
+              eventIndexInTx,
+              contractAddress,
+              eventSelector,
+              new Set([scope.toString()]),
+            ),
+            jobId,
+          );
+        }
+      }),
+    );
   }
 
   /**
@@ -131,76 +132,88 @@ export class PrivateEventStore implements StagedStore {
    * @returns - The event log contents, augmented with metadata about the transaction and block in which the event was
    * included.
    */
-  public async getPrivateEvents(
+  public getPrivateEvents(
     eventSelector: EventSelector,
     filter: PrivateEventStoreFilter,
   ): Promise<PackedPrivateEvent[]> {
-    const events: Array<{
-      l2BlockNumber: number;
-      txIndexInBlock: number;
-      eventIndexInTx: number;
-      event: PackedPrivateEvent;
-    }> = [];
+    return this.#store.transactionAsync(async () => {
+      const key = this.#keyFor(filter.contractAddress, eventSelector);
+      const targetScopes = new Set(filter.scopes.map(s => s.toString()));
 
-    const key = this.#keyFor(filter.contractAddress, eventSelector);
-    const targetScopes = new Set(filter.scopes.map(s => s.toString()));
+      // Map from eventId to the promise that reads the event buffer.
+      // We start reads during iteration to keep DB requests pending and avoid IndexedDB auto-commit.
+      const eventReadPromises: Map<string, Promise<Buffer | undefined>> = new Map();
 
-    const eventIds: string[] = await toArray(this.#eventsByContractAndEventSelector.getValuesAsync(key));
+      for await (const eventId of this.#eventsByContractAndEventSelector.getValuesAsync(key)) {
+        eventReadPromises.set(eventId, this.#events.getAsync(eventId));
+      }
 
-    for (const eventId of eventIds) {
-      const eventBuffer = await this.#events.getAsync(eventId);
+      const eventIds = [...eventReadPromises.keys()];
+      const eventBuffers = await Promise.all(eventReadPromises.values());
 
-      // Defensive, if it happens, there's a problem with how we're handling #eventsByContractAndEventSelector
-      if (!eventBuffer) {
-        this.logger.verbose(
-          `EventId ${eventId} does not exist in main index but it is referenced from contract event selector index`,
-        );
-        continue;
-      }
+      const events: Array<{
+        l2BlockNumber: number;
+        txIndexInBlock: number;
+        eventIndexInTx: number;
+        event: PackedPrivateEvent;
+      }> = [];
 
-      const storedPrivateEvent = StoredPrivateEvent.fromBuffer(eventBuffer);
+      for (let i = 0; i < eventIds.length; i++) {
+        const eventId = eventIds[i];
+        const eventBuffer = eventBuffers[i];
 
-      // Filter by block range
-      if (storedPrivateEvent.l2BlockNumber < filter.fromBlock || storedPrivateEvent.l2BlockNumber >= filter.toBlock) {
-        continue;
-      }
+        // Defensive, if it happens, there's a problem with how we're handling #eventsByContractAndEventSelector
+        if (!eventBuffer) {
+          this.logger.verbose(
+            `EventId ${eventId} does not exist in main index but it is referenced from contract event selector index`,
+          );
+          continue;
+        }
 
-      // Filter by scopes
-      if (storedPrivateEvent.scopes.intersection(targetScopes).size === 0) {
-        continue;
-      }
+        const storedPrivateEvent = StoredPrivateEvent.fromBuffer(eventBuffer);
+
+        // Filter by block range
+        if (storedPrivateEvent.l2BlockNumber < filter.fromBlock || storedPrivateEvent.l2BlockNumber >= filter.toBlock) {
+          continue;
+        }
 
-      // Filter by txHash
-      if (filter.txHash && !storedPrivateEvent.txHash.equals(filter.txHash)) {
-        continue;
+        // Filter by scopes
+        if (storedPrivateEvent.scopes.intersection(targetScopes).size === 0) {
+          continue;
+        }
+
+        // Filter by txHash
+        if (filter.txHash && !storedPrivateEvent.txHash.equals(filter.txHash)) {
+          continue;
+        }
+
+        events.push({
+          l2BlockNumber: storedPrivateEvent.l2BlockNumber,
+          txIndexInBlock: storedPrivateEvent.txIndexInBlock,
+          eventIndexInTx: storedPrivateEvent.eventIndexInTx,
+          event: {
+            packedEvent: storedPrivateEvent.msgContent,
+            l2BlockNumber: BlockNumber(storedPrivateEvent.l2BlockNumber),
+            txHash: storedPrivateEvent.txHash,
+            l2BlockHash: storedPrivateEvent.l2BlockHash,
+            eventSelector,
+          },
+        });
       }
 
-      events.push({
-        l2BlockNumber: storedPrivateEvent.l2BlockNumber,
-        txIndexInBlock: storedPrivateEvent.txIndexInBlock,
-        eventIndexInTx: storedPrivateEvent.eventIndexInTx,
-        event: {
-          packedEvent: storedPrivateEvent.msgContent,
-          l2BlockNumber: BlockNumber(storedPrivateEvent.l2BlockNumber),
-          txHash: storedPrivateEvent.txHash,
-          l2BlockHash: storedPrivateEvent.l2BlockHash,
-          eventSelector,
-        },
+      // Sort by block number, then by tx index within block, then by event index within tx
+      events.sort((a, b) => {
+        if (a.l2BlockNumber !== b.l2BlockNumber) {
+          return a.l2BlockNumber - b.l2BlockNumber;
+        }
+        if (a.txIndexInBlock !== b.txIndexInBlock) {
+          return a.txIndexInBlock - b.txIndexInBlock;
+        }
+        return a.eventIndexInTx - b.eventIndexInTx;
       });
-    }
 
-    // Sort by block number, then by tx index within block, then by event index within tx
-    events.sort((a, b) => {
-      if (a.l2BlockNumber !== b.l2BlockNumber) {
-        return a.l2BlockNumber - b.l2BlockNumber;
-      }
-      if (a.txIndexInBlock !== b.txIndexInBlock) {
-        return a.txIndexInBlock - b.txIndexInBlock;
-      }
-      return a.eventIndexInTx - b.eventIndexInTx;
+      return events.map(ev => ev.event);
     });
-
-    return events.map(ev => ev.event);
   }
 
   /**
@@ -221,29 +234,39 @@ export class PrivateEventStore implements StagedStore {
    * IMPORTANT: This method must be called within a transaction to ensure atomicity.
    */
   public async rollback(blockNumber: number, synchedBlockNumber: number): Promise<void> {
-    let removedCount = 0;
+    // First pass: collect all event IDs for all blocks, starting reads during iteration to keep tx alive.
+    const eventsByBlock: Map<number, { eventId: string; eventReadPromise: Promise<Buffer | undefined> }[]> = new Map();
 
     for (let block = blockNumber + 1; block <= synchedBlockNumber; block++) {
-      const eventIds: string[] = await toArray(this.#eventsByBlockNumber.getValuesAsync(block));
+      const blockEvents: { eventId: string; eventReadPromise: Promise<Buffer | undefined> }[] = [];
+      for await (const eventId of this.#eventsByBlockNumber.getValuesAsync(block)) {
+        // Start read immediately during iteration to keep IndexedDB transaction alive
+        blockEvents.push({ eventId, eventReadPromise: this.#events.getAsync(eventId) });
+      }
+      if (blockEvents.length > 0) {
+        eventsByBlock.set(block, blockEvents);
+      }
+    }
 
-      if (eventIds.length > 0) {
-        await this.#eventsByBlockNumber.delete(block);
+    // Second pass: await reads and perform deletes
+    let removedCount = 0;
+    for (const [block, events] of eventsByBlock) {
+      await this.#eventsByBlockNumber.delete(block);
 
-        for (const eventId of eventIds) {
-          const buffer = await this.#events.getAsync(eventId);
-          if (!buffer) {
-            throw new Error(`Event not found for eventId ${eventId}`);
-          }
+      for (const { eventId, eventReadPromise } of events) {
+        const buffer = await eventReadPromise;
+        if (!buffer) {
+          throw new Error(`Event not found for eventId ${eventId}`);
+        }
 
-          const entry = StoredPrivateEvent.fromBuffer(buffer);
-          await this.#events.delete(eventId);
-          await this.#eventsByContractAndEventSelector.deleteValue(
-            this.#keyFor(entry.contractAddress, entry.eventSelector),
-            eventId,
-          );
+        const entry = StoredPrivateEvent.fromBuffer(buffer);
+        await this.#events.delete(eventId);
+        await this.#eventsByContractAndEventSelector.deleteValue(
+          this.#keyFor(entry.contractAddress, entry.eventSelector),
+          eventId,
+        );
 
-          removedCount++;
-        }
+        removedCount++;
       }
     }
 
@@ -260,28 +283,30 @@ export class PrivateEventStore implements StagedStore {
    *
    * @param jobId - The jobId identifying which staged data to commit
    */
-  commit(jobId: string): Promise<void> {
-    return this.#withJobLock(jobId, async () => {
-      for (const [eventId, entry] of this.#getEventsForJob(jobId).entries()) {
-        const lookupKey = this.#keyFor(entry.contractAddress, entry.eventSelector);
-        this.logger.verbose('storing private event log', { eventId, lookupKey });
-
-        await Promise.all([
-          this.#events.set(eventId, entry.toBuffer()),
-          this.#eventsByContractAndEventSelector.set(lookupKey, eventId),
-          this.#eventsByBlockNumber.set(entry.l2BlockNumber, eventId),
-        ]);
-      }
+  async commit(jobId: string): Promise<void> {
+    // Note: Don't use #withJobLock here - commit runs within JobCoordinator's transactionAsync,
+    // and awaiting the lock would create a microtask boundary with no pending DB request,
+    // causing IndexedDB to auto-commit the transaction.
+    for (const [eventId, entry] of this.#getEventsForJob(jobId).entries()) {
+      const lookupKey = this.#keyFor(entry.contractAddress, entry.eventSelector);
+      this.logger.verbose('storing private event log', { eventId, lookupKey });
+
+      await Promise.all([
+        this.#events.set(eventId, entry.toBuffer()),
+        this.#eventsByContractAndEventSelector.set(lookupKey, eventId),
+        this.#eventsByBlockNumber.set(entry.l2BlockNumber, eventId),
+      ]);
+    }
 
-      this.#clearJobData(jobId);
-    });
+    this.#clearJobData(jobId);
   }
 
   /**
    * Discards in memory job data without persisting it.
    */
   discardStaged(jobId: string): Promise<void> {
-    return this.#withJobLock(jobId, () => Promise.resolve(this.#clearJobData(jobId)));
+    this.#clearJobData(jobId);
+    return Promise.resolve();
   }
 
   /**
@@ -290,13 +315,11 @@ export class PrivateEventStore implements StagedStore {
    * Returns undefined if the event does not exist in the store overall.
    */
   async #readEvent(eventId: string, jobId: string): Promise<StoredPrivateEvent | undefined> {
-    const eventForJob = this.#getEventsForJob(jobId).get(eventId);
-    if (eventForJob) {
-      return eventForJob;
-    }
-
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
     const buffer = await this.#events.getAsync(eventId);
-    return buffer ? StoredPrivateEvent.fromBuffer(buffer) : undefined;
+    const eventForJob = this.#getEventsForJob(jobId).get(eventId);
+    return eventForJob ?? (buffer ? StoredPrivateEvent.fromBuffer(buffer) : undefined);
   }
 
   /**

package/src/storage/tagging_store/recipient_tagging_store.ts

@@ -45,7 +45,11 @@ export class RecipientTaggingStore implements StagedStore {
   }
 
   async #readHighestAgedIndex(jobId: string, secret: string): Promise<number | undefined> {
-    return this.#getHighestAgedIndexForJob(jobId).get(secret) ?? (await this.#highestAgedIndex.getAsync(secret));
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#highestAgedIndex.getAsync(secret);
+    const staged = this.#getHighestAgedIndexForJob(jobId).get(secret);
+    return staged ?? dbValue;
   }
 
   #writeHighestAgedIndex(jobId: string, secret: string, index: number) {
@@ -62,9 +66,11 @@ export class RecipientTaggingStore implements StagedStore {
   }
 
   async #readHighestFinalizedIndex(jobId: string, secret: string): Promise<number | undefined> {
-    return (
-      this.#getHighestFinalizedIndexForJob(jobId).get(secret) ?? (await this.#highestFinalizedIndex.getAsync(secret))
-    );
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#highestFinalizedIndex.getAsync(secret);
+    const staged = this.#getHighestFinalizedIndexForJob(jobId).get(secret);
+    return staged ?? dbValue;
   }
 
   #writeHighestFinalizedIndex(jobId: string, secret: string, index: number) {
@@ -101,29 +107,33 @@ export class RecipientTaggingStore implements StagedStore {
   }
 
   getHighestAgedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
-    return this.#readHighestAgedIndex(jobId, secret.toString());
+    return this.#store.transactionAsync(() => this.#readHighestAgedIndex(jobId, secret.toString()));
   }
 
-  async updateHighestAgedIndex(secret: DirectionalAppTaggingSecret, index: number, jobId: string): Promise<void> {
-    const currentIndex = await this.#readHighestAgedIndex(jobId, secret.toString());
-    if (currentIndex !== undefined && index <= currentIndex) {
-      // Log sync should never set a lower highest aged index.
-      throw new Error(`New highest aged index (${index}) must be higher than the current one (${currentIndex})`);
-    }
-    this.#writeHighestAgedIndex(jobId, secret.toString(), index);
+  updateHighestAgedIndex(secret: DirectionalAppTaggingSecret, index: number, jobId: string): Promise<void> {
+    return this.#store.transactionAsync(async () => {
+      const currentIndex = await this.#readHighestAgedIndex(jobId, secret.toString());
+      if (currentIndex !== undefined && index <= currentIndex) {
+        // Log sync should never set a lower highest aged index.
+        throw new Error(`New highest aged index (${index}) must be higher than the current one (${currentIndex})`);
+      }
+      this.#writeHighestAgedIndex(jobId, secret.toString(), index);
+    });
   }
 
   getHighestFinalizedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
-    return this.#readHighestFinalizedIndex(jobId, secret.toString());
+    return this.#store.transactionAsync(() => this.#readHighestFinalizedIndex(jobId, secret.toString()));
   }
 
-  async updateHighestFinalizedIndex(secret: DirectionalAppTaggingSecret, index: number, jobId: string): Promise<void> {
-    const currentIndex = await this.#readHighestFinalizedIndex(jobId, secret.toString());
-    if (currentIndex !== undefined && index < currentIndex) {
-      // Log sync should never set a lower highest finalized index but it can happen that it would try to set the same
-      // one because we are loading logs from highest aged index + 1 and not from the highest finalized index.
-      throw new Error(`New highest finalized index (${index}) must be higher than the current one (${currentIndex})`);
-    }
-    this.#writeHighestFinalizedIndex(jobId, secret.toString(), index);
+  updateHighestFinalizedIndex(secret: DirectionalAppTaggingSecret, index: number, jobId: string): Promise<void> {
+    return this.#store.transactionAsync(async () => {
+      const currentIndex = await this.#readHighestFinalizedIndex(jobId, secret.toString());
+      if (currentIndex !== undefined && index < currentIndex) {
+        // Log sync should never set a lower highest finalized index but it can happen that it would try to set the same
+        // one because we are loading logs from highest aged index + 1 and not from the highest finalized index.
+        throw new Error(`New highest finalized index (${index}) must be higher than the current one (${currentIndex})`);
+      }
+      this.#writeHighestFinalizedIndex(jobId, secret.toString(), index);
+    });
   }
 }

package/src/storage/tagging_store/sender_address_book_store.ts

@@ -16,27 +16,33 @@ export class SenderAddressBookStore {
     this.#addressBook = this.#store.openMap('address_book');
   }
 
-  async addSender(address: AztecAddress): Promise<boolean> {
-    if (await this.#addressBook.hasAsync(address.toString())) {
-      return false;
-    }
+  addSender(address: AztecAddress): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      if (await this.#addressBook.hasAsync(address.toString())) {
+        return false;
+      }
 
-    await this.#addressBook.set(address.toString(), true);
+      await this.#addressBook.set(address.toString(), true);
 
-    return true;
+      return true;
+    });
   }
 
-  async getSenders(): Promise<AztecAddress[]> {
-    return (await toArray(this.#addressBook.keysAsync())).map(AztecAddress.fromString);
+  getSenders(): Promise<AztecAddress[]> {
+    return this.#store.transactionAsync(async () => {
+      return (await toArray(this.#addressBook.keysAsync())).map(AztecAddress.fromString);
+    });
   }
 
-  async removeSender(address: AztecAddress): Promise<boolean> {
-    if (!(await this.#addressBook.hasAsync(address.toString()))) {
-      return false;
-    }
+  removeSender(address: AztecAddress): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      if (!(await this.#addressBook.hasAsync(address.toString()))) {
+        return false;
+      }
 
-    await this.#addressBook.delete(address.toString());
+      await this.#addressBook.delete(address.toString());
 
-    return true;
+      return true;
+    });
   }
 }