@aztec/pxe 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7

package/src/storage/tagging_store/sender_tagging_store.ts

@@ -1,10 +1,15 @@
-import { toArray } from '@aztec/foundation/iterable';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
-import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
-import { TxHash } from '@aztec/stdlib/tx';
+import { AppTaggingSecret, SiloedTag, type TaggingIndexRange } from '@aztec/stdlib/logs';
+import { TxEffect, TxHash } from '@aztec/stdlib/tx';
 
 import type { StagedStore } from '../../job_coordinator/job_coordinator.js';
-import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js';
+import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, unfinalizedTaggingIndexesWindowEnd } from '../../tagging/constants.js';
+
+/** A tx still awaiting finalization, and the highest tagging index it used for one secret. */
+export type PendingTx = { txHash: string; highestIndex: number };
+
+/** Internal representation of a pending index range entry. */
+type PendingIndexesEntry = PendingTx & { lowestIndex: number };
 
 /**
  * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class
@@ -16,20 +21,19 @@ export class SenderTaggingStore implements StagedStore {
 
   #store: AztecAsyncKVStore;
 
-  // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained
-  // the private logs with tags corresponding to these indexes has not been finalized yet.
-  //
-  // We don't store just the highest index because if their transaction is dropped we'd then need the information about
-  // the lower pending indexes. For each secret-tx pair however we only store the largest index used in that tx, since
-  // the smaller ones are irrelevant due to tx atomicity.
+  // Stores the pending index ranges for each directional app tagging secret. Pending here means that the tx that
+  // contained the private logs with tags corresponding to these indexes has not been finalized yet.
   //
-  // TODO(#17615): This assumes no logs are used in the non-revertible phase.
+  // We store the full range (lowestIndex, highestIndex) for each secret-tx pair because transactions can partially
+  // revert, in which case only some logs (from the non-revertible phase) survive onchain. By storing the range,
+  // we can expand it and check each individual siloed tag against the TxEffect to determine which indexes made it
+  // onchain.
   //
-  // directional app tagging secret => { pending index, txHash }[]
-  #pendingIndexes: AztecAsyncMap<string, { index: number; txHash: string }[]>;
+  // directional app tagging secret => { lowestIndex, highestIndex, txHash }[]
+  #pendingIndexes: AztecAsyncMap<string, PendingIndexesEntry[]>;
 
-  // jobId => directional app tagging secret => { pending index, txHash }[]
-  #pendingIndexesForJob: Map<string, Map<string, { index: number; txHash: string }[]>>;
+  // jobId => directional app tagging secret => { lowestIndex, highestIndex, txHash }[]
+  #pendingIndexesForJob: Map<string, Map<string, PendingIndexesEntry[]>>;
 
   // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last
   // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence
@@ -51,7 +55,7 @@ export class SenderTaggingStore implements StagedStore {
     this.#lastFinalizedIndexesForJob = new Map();
   }
 
-  #getPendingIndexesForJob(jobId: string): Map<string, { index: number; txHash: string }[]> {
+  #getPendingIndexesForJob(jobId: string): Map<string, PendingIndexesEntry[]> {
     let pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId);
     if (!pendingIndexesForJob) {
       pendingIndexesForJob = new Map();
@@ -69,32 +73,24 @@ export class SenderTaggingStore implements StagedStore {
     return jobStagedLastFinalizedIndexes;
   }
 
-  async #readPendingIndexes(jobId: string, secret: string): Promise<{ index: number; txHash: string }[]> {
-    const jobStagedPendingIndexes = this.#getPendingIndexesForJob(jobId);
-    const pendingIndexes = jobStagedPendingIndexes.get(secret);
-    // We return the staged version of this if it exists, if not, we read from the DB.
-    // If the DB also has nothing, we return an empty array [].
-    return pendingIndexes !== undefined ? pendingIndexes : ((await this.#pendingIndexes.getAsync(secret)) ?? []);
+  async #readPendingIndexes(jobId: string, secret: string): Promise<PendingIndexesEntry[]> {
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#pendingIndexes.getAsync(secret);
+    const staged = this.#getPendingIndexesForJob(jobId).get(secret);
+    return staged !== undefined ? staged : (dbValue ?? []);
   }
 
-  #writePendingIndexes(jobId: string, secret: string, pendingIndexes: { index: number; txHash: string }[]) {
+  #writePendingIndexes(jobId: string, secret: string, pendingIndexes: PendingIndexesEntry[]) {
     this.#getPendingIndexesForJob(jobId).set(secret, pendingIndexes);
   }
 
-  /**
-   * Returns a job view of all the secrets that have a corresponding list of pending indexes either in persistent
-   * storage or the current job.
-   */
-  async #readSecretsWithPendingIndexes(jobId: string): Promise<string[]> {
-    const storedSecrets = new Set(await toArray(this.#pendingIndexes.keysAsync()));
-    const stagedSecrets = this.#getPendingIndexesForJob(jobId).keys();
-    return [...storedSecrets.union(new Set(stagedSecrets))];
-  }
-
   async #readLastFinalizedIndex(jobId: string, secret: string): Promise<number | undefined> {
-    return (
-      this.#getLastFinalizedIndexesForJob(jobId).get(secret) ?? (await this.#lastFinalizedIndexes.getAsync(secret))
-    );
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#lastFinalizedIndexes.getAsync(secret);
+    const staged = this.#getLastFinalizedIndexesForJob(jobId).get(secret);
+    return staged ?? dbValue;
   }
 
   #writeLastFinalizedIndex(jobId: string, secret: string, lastFinalizedIndex: number) {
@@ -135,98 +131,144 @@ export class SenderTaggingStore implements StagedStore {
   }
 
   /**
-   * Stores pending indexes.
-   * @remarks Ignores the index if the same preTag + txHash combination already exists in the db with the same index.
-   * This is expected to happen because whenever we start sync we start from the last finalized index and we can have
-   * pending indexes already stored from previous syncs.
-   * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be
-   * stored in the db.
-   * @param txHash - The tx in which the pretags were used in private logs.
+   * Stores pending index ranges, rejecting any range that disagrees with an already-stored one.
+   * @remarks If the same (secret, txHash) pair already exists in the db with an equal range, it's a no-op. This is
+   * expected to happen because whenever we start sync we start from the last finalized index and we can have pending
+   * ranges already stored from previous syncs. If the ranges differ, it throws an error as that indicates a bug in
+   * callers that record indexes at prove time. Discovery from onchain logs must use `mergePendingIndexes` instead.
+   * @param ranges - The tagging index ranges containing the directional app tagging secrets and the index ranges that are
+   * to be stored in the db.
+   * @param txHash - The tx in which the tagging indexes were used in private logs.
    * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details.
-   * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care
-   * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch
-   * bugs.
-   * @throws If the newly stored pending index is further than window length from the highest finalized index for the
-   * same secret. This is enforced in order to give a guarantee to a recipient that he doesn't need to look further than
-   * window length ahead of the highest finalized index.
-   * @throws If a secret + txHash pair already exists in the db with a different index value. It should never happen
-   * that we would attempt to store a different index for a given secret-txHash pair because we always store just the
-   * highest index for a given secret-txHash pair. Hence this is a good way to catch bugs.
-   * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret.
-   * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher
-   * indexes than finalized ones.
+   * @throws If the highestIndex is further than window length from the highest finalized index for the same secret.
+   * @throws If the lowestIndex is lower than or equal to the last finalized index for the same secret.
+   * @throws If a different range already exists for the same (secret, txHash) pair.
    */
-  async storePendingIndexes(preTags: PreTag[], txHash: TxHash, jobId: string) {
-    // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash
-    // pair. Below we check that this is the case.
-    const secretsSet = new Set(preTags.map(preTag => preTag.secret.toString()));
-    if (secretsSet.size !== preTags.length) {
-      throw new Error(`Duplicate secrets found when storing pending indexes`);
-    }
+  storePendingIndexes(ranges: TaggingIndexRange[], txHash: TxHash, jobId: string): Promise<void> {
+    return this.#storePendingIndexes(ranges, txHash, jobId, false);
+  }
 
-    for (const { secret, index } of preTags) {
-      // First we check that for any secret the highest used index in tx is not further than window length from
-      // the highest finalized index.
-      const finalizedIndex = (await this.getLastFinalizedIndex(secret, jobId)) ?? 0;
-      if (index > finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) {
-        throw new Error(
-          `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex}.
-          Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team
-          to increase it!`,
-        );
-      }
+  /**
+   * Stores pending index ranges, widening an existing entry for the same (secret, txHash) pair to the union of the
+   * stored and incoming ranges instead of throwing on a mismatch. Discovery from onchain logs needs this: it may see
+   * only the surviving (non-revertible phase) sub-range of a partially reverted tx recorded at prove time (the
+   * finalized receipt step of the sync resolves that difference), or indexes beyond a partially discovered entry
+   * when a tx from another PXE straddles a sync window boundary. Callers that record indexes at prove time must use
+   * `storePendingIndexes` instead, so that a range disagreement surfaces as a bug rather than being absorbed.
+   * @param ranges - The tagging index ranges containing the directional app tagging secrets and the index ranges that are
+   * to be stored in the db.
+   * @param txHash - The tx in which the tagging indexes were used in private logs.
+   * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details.
+   * @throws If the highestIndex is further than window length from the highest finalized index for the same secret.
+   * @throws If the lowestIndex is lower than or equal to the last finalized index for the same secret.
+   */
+  mergePendingIndexes(ranges: TaggingIndexRange[], txHash: TxHash, jobId: string): Promise<void> {
+    return this.#storePendingIndexes(ranges, txHash, jobId, true);
+  }
 
-      // Throw if the new pending index is lower than or equal to the last finalized index
-      const secretStr = secret.toString();
-      const lastFinalizedIndex = await this.#readLastFinalizedIndex(jobId, secretStr);
-      if (lastFinalizedIndex !== undefined && index <= lastFinalizedIndex) {
-        throw new Error(
-          `Cannot store pending index ${index} for secret ${secretStr}: ` +
-            `it is lower than or equal to the last finalized index ${lastFinalizedIndex}`,
-        );
-      }
+  #storePendingIndexes(
+    ranges: TaggingIndexRange[],
+    txHash: TxHash,
+    jobId: string,
+    mergeExisting: boolean,
+  ): Promise<void> {
+    if (ranges.length === 0) {
+      return Promise.resolve();
+    }
 
-      // Check if this secret + txHash combination already exists
-      const txHashStr = txHash.toString();
-      const existingForSecret = await this.#readPendingIndexes(jobId, secretStr);
-      const existingForSecretAndTx = existingForSecret.find(entry => entry.txHash === txHashStr);
+    const txHashStr = txHash.toString();
+
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const rangeReadPromises = ranges.map(range => ({
+        range,
+        secretStr: range.extendedSecret.toString(),
+        pending: this.#readPendingIndexes(jobId, range.extendedSecret.toString()),
+        finalized: this.#readLastFinalizedIndex(jobId, range.extendedSecret.toString()),
+      }));
+
+      // Await all reads together
+      const rangeData = await Promise.all(
+        rangeReadPromises.map(async item => ({
+          ...item,
+          pendingData: await item.pending,
+          finalizedIndex: await item.finalized,
+        })),
+      );
+
+      // Process in memory and validate
+      for (const { range, secretStr, pendingData, finalizedIndex } of rangeData) {
+        const windowEnd = unfinalizedTaggingIndexesWindowEnd(finalizedIndex);
+        if (range.highestIndex >= windowEnd) {
+          throw windowExceededError(range.highestIndex, windowEnd, finalizedIndex);
+        }
+
+        // Throw if the lowest index is lower than or equal to the last finalized index
+        if (finalizedIndex !== undefined && range.lowestIndex <= finalizedIndex) {
+          throw new Error(
+            `Cannot store pending index range [${range.lowestIndex}, ${range.highestIndex}] for secret ${secretStr}: ` +
+              `lowestIndex is lower than or equal to the last finalized index ${finalizedIndex}`,
+          );
+        }
 
-      if (existingForSecretAndTx) {
-        // If it exists with a different index, throw an error
-        if (existingForSecretAndTx.index !== index) {
+        // Check if an entry with the same txHash already exists
+        const existingEntry = pendingData.find(entry => entry.txHash === txHashStr);
+
+        let updatedPending: PendingIndexesEntry[] | undefined;
+        if (!existingEntry) {
+          updatedPending = [
+            ...pendingData,
+            { lowestIndex: range.lowestIndex, highestIndex: range.highestIndex, txHash: txHashStr },
+          ];
+        } else if (mergeExisting) {
+          // Widen the entry to the union of both ranges, never shrink it: replacing would drop prove-time
+          // indexes the chain doesn't show (partially reverted tx), and skipping would drop onchain indexes
+          // discovered in a later sync window (tx straddling the window boundary).
+          const lowestIndex = Math.min(existingEntry.lowestIndex, range.lowestIndex);
+          const highestIndex = Math.max(existingEntry.highestIndex, range.highestIndex);
+          if (lowestIndex !== existingEntry.lowestIndex || highestIndex !== existingEntry.highestIndex) {
+            updatedPending = pendingData.map(entry =>
+              entry === existingEntry ? { lowestIndex, highestIndex, txHash: entry.txHash } : entry,
+            );
+          }
+        } else if (
+          existingEntry.lowestIndex !== range.lowestIndex ||
+          existingEntry.highestIndex !== range.highestIndex
+        ) {
+          // Different ranges for the same (secret, txHash) indicate a bug in callers that record indexes at prove
+          // time.
           throw new Error(
-            `Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` +
-              `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`,
+            `Conflicting range for secret ${secretStr} and txHash ${txHashStr}: ` +
+              `existing [${existingEntry.lowestIndex}, ${existingEntry.highestIndex}] vs ` +
+              `new [${range.lowestIndex}, ${range.highestIndex}]`,
           );
         }
-        // If it exists with the same index, ignore the update (no-op)
-      } else {
-        // If it doesn't exist, add it
-        this.#writePendingIndexes(jobId, secretStr, [...existingForSecret, { index, txHash: txHashStr }]);
+        // Remaining cases (a merge whose union equals the stored range, or an identical range without
+        // mergeExisting): duplicate evidence, nothing to write.
+
+        if (updatedPending) {
+          this.#writePendingIndexes(jobId, secretStr, updatedPending);
+        }
       }
-    }
+    });
   }
 
   /**
-   * Returns the transaction hashes of all pending transactions that contain indexes within a specified range
-   * for a given directional app tagging secret.
+   * Returns the pending txs whose highest index falls within [startIndex, endIndex) for a given directional app
+   * tagging secret. The highest index is what decides whether a tx belongs to the window, so it alone is matched
+   * against the bounds, and it is also the only index a caller needs: a tx whose highest index is onchain has every
+   * lower one onchain too. A secret holds at most one entry per tx hash, so no tx hash appears twice in the result.
    * @param secret - The directional app tagging secret to query pending indexes for.
    * @param startIndex - The lower bound of the index range (inclusive).
    * @param endIndex - The upper bound of the index range (exclusive).
-   * @returns An array of unique transaction hashes for pending transactions that contain indexes in the range
-   * [startIndex, endIndex). Returns an empty array if no pending indexes exist in the range.
    */
-  async getTxHashesOfPendingIndexes(
-    secret: DirectionalAppTaggingSecret,
-    startIndex: number,
-    endIndex: number,
-    jobId: string,
-  ): Promise<TxHash[]> {
-    const existing = await this.#readPendingIndexes(jobId, secret.toString());
-    const txHashes = existing
-      .filter(entry => entry.index >= startIndex && entry.index < endIndex)
-      .map(entry => entry.txHash);
-    return Array.from(new Set(txHashes)).map(TxHash.fromString);
+  getPendingTxs(secret: AppTaggingSecret, startIndex: number, endIndex: number, jobId: string): Promise<PendingTx[]> {
+    return this.#store.transactionAsync(async () => {
+      const existing = await this.#readPendingIndexes(jobId, secret.toString());
+      return existing
+        .filter(entry => entry.highestIndex >= startIndex && entry.highestIndex < endIndex)
+        .map(entry => ({ txHash: entry.txHash, highestIndex: entry.highestIndex }));
+    });
   }
 
   /**
@@ -234,8 +276,8 @@ export class SenderTaggingStore implements StagedStore {
    * @param secret - The secret to get the last finalized index for.
    * @returns The last (highest) finalized index for the given secret.
    */
-  getLastFinalizedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
-    return this.#readLastFinalizedIndex(jobId, secret.toString());
+  getLastFinalizedIndex(secret: AppTaggingSecret, jobId: string): Promise<number | undefined> {
+    return this.#store.transactionAsync(() => this.#readLastFinalizedIndex(jobId, secret.toString()));
   }
 
   /**
@@ -244,102 +286,263 @@ export class SenderTaggingStore implements StagedStore {
    * @param secret - The directional app tagging secret to query the last used index for.
    * @returns The last used index.
    */
-  async getLastUsedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
+  getLastUsedIndex(secret: AppTaggingSecret, jobId: string): Promise<number | undefined> {
     const secretStr = secret.toString();
-    const pendingTxScopedIndexes = await this.#readPendingIndexes(jobId, secretStr);
-    const pendingIndexes = pendingTxScopedIndexes.map(entry => entry.index);
 
-    if (pendingTxScopedIndexes.length === 0) {
-      return this.#readLastFinalizedIndex(jobId, secretStr);
-    }
+    return this.#store.transactionAsync(async () => {
+      const pendingPromise = this.#readPendingIndexes(jobId, secretStr);
+      const finalizedPromise = this.#readLastFinalizedIndex(jobId, secretStr);
+
+      const [pendingEntries, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]);
 
-    // As the last used index we return the highest one from the pending indexes. Note that this value will be always
-    // higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
-    return Math.max(...pendingIndexes);
+      if (pendingEntries.length === 0) {
+        return lastFinalized;
+      }
+
+      // As the last used index we return the highest one from the pending index ranges. Note that this value will be
+      // always higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
+      return Math.max(...pendingEntries.map(entry => entry.highestIndex));
+    });
   }
 
   /**
    * Drops all pending indexes corresponding to the given transaction hashes.
    */
-  async dropPendingIndexes(txHashes: TxHash[], jobId: string) {
+  dropPendingIndexes(txHashes: TxHash[], jobId: string): Promise<void> {
     if (txHashes.length === 0) {
-      return;
+      return Promise.resolve();
     }
 
     const txHashStrings = new Set<string>(txHashes.map(txHash => txHash.toString()));
-    const allSecrets = await this.#readSecretsWithPendingIndexes(jobId);
-
-    for (const secret of allSecrets) {
-      const pendingData = await this.#readPendingIndexes(jobId, secret);
-      if (pendingData) {
-        const filtered = pendingData.filter(item => !txHashStrings.has(item.txHash));
-        if (filtered.length === 0) {
-          this.#writePendingIndexes(jobId, secret, []);
-        } else if (filtered.length !== pendingData.length) {
-          // Some items were filtered out, so update the pending data
-          this.#writePendingIndexes(jobId, secret, filtered);
+
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const secretReadPromises: Map<string, Promise<PendingIndexesEntry[]>> = new Map();
+
+      for await (const secret of this.#pendingIndexes.keysAsync()) {
+        secretReadPromises.set(secret, this.#readPendingIndexes(jobId, secret));
+      }
+
+      // Add staged-only secrets (sync, no DB)
+      for (const secret of this.#getPendingIndexesForJob(jobId).keys()) {
+        if (!secretReadPromises.has(secret)) {
+          secretReadPromises.set(secret, Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []));
         }
-        // else: No items were filtered out (txHashes not found for this secret) --> no-op
       }
-    }
+
+      // Await all reads together
+      const secrets = [...secretReadPromises.keys()];
+      const pendingDataResults = await Promise.all(secretReadPromises.values());
+
+      // Process in memory
+      for (let i = 0; i < secrets.length; i++) {
+        const secret = secrets[i];
+        const pendingData = pendingDataResults[i];
+
+        if (pendingData && pendingData.length > 0) {
+          const filtered = pendingData.filter(item => !txHashStrings.has(item.txHash));
+          if (filtered.length === 0) {
+            this.#writePendingIndexes(jobId, secret, []);
+          } else if (filtered.length !== pendingData.length) {
+            // Some items were filtered out, so update the pending data
+            this.#writePendingIndexes(jobId, secret, filtered);
+          }
+          // else: No items were filtered out (txHashes not found for this secret) --> no-op
+        }
+      }
+    });
+  }
+
+  /** Prefetches all pending and finalized index data for every secret (from both DB and staged writes). */
+  #getSecretsWithPendingData(
+    jobId: string,
+  ): Promise<{ secret: string; pendingData: PendingIndexesEntry[]; lastFinalized: number | undefined }[]> {
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const secretDataPromises: Map<
+        string,
+        { pending: Promise<PendingIndexesEntry[]>; finalized: Promise<number | undefined> }
+      > = new Map();
+
+      for await (const secret of this.#pendingIndexes.keysAsync()) {
+        secretDataPromises.set(secret, {
+          pending: this.#readPendingIndexes(jobId, secret),
+          finalized: this.#readLastFinalizedIndex(jobId, secret),
+        });
+      }
+
+      // Add staged-only secrets (sync, no DB)
+      for (const secret of this.#getPendingIndexesForJob(jobId).keys()) {
+        if (!secretDataPromises.has(secret)) {
+          secretDataPromises.set(secret, {
+            pending: Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []),
+            finalized: Promise.resolve(this.#getLastFinalizedIndexesForJob(jobId).get(secret)),
+          });
+        }
+      }
+
+      // Await all reads together
+      const secrets = [...secretDataPromises.keys()];
+      const dataResults = await Promise.all(
+        secrets.map(async secret => ({
+          secret,
+          pendingData: await secretDataPromises.get(secret)!.pending,
+          lastFinalized: await secretDataPromises.get(secret)!.finalized,
+        })),
+      );
+
+      return dataResults.filter(r => r.pendingData.length > 0);
+    });
   }
 
   /**
    * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending
-   * indexes.
+   * indexes. Applies to every secret the txs used, so the caller must hold tx-level evidence that the whole tx
+   * finalized. Callers holding evidence about a single secret must use {@link finalizePendingIndexesOfSecret} instead.
    */
-  async finalizePendingIndexes(txHashes: TxHash[], jobId: string) {
+  finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise<void> {
+    return this.#finalizePendingIndexes(txHashes, jobId);
+  }
+
+  /**
+   * Same as {@link finalizePendingIndexes}, but restricted to the pending indexes of a single secret.
+   *
+   * Finalizing every secret off single-secret evidence would be unsound: a tx whose execution partially reverted can
+   * have all of one secret's tags onchain and none of another's, and the second secret's indexes must not be recorded
+   * as finalized when they never reached the chain.
+   */
+  finalizePendingIndexesOfSecret(secret: AppTaggingSecret, txHashes: TxHash[], jobId: string): Promise<void> {
+    return this.#finalizePendingIndexes(txHashes, jobId, secret.toString());
+  }
+
+  async #finalizePendingIndexes(txHashes: TxHash[], jobId: string, onlySecret?: string): Promise<void> {
     if (txHashes.length === 0) {
       return;
     }
 
-    for (const txHash of txHashes) {
-      const txHashStr = txHash.toString();
-
-      const allSecrets = await this.#readSecretsWithPendingIndexes(jobId);
+    const txHashStrings = new Set(txHashes.map(tx => tx.toString()));
+    const secretsWithData = (await this.#getSecretsWithPendingData(jobId)).filter(
+      ({ secret }) => onlySecret === undefined || secret === onlySecret,
+    );
 
-      for (const secret of allSecrets) {
-        const pendingData = await this.#readPendingIndexes(jobId, secret);
-        if (!pendingData) {
-          continue;
-        }
+    for (const { secret, pendingData, lastFinalized } of secretsWithData) {
+      let currentPending = pendingData;
+      let currentFinalized = lastFinalized;
 
-        const matchingIndexes = pendingData.filter(item => item.txHash === txHashStr).map(item => item.index);
-        if (matchingIndexes.length === 0) {
+      // Process all txHashes for this secret
+      for (const txHashStr of txHashStrings) {
+        const matchingEntries = currentPending.filter(item => item.txHash === txHashStr);
+        if (matchingEntries.length === 0) {
+          // This is expected as a higher index might have already been finalized which would lead to pruning of
+          // pending entries.
           continue;
         }
 
-        if (matchingIndexes.length > 1) {
+        if (matchingEntries.length > 1) {
           // We should always just store the highest pending index for a given tx hash and secret because the lower
           // values are irrelevant.
-          throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`);
+          throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`);
         }
 
-        let lastFinalized = await this.#readLastFinalizedIndex(jobId, secret);
-        const newFinalized = matchingIndexes[0];
+        const newFinalized = matchingEntries[0].highestIndex;
 
-        if (newFinalized < (lastFinalized ?? 0)) {
+        if (newFinalized < (currentFinalized ?? 0)) {
           // This should never happen because when last finalized index was finalized we should have pruned the lower
           // pending indexes.
           throw new Error(
-            `New finalized index ${newFinalized} is smaller than the current last finalized index ${lastFinalized}`,
+            `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`,
           );
         }
 
-        this.#writeLastFinalizedIndex(jobId, secret, newFinalized);
-        lastFinalized = newFinalized;
+        currentFinalized = newFinalized;
 
         // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we
         // cannot control the order in which transactions are finalized, there may be pending indexes that are now
         // obsolete because they are lower than the most recently finalized index. For this reason, we prune these
         // outdated pending indexes.
-        const remainingItemsOfHigherIndex = pendingData.filter(item => item.index > (lastFinalized ?? 0));
-        if (remainingItemsOfHigherIndex.length === 0) {
-          this.#writePendingIndexes(jobId, secret, []);
-        } else {
-          this.#writePendingIndexes(jobId, secret, remainingItemsOfHigherIndex);
+        currentPending = currentPending.filter(item => item.highestIndex > currentFinalized!);
+      }
+
+      // Write final state if changed
+      if (currentFinalized !== lastFinalized) {
+        this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!);
+      }
+      if (currentPending !== pendingData) {
+        this.#writePendingIndexes(jobId, secret, currentPending);
+      }
+    }
+  }
+
+  /**
+   * Handles finalization of pending indexes for a transaction whose execution was partially reverted.
+   * Recomputes the siloed tags for each pending index of the given tx and checks which ones appear in the
+   * TxEffect's private logs (i.e., which ones made it onchain). Those that survived are finalized; those that
+   * didn't are dropped.
+   * @param txEffect - The tx effect of the partially reverted transaction.
+   * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details.
+   */
+  async finalizePendingIndexesOfAPartiallyRevertedTx(txEffect: TxEffect, jobId: string): Promise<void> {
+    const txHashStr = txEffect.txHash.toString();
+
+    // Build a set of all siloed tag values that made it onchain (first field of each private log).
+    const onChainTags = new Set<string>(txEffect.privateLogs.map(log => log.fields[0].toString()));
+
+    const secretsWithData = await this.#getSecretsWithPendingData(jobId);
+
+    for (const { secret, pendingData, lastFinalized } of secretsWithData) {
+      const matchingEntries = pendingData.filter(item => item.txHash === txHashStr);
+      if (matchingEntries.length === 0) {
+        // This is expected as a higher index might have already been finalized which would lead to pruning of
+        // pending entries.
+        continue;
+      }
+
+      if (matchingEntries.length > 1) {
+        // We should always just store the highest pending index for a given tx hash and secret because the lower
+        // values are irrelevant.
+        throw new Error(`Multiple pending entries found for tx hash ${txHashStr} and secret ${secret}`);
+      }
+
+      const pendingEntry = matchingEntries[0];
+
+      // Expand each matching entry's range and recompute siloed tags for each index.
+      const appTaggingSecret = AppTaggingSecret.fromString(secret);
+      let highestSurvivingIndex: number | undefined;
+
+      for (let index = pendingEntry.lowestIndex; index <= pendingEntry.highestIndex; index++) {
+        const siloedTag = await SiloedTag.compute({ extendedSecret: appTaggingSecret, index });
+        if (onChainTags.has(siloedTag.value.toString())) {
+          highestSurvivingIndex = highestSurvivingIndex !== undefined ? Math.max(highestSurvivingIndex, index) : index;
         }
       }
+
+      // Remove all entries for this txHash from pending (both surviving and non-surviving).
+      let currentPending = pendingData.filter(item => item.txHash !== txHashStr);
+
+      if (highestSurvivingIndex !== undefined) {
+        const newFinalized = Math.max(lastFinalized ?? 0, highestSurvivingIndex);
+        this.#writeLastFinalizedIndex(jobId, secret, newFinalized);
+
+        // Prune pending indexes that are now <= the finalized index.
+        currentPending = currentPending.filter(item => item.highestIndex > newFinalized);
+      }
+
+      this.#writePendingIndexes(jobId, secret, currentPending);
     }
   }
 }
+
+/** Builds the error thrown when a pending tag index is at or past the unfinalized tagging window end. */
+export function windowExceededError(
+  highestIndex: number,
+  windowEnd: number,
+  finalizedIndex: number | undefined,
+): Error {
+  const finalizedDescription =
+    finalizedIndex === undefined ? 'no index finalized yet' : `highest finalized index ${finalizedIndex}`;
+  return new Error(
+    `Highest used index ${highestIndex} is at or past the window end ${windowEnd} (${finalizedDescription}). ` +
+      `Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. ` +
+      `Contact the Aztec team to increase it!`,
+  );
+}