@aztec/pxe 0.0.1-commit.d3ec352c → 0.0.1-commit.d6f2b3f94

package/src/storage/tagging_store/sender_tagging_store.ts

@@ -0,0 +1,429 @@
+import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
+import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
+import { TxHash } from '@aztec/stdlib/tx';
+
+import type { StagedStore } from '../../job_coordinator/job_coordinator.js';
+import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js';
+
+/**
+ * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class
+ * is called RecipientTaggingStore. We have the data stores separate for sender and recipient because
+ * the algorithms are completely disjoint and there is not data reuse between the two.
+ */
+export class SenderTaggingStore implements StagedStore {
+  readonly storeName = 'sender_tagging';
+
+  #store: AztecAsyncKVStore;
+
+  // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained
+  // the private logs with tags corresponding to these indexes has not been finalized yet.
+  //
+  // We don't store just the highest index because if their transaction is dropped we'd then need the information about
+  // the lower pending indexes. For each secret-tx pair however we only store the largest index used in that tx, since
+  // the smaller ones are irrelevant due to tx atomicity.
+  //
+  // TODO(#17615): This assumes no logs are used in the non-revertible phase.
+  //
+  // directional app tagging secret => { pending index, txHash }[]
+  #pendingIndexes: AztecAsyncMap<string, { index: number; txHash: string }[]>;
+
+  // jobId => directional app tagging secret => { pending index, txHash }[]
+  #pendingIndexesForJob: Map<string, Map<string, { index: number; txHash: string }[]>>;
+
+  // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last
+  // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence
+  // we don't need to store the history.
+  //
+  // directional app tagging secret => highest finalized index
+  #lastFinalizedIndexes: AztecAsyncMap<string, number>;
+
+  // jobId => directional app tagging secret => highest finalized index
+  #lastFinalizedIndexesForJob: Map<string, Map<string, number>>;
+
+  constructor(store: AztecAsyncKVStore) {
+    this.#store = store;
+
+    this.#pendingIndexes = this.#store.openMap('pending_indexes');
+    this.#lastFinalizedIndexes = this.#store.openMap('last_finalized_indexes');
+
+    this.#pendingIndexesForJob = new Map();
+    this.#lastFinalizedIndexesForJob = new Map();
+  }
+
+  #getPendingIndexesForJob(jobId: string): Map<string, { index: number; txHash: string }[]> {
+    let pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId);
+    if (!pendingIndexesForJob) {
+      pendingIndexesForJob = new Map();
+      this.#pendingIndexesForJob.set(jobId, pendingIndexesForJob);
+    }
+    return pendingIndexesForJob;
+  }
+
+  #getLastFinalizedIndexesForJob(jobId: string): Map<string, number> {
+    let jobStagedLastFinalizedIndexes = this.#lastFinalizedIndexesForJob.get(jobId);
+    if (!jobStagedLastFinalizedIndexes) {
+      jobStagedLastFinalizedIndexes = new Map();
+      this.#lastFinalizedIndexesForJob.set(jobId, jobStagedLastFinalizedIndexes);
+    }
+    return jobStagedLastFinalizedIndexes;
+  }
+
+  async #readPendingIndexes(jobId: string, secret: string): Promise<{ index: number; txHash: string }[]> {
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#pendingIndexes.getAsync(secret);
+    const staged = this.#getPendingIndexesForJob(jobId).get(secret);
+    return staged !== undefined ? staged : (dbValue ?? []);
+  }
+
+  #writePendingIndexes(jobId: string, secret: string, pendingIndexes: { index: number; txHash: string }[]) {
+    this.#getPendingIndexesForJob(jobId).set(secret, pendingIndexes);
+  }
+
+  async #readLastFinalizedIndex(jobId: string, secret: string): Promise<number | undefined> {
+    // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+    // are no pending read requests). The staged value still takes precedence if it exists.
+    const dbValue = await this.#lastFinalizedIndexes.getAsync(secret);
+    const staged = this.#getLastFinalizedIndexesForJob(jobId).get(secret);
+    return staged ?? dbValue;
+  }
+
+  #writeLastFinalizedIndex(jobId: string, secret: string, lastFinalizedIndex: number) {
+    this.#getLastFinalizedIndexesForJob(jobId).set(secret, lastFinalizedIndex);
+  }
+
+  /**
+   * Writes all job-specific in-memory data to persistent storage.
+   *
+   * @remark This method must run in a DB transaction context. It's designed to be called from JobCoordinator#commitJob.
+   */
+  async commit(jobId: string): Promise<void> {
+    const pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId);
+    if (pendingIndexesForJob) {
+      for (const [secret, pendingIndexes] of pendingIndexesForJob.entries()) {
+        if (pendingIndexes.length === 0) {
+          await this.#pendingIndexes.delete(secret);
+        } else {
+          await this.#pendingIndexes.set(secret, pendingIndexes);
+        }
+      }
+    }
+
+    const lastFinalizedIndexesForJob = this.#lastFinalizedIndexesForJob.get(jobId);
+    if (lastFinalizedIndexesForJob) {
+      for (const [secret, lastFinalizedIndex] of lastFinalizedIndexesForJob.entries()) {
+        await this.#lastFinalizedIndexes.set(secret, lastFinalizedIndex);
+      }
+    }
+
+    return this.discardStaged(jobId);
+  }
+
+  discardStaged(jobId: string): Promise<void> {
+    this.#pendingIndexesForJob.delete(jobId);
+    this.#lastFinalizedIndexesForJob.delete(jobId);
+    return Promise.resolve();
+  }
+
+  /**
+   * Stores pending indexes.
+   * @remarks Ignores the index if the same preTag + txHash combination already exists in the db with the same index.
+   * This is expected to happen because whenever we start sync we start from the last finalized index and we can have
+   * pending indexes already stored from previous syncs.
+   * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be
+   * stored in the db.
+   * @param txHash - The tx in which the pretags were used in private logs.
+   * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details.
+   * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care
+   * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch
+   * bugs.
+   * @throws If the newly stored pending index is further than window length from the highest finalized index for the
+   * same secret. This is enforced in order to give a guarantee to a recipient that he doesn't need to look further than
+   * window length ahead of the highest finalized index.
+   * @throws If a secret + txHash pair already exists in the db with a different index value. It should never happen
+   * that we would attempt to store a different index for a given secret-txHash pair because we always store just the
+   * highest index for a given secret-txHash pair. Hence this is a good way to catch bugs.
+   * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret.
+   * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher
+   * indexes than finalized ones.
+   */
+  storePendingIndexes(preTags: PreTag[], txHash: TxHash, jobId: string): Promise<void> {
+    if (preTags.length === 0) {
+      return Promise.resolve();
+    }
+
+    // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash
+    // pair. Below we check that this is the case.
+    const secretsSet = new Set(preTags.map(preTag => preTag.secret.toString()));
+    if (secretsSet.size !== preTags.length) {
+      return Promise.reject(new Error(`Duplicate secrets found when storing pending indexes`));
+    }
+
+    const txHashStr = txHash.toString();
+
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const preTagReadPromises = preTags.map(({ secret, index }) => {
+        const secretStr = secret.toString();
+        return {
+          secret,
+          secretStr,
+          index,
+          pending: this.#readPendingIndexes(jobId, secretStr),
+          finalized: this.#readLastFinalizedIndex(jobId, secretStr),
+        };
+      });
+
+      // Await all reads together
+      const preTagData = await Promise.all(
+        preTagReadPromises.map(async item => ({
+          ...item,
+          pendingData: await item.pending,
+          finalizedIndex: await item.finalized,
+        })),
+      );
+
+      // Process in memory and validate
+      for (const { secretStr, index, pendingData, finalizedIndex } of preTagData) {
+        // First we check that for any secret the highest used index in tx is not further than window length from
+        // the highest finalized index.
+        if (index > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) {
+          throw new Error(
+            `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex ?? 0}.
+            Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team
+            to increase it!`,
+          );
+        }
+
+        // Throw if the new pending index is lower than or equal to the last finalized index
+        if (finalizedIndex !== undefined && index <= finalizedIndex) {
+          throw new Error(
+            `Cannot store pending index ${index} for secret ${secretStr}: ` +
+              `it is lower than or equal to the last finalized index ${finalizedIndex}`,
+          );
+        }
+
+        // Check if this secret + txHash combination already exists
+        const existingForSecretAndTx = pendingData.find(entry => entry.txHash === txHashStr);
+
+        if (existingForSecretAndTx) {
+          // If it exists with a different index, throw an error
+          if (existingForSecretAndTx.index !== index) {
+            throw new Error(
+              `Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` +
+                `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`,
+            );
+          }
+          // If it exists with the same index, ignore the update (no-op)
+        } else {
+          // If it doesn't exist, add it
+          this.#writePendingIndexes(jobId, secretStr, [...pendingData, { index, txHash: txHashStr }]);
+        }
+      }
+    });
+  }
+
+  /**
+   * Returns the transaction hashes of all pending transactions that contain indexes within a specified range
+   * for a given directional app tagging secret.
+   * @param secret - The directional app tagging secret to query pending indexes for.
+   * @param startIndex - The lower bound of the index range (inclusive).
+   * @param endIndex - The upper bound of the index range (exclusive).
+   * @returns An array of unique transaction hashes for pending transactions that contain indexes in the range
+   * [startIndex, endIndex). Returns an empty array if no pending indexes exist in the range.
+   */
+  getTxHashesOfPendingIndexes(
+    secret: DirectionalAppTaggingSecret,
+    startIndex: number,
+    endIndex: number,
+    jobId: string,
+  ): Promise<TxHash[]> {
+    return this.#store.transactionAsync(async () => {
+      const existing = await this.#readPendingIndexes(jobId, secret.toString());
+      const txHashes = existing
+        .filter(entry => entry.index >= startIndex && entry.index < endIndex)
+        .map(entry => entry.txHash);
+      return Array.from(new Set(txHashes)).map(TxHash.fromString);
+    });
+  }
+
+  /**
+   * Returns the last (highest) finalized index for a given secret.
+   * @param secret - The secret to get the last finalized index for.
+   * @returns The last (highest) finalized index for the given secret.
+   */
+  getLastFinalizedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
+    return this.#store.transactionAsync(() => this.#readLastFinalizedIndex(jobId, secret.toString()));
+  }
+
+  /**
+   * Returns the last used index for a given directional app tagging secret, considering both finalized and pending
+   * indexes.
+   * @param secret - The directional app tagging secret to query the last used index for.
+   * @returns The last used index.
+   */
+  getLastUsedIndex(secret: DirectionalAppTaggingSecret, jobId: string): Promise<number | undefined> {
+    const secretStr = secret.toString();
+
+    return this.#store.transactionAsync(async () => {
+      const pendingPromise = this.#readPendingIndexes(jobId, secretStr);
+      const finalizedPromise = this.#readLastFinalizedIndex(jobId, secretStr);
+
+      const [pendingTxScopedIndexes, lastFinalized] = await Promise.all([pendingPromise, finalizedPromise]);
+      const pendingIndexes = pendingTxScopedIndexes.map(entry => entry.index);
+
+      if (pendingTxScopedIndexes.length === 0) {
+        return lastFinalized;
+      }
+
+      // As the last used index we return the highest one from the pending indexes. Note that this value will be always
+      // higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
+      return Math.max(...pendingIndexes);
+    });
+  }
+
+  /**
+   * Drops all pending indexes corresponding to the given transaction hashes.
+   */
+  dropPendingIndexes(txHashes: TxHash[], jobId: string): Promise<void> {
+    if (txHashes.length === 0) {
+      return Promise.resolve();
+    }
+
+    const txHashStrings = new Set<string>(txHashes.map(txHash => txHash.toString()));
+
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const secretReadPromises: Map<string, Promise<{ index: number; txHash: string }[]>> = new Map();
+
+      for await (const secret of this.#pendingIndexes.keysAsync()) {
+        secretReadPromises.set(secret, this.#readPendingIndexes(jobId, secret));
+      }
+
+      // Add staged-only secrets (sync, no DB)
+      for (const secret of this.#getPendingIndexesForJob(jobId).keys()) {
+        if (!secretReadPromises.has(secret)) {
+          secretReadPromises.set(secret, Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []));
+        }
+      }
+
+      // Await all reads together
+      const secrets = [...secretReadPromises.keys()];
+      const pendingDataResults = await Promise.all(secretReadPromises.values());
+
+      // Process in memory
+      for (let i = 0; i < secrets.length; i++) {
+        const secret = secrets[i];
+        const pendingData = pendingDataResults[i];
+
+        if (pendingData && pendingData.length > 0) {
+          const filtered = pendingData.filter(item => !txHashStrings.has(item.txHash));
+          if (filtered.length === 0) {
+            this.#writePendingIndexes(jobId, secret, []);
+          } else if (filtered.length !== pendingData.length) {
+            // Some items were filtered out, so update the pending data
+            this.#writePendingIndexes(jobId, secret, filtered);
+          }
+          // else: No items were filtered out (txHashes not found for this secret) --> no-op
+        }
+      }
+    });
+  }
+
+  /**
+   * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending
+   * indexes.
+   */
+  finalizePendingIndexes(txHashes: TxHash[], jobId: string): Promise<void> {
+    if (txHashes.length === 0) {
+      return Promise.resolve();
+    }
+
+    const txHashStrings = new Set(txHashes.map(tx => tx.toString()));
+
+    return this.#store.transactionAsync(async () => {
+      // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+      const secretDataPromises: Map<
+        string,
+        { pending: Promise<{ index: number; txHash: string }[]>; finalized: Promise<number | undefined> }
+      > = new Map();
+
+      for await (const secret of this.#pendingIndexes.keysAsync()) {
+        secretDataPromises.set(secret, {
+          pending: this.#readPendingIndexes(jobId, secret),
+          finalized: this.#readLastFinalizedIndex(jobId, secret),
+        });
+      }
+
+      // Add staged-only secrets (sync, no DB)
+      for (const secret of this.#getPendingIndexesForJob(jobId).keys()) {
+        if (!secretDataPromises.has(secret)) {
+          secretDataPromises.set(secret, {
+            pending: Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []),
+            finalized: Promise.resolve(this.#getLastFinalizedIndexesForJob(jobId).get(secret)),
+          });
+        }
+      }
+
+      // Await all reads together
+      const secrets = [...secretDataPromises.keys()];
+      const dataResults = await Promise.all(
+        secrets.map(async secret => ({
+          secret,
+          pendingData: await secretDataPromises.get(secret)!.pending,
+          lastFinalized: await secretDataPromises.get(secret)!.finalized,
+        })),
+      );
+
+      // Process all txHashes for each secret in memory
+      for (const { secret, pendingData, lastFinalized } of dataResults) {
+        if (!pendingData || pendingData.length === 0) {
+          continue;
+        }
+
+        let currentPending = pendingData;
+        let currentFinalized = lastFinalized;
+
+        // Process all txHashes for this secret
+        for (const txHashStr of txHashStrings) {
+          const matchingIndexes = currentPending.filter(item => item.txHash === txHashStr).map(item => item.index);
+          if (matchingIndexes.length === 0) {
+            continue;
+          }
+
+          if (matchingIndexes.length > 1) {
+            // We should always just store the highest pending index for a given tx hash and secret because the lower
+            // values are irrelevant.
+            throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`);
+          }
+
+          const newFinalized = matchingIndexes[0];
+
+          if (newFinalized < (currentFinalized ?? 0)) {
+            // This should never happen because when last finalized index was finalized we should have pruned the lower
+            // pending indexes.
+            throw new Error(
+              `New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`,
+            );
+          }
+
+          currentFinalized = newFinalized;
+
+          // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we
+          // cannot control the order in which transactions are finalized, there may be pending indexes that are now
+          // obsolete because they are lower than the most recently finalized index. For this reason, we prune these
+          // outdated pending indexes.
+          currentPending = currentPending.filter(item => item.index > currentFinalized!);
+        }
+
+        // Write final state if changed
+        if (currentFinalized !== lastFinalized) {
+          this.#writeLastFinalizedIndex(jobId, secret, currentFinalized!);
+        }
+        if (currentPending !== pendingData) {
+          this.#writePendingIndexes(jobId, secret, currentPending);
+        }
+      }
+    });
+  }
+}

package/src/tagging/constants.ts

@@ -1,2 +1,10 @@
-// Half the size of the window we slide over the tagging indexes.
-export const WINDOW_HALF_SIZE = 10;
+// This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app
+// tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function.
+// This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than
+// MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note
+// and the note is nullified in the same tx.
+//
+// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than
+// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use
+// cases.
+export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;

package/src/tagging/get_all_logs_by_tags.ts

@@ -0,0 +1,68 @@
+import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { BlockHash } from '@aztec/stdlib/block';
+import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit';
+import type { AztecNode } from '@aztec/stdlib/interfaces/client';
+import type { SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs';
+
+/**
+ * Generic pagination helper that fetches all pages of results.
+ * @param numTags - The number of tags being queried (determines result array size).
+ * @param fetchPage - Function that fetches a single page of results given a page number.
+ * @returns An array of arrays, one per tag, containing all results across all pages.
+ */
+async function getAllPages<T>(numTags: number, fetchPage: (page: number) => Promise<T[][]>): Promise<T[][]> {
+  const allResultsPerTag: T[][] = Array.from({ length: numTags }, () => []);
+  let page = 0;
+  let hasMore = true;
+
+  while (hasMore) {
+    const resultsPage = await fetchPage(page);
+    hasMore = false;
+
+    for (let i = 0; i < resultsPage.length; i++) {
+      allResultsPerTag[i].push(...resultsPage[i]);
+      if (resultsPage[i].length === MAX_LOGS_PER_TAG) {
+        hasMore = true;
+      }
+    }
+    page++;
+  }
+
+  return allResultsPerTag;
+}
+
+/**
+ * Fetches all private logs for the given tags, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param tags - The siloed tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */
+export function getAllPrivateLogsByTags(
+  aztecNode: AztecNode,
+  tags: SiloedTag[],
+  anchorBlockHash: BlockHash,
+): Promise<TxScopedL2Log[][]> {
+  return getAllPages(tags.length, page => aztecNode.getPrivateLogsByTags(tags, page, anchorBlockHash));
+}
+
+/**
+ * Fetches all public logs for the given tags from a contract, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param contractAddress - The contract address to search logs for.
+ * @param tags - The tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */
+export function getAllPublicLogsByTagsFromContract(
+  aztecNode: AztecNode,
+  contractAddress: AztecAddress,
+  tags: Tag[],
+  anchorBlockHash: BlockHash,
+): Promise<TxScopedL2Log[][]> {
+  return getAllPages(tags.length, page =>
+    aztecNode.getPublicLogsByTagsFromContract(contractAddress, tags, page, anchorBlockHash),
+  );
+}

package/src/tagging/index.ts

@@ -1,6 +1,19 @@
-export * from './tag.js';
-export * from './constants.js';
-export * from './siloed_tag.js';
-export * from './utils.js';
-export { DirectionalAppTaggingSecret } from '@aztec/stdlib/logs';
+/**
+ * This directory contains the sender and recipient synchronization algorithms that use tagging.
+ *
+ * The objective of the sender sync algorithm is to determine which tags have already been used by a sender, thereby
+ * deciding which tag should be used next.
+ *
+ * The objective of the recipient sync algorithm is to load and process the corresponding logs.
+ *
+ * @module tagging
+ */
+
+export { loadPrivateLogsForSenderRecipientPair } from './recipient_sync/load_private_logs_for_sender_recipient_pair.js';
+export { syncSenderTaggingIndexes } from './sender_sync/sync_sender_tagging_indexes.js';
+export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from './constants.js';
+export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './get_all_logs_by_tags.js';
+
+// Re-export tagging-related types from stdlib
+export { DirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs';
 export { type PreTag } from '@aztec/stdlib/logs';

package/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts

@@ -0,0 +1,143 @@
+import type { BlockNumber } from '@aztec/foundation/branded-types';
+import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { BlockHash } from '@aztec/stdlib/block';
+import type { AztecNode } from '@aztec/stdlib/interfaces/client';
+import type { DirectionalAppTaggingSecret, TxScopedL2Log } from '@aztec/stdlib/logs';
+
+import type { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js';
+import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../constants.js';
+import { findHighestIndexes } from './utils/find_highest_indexes.js';
+import { loadLogsForRange } from './utils/load_logs_for_range.js';
+
+/**
+ * Loads private logs for `app` and  sender-recipient pair defined by `secret` and updates the highest aged and
+ * finalized indexes in the db. At most load logs from blocks up to and including `anchorBlockNumber`.
+ *
+ * @dev This function can be safely executed "in parallel" for other sender-recipient pairs because the data in
+ * in the tagging data provider is indexed by the secret and hence completely disjoint.
+ */
+export async function loadPrivateLogsForSenderRecipientPair(
+  secret: DirectionalAppTaggingSecret,
+  app: AztecAddress,
+  aztecNode: AztecNode,
+  taggingStore: RecipientTaggingStore,
+  anchorBlockNumber: BlockNumber,
+  anchorBlockHash: BlockHash,
+  jobId: string,
+): Promise<TxScopedL2Log[]> {
+  // # Explanation of how the algorithm works
+  // When we perform the sync we will look at logs that correspond to the tagging index range
+  // (highestAgedIndex, highestFinalizedIndex + WINDOW_LEN]
+  //
+  // highestAgedIndex is the highest index that was used in a tx that is included in a block at least
+  //                  `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds ago.
+  // highestFinalizedIndex is the highest index that was used in a tx that is included in a finalized block.
+  //
+  // "(" denotes an open end of the range - the index is not included in the range.
+  // "]" denotes a closed end of the range - the index is included in the range.
+  //
+  // ## Explanation of highestAgedIndex
+  //
+  // highestAgedIndex is chosen such that for all tagging indexes `i <= highestAgedIndex` we know that no new logs can
+  // ever appear.
+  //
+  // This relies on the "maximum inclusion timestamp" rule enforced by the kernel and rollup circuits:
+  // - a transaction's maximum inclusion timestamp is at most `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds after
+  //   the timestamp of its anchor block; and
+  // - a rollup only includes transactions whose inclusion timestamp is >= the L2 block's timestamp.
+  //
+  // Suppose some device used index `I` in a transaction anchored to block `B_N` at time `N`, and that block is now at
+  // least `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds in the past. Then there is no possibility of any *other* device
+  // trying to use an index <= `I` while anchoring to a *newer* block than `B_N` because if we were anchoring to
+  // a newer block than `B_N` then we would already have seen the log with index `I` and hence the device would have
+  // chosen a larger index.
+  //    If that *other* device would anchor to a block older than `B_N` then that tx could never be included in a block
+  // because it would already have been expired.
+  //
+  // Therefore, once we see that index `I` has been used in a block that is at least `MAX_INCLUDE_BY_TIMESTAMP_DURATION`
+  // seconds old, we can safely stop syncing logs for all indexes <= `I` and set highestAgedIndex = `I`.
+  //
+  // ## Explanation of the upper bound `highestFinalizedIndex + WINDOW_LEN`
+  //
+  // When a sender chooses a tagging index, they will select an index that is at most `WINDOW_LEN` greater than
+  // the highest finalized index. If that index was already used, they will throw an error. For this reason we
+  // don't have to look further than `highestFinalizedIndex + WINDOW_LEN`.
+
+  let finalizedBlockNumber: number, currentTimestamp: bigint;
+  {
+    const [l2Tips, latestBlockHeader] = await Promise.all([aztecNode.getL2Tips(), aztecNode.getBlockHeader('latest')]);
+
+    if (!latestBlockHeader) {
+      throw new Error('Node failed to return latest block header when syncing logs');
+    }
+
+    [finalizedBlockNumber, currentTimestamp] = [
+      l2Tips.finalized.block.number,
+      latestBlockHeader.globalVariables.timestamp,
+    ];
+  }
+
+  let start: number, end: number;
+  {
+    const currentHighestAgedIndex = await taggingStore.getHighestAgedIndex(secret, jobId);
+    const currentHighestFinalizedIndex = await taggingStore.getHighestFinalizedIndex(secret, jobId);
+
+    // We don't want to include the highest aged index so we start from `currentHighestAgedIndex + 1` (or 0 if not set)
+    start = currentHighestAgedIndex === undefined ? 0 : currentHighestAgedIndex + 1;
+
+    // The highest index a sender can choose is "highest finalized index + window length" but given that
+    // `loadLogsForRange` expects an exclusive `end` we add 1.
+    end = (currentHighestFinalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1;
+  }
+
+  const logs: TxScopedL2Log[] = [];
+
+  while (true) {
+    // Get private logs with their block timestamps and corresponding tagging indexes
+    const privateLogsWithIndexes = await loadLogsForRange(
+      secret,
+      app,
+      aztecNode,
+      start,
+      end,
+      anchorBlockNumber,
+      anchorBlockHash,
+    );
+
+    if (privateLogsWithIndexes.length === 0) {
+      break;
+    }
+
+    logs.push(...privateLogsWithIndexes.map(({ log }) => log));
+
+    const { highestAgedIndex, highestFinalizedIndex } = findHighestIndexes(
+      privateLogsWithIndexes,
+      currentTimestamp,
+      finalizedBlockNumber,
+    );
+
+    // Store updates in data provider and update local variables
+    if (highestAgedIndex !== undefined) {
+      await taggingStore.updateHighestAgedIndex(secret, highestAgedIndex, jobId);
+    }
+
+    if (highestFinalizedIndex === undefined) {
+      // We have not found a new highest finalized index, so there is no need to move the window forward.
+      break;
+    }
+
+    if (highestAgedIndex !== undefined && highestAgedIndex > highestFinalizedIndex) {
+      // This is just a sanity check as this should never happen.
+      throw new Error('Highest aged index lower than highest finalized index invariant violated');
+    }
+
+    await taggingStore.updateHighestFinalizedIndex(secret, highestFinalizedIndex, jobId);
+
+    // For the next iteration we want to look only at indexes for which we have not attempted to load logs yet while
+    // ensuring that we do not look further than WINDOW_LEN ahead of the highest finalized index.
+    start = end;
+    end = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; // `end` is exclusive so we add 1.
+  }
+
+  return logs;
+}