@aztec/pxe 0.0.1-commit.fcb71a6 → 0.0.1-commit.ff7989d6c

package/dest/storage/tagging_store/sender_tagging_store.js

@@ -1,11 +1,11 @@
-import { toArray } from '@aztec/foundation/iterable';
 import { TxHash } from '@aztec/stdlib/tx';
-import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/index.js';
+import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/constants.js';
 /**
  * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class
- * is called RecipientTaggingStore. We have the providers separate for the sender and recipient because
+ * is called RecipientTaggingStore. We have the data stores separate for sender and recipient because
  * the algorithms are completely disjoint and there is not data reuse between the two.
  */ export class SenderTaggingStore {
+    storeName = 'sender_tagging';
     #store;
     // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained
     // the private logs with tags corresponding to these indexes has not been finalized yet.
@@ -15,15 +15,89 @@ import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/index.js';
     // the smaller ones are irrelevant due to tx atomicity.
     //
     // TODO(#17615): This assumes no logs are used in the non-revertible phase.
+    //
+    // directional app tagging secret => { pending index, txHash }[]
     #pendingIndexes;
+    // jobId => directional app tagging secret => { pending index, txHash }[]
+    #pendingIndexesForJob;
     // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last
     // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence
     // we don't need to store the history.
+    //
+    // directional app tagging secret => highest finalized index
     #lastFinalizedIndexes;
+    // jobId => directional app tagging secret => highest finalized index
+    #lastFinalizedIndexesForJob;
     constructor(store){
         this.#store = store;
         this.#pendingIndexes = this.#store.openMap('pending_indexes');
         this.#lastFinalizedIndexes = this.#store.openMap('last_finalized_indexes');
+        this.#pendingIndexesForJob = new Map();
+        this.#lastFinalizedIndexesForJob = new Map();
+    }
+    #getPendingIndexesForJob(jobId) {
+        let pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId);
+        if (!pendingIndexesForJob) {
+            pendingIndexesForJob = new Map();
+            this.#pendingIndexesForJob.set(jobId, pendingIndexesForJob);
+        }
+        return pendingIndexesForJob;
+    }
+    #getLastFinalizedIndexesForJob(jobId) {
+        let jobStagedLastFinalizedIndexes = this.#lastFinalizedIndexesForJob.get(jobId);
+        if (!jobStagedLastFinalizedIndexes) {
+            jobStagedLastFinalizedIndexes = new Map();
+            this.#lastFinalizedIndexesForJob.set(jobId, jobStagedLastFinalizedIndexes);
+        }
+        return jobStagedLastFinalizedIndexes;
+    }
+    async #readPendingIndexes(jobId, secret) {
+        // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+        // are no pending read requests). The staged value still takes precedence if it exists.
+        const dbValue = await this.#pendingIndexes.getAsync(secret);
+        const staged = this.#getPendingIndexesForJob(jobId).get(secret);
+        return staged !== undefined ? staged : dbValue ?? [];
+    }
+    #writePendingIndexes(jobId, secret, pendingIndexes) {
+        this.#getPendingIndexesForJob(jobId).set(secret, pendingIndexes);
+    }
+    async #readLastFinalizedIndex(jobId, secret) {
+        // Always issue DB read to keep IndexedDB transaction alive (they auto-commit when a new micro-task starts and there
+        // are no pending read requests). The staged value still takes precedence if it exists.
+        const dbValue = await this.#lastFinalizedIndexes.getAsync(secret);
+        const staged = this.#getLastFinalizedIndexesForJob(jobId).get(secret);
+        return staged ?? dbValue;
+    }
+    #writeLastFinalizedIndex(jobId, secret, lastFinalizedIndex) {
+        this.#getLastFinalizedIndexesForJob(jobId).set(secret, lastFinalizedIndex);
+    }
+    /**
+   * Writes all job-specific in-memory data to persistent storage.
+   *
+   * @remark This method must run in a DB transaction context. It's designed to be called from JobCoordinator#commitJob.
+   */ async commit(jobId) {
+        const pendingIndexesForJob = this.#pendingIndexesForJob.get(jobId);
+        if (pendingIndexesForJob) {
+            for (const [secret, pendingIndexes] of pendingIndexesForJob.entries()){
+                if (pendingIndexes.length === 0) {
+                    await this.#pendingIndexes.delete(secret);
+                } else {
+                    await this.#pendingIndexes.set(secret, pendingIndexes);
+                }
+            }
+        }
+        const lastFinalizedIndexesForJob = this.#lastFinalizedIndexesForJob.get(jobId);
+        if (lastFinalizedIndexesForJob) {
+            for (const [secret, lastFinalizedIndex] of lastFinalizedIndexesForJob.entries()){
+                await this.#lastFinalizedIndexes.set(secret, lastFinalizedIndex);
+            }
+        }
+        return this.discardStaged(jobId);
+    }
+    discardStaged(jobId) {
+        this.#pendingIndexesForJob.delete(jobId);
+        this.#lastFinalizedIndexesForJob.delete(jobId);
+        return Promise.resolve();
     }
     /**
    * Stores pending indexes.
@@ -33,6 +107,7 @@ import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/index.js';
    * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be
    * stored in the db.
    * @param txHash - The tx in which the pretags were used in private logs.
+   * @param jobId - job context for staged writes to this store. See `JobCoordinator` for more details.
    * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care
    * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch
    * bugs.
@@ -45,49 +120,68 @@ import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/index.js';
    * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret.
    * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher
    * indexes than finalized ones.
-   */ async storePendingIndexes(preTags, txHash) {
+   */ storePendingIndexes(preTags, txHash, jobId) {
+        if (preTags.length === 0) {
+            return Promise.resolve();
+        }
         // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash
         // pair. Below we check that this is the case.
         const secretsSet = new Set(preTags.map((preTag)=>preTag.secret.toString()));
         if (secretsSet.size !== preTags.length) {
-            throw new Error(`Duplicate secrets found when storing pending indexes`);
+            return Promise.reject(new Error(`Duplicate secrets found when storing pending indexes`));
         }
-        for (const { secret, index } of preTags){
-            // First we check that for any secret the highest used index in tx is not further than window length from
-            // the highest finalized index.
-            const finalizedIndex = await this.getLastFinalizedIndex(secret) ?? 0;
-            if (index > finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) {
-                throw new Error(`Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex}.
-          Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team
-          to increase it!`);
-            }
-            // Throw if the new pending index is lower than or equal to the last finalized index
-            const secretStr = secret.toString();
-            const lastFinalizedIndex = await this.#lastFinalizedIndexes.getAsync(secretStr);
-            if (lastFinalizedIndex !== undefined && index <= lastFinalizedIndex) {
-                throw new Error(`Cannot store pending index ${index} for secret ${secretStr}: ` + `it is lower than or equal to the last finalized index ${lastFinalizedIndex}`);
-            }
-            // Check if this secret + txHash combination already exists
-            const txHashStr = txHash.toString();
-            const existingForSecret = await this.#pendingIndexes.getAsync(secretStr) ?? [];
-            const existingForSecretAndTx = existingForSecret.find((entry)=>entry.txHash === txHashStr);
-            if (existingForSecretAndTx) {
-                // If it exists with a different index, throw an error
-                if (existingForSecretAndTx.index !== index) {
-                    throw new Error(`Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` + `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`);
+        const txHashStr = txHash.toString();
+        return this.#store.transactionAsync(async ()=>{
+            // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+            const preTagReadPromises = preTags.map(({ secret, index })=>{
+                const secretStr = secret.toString();
+                return {
+                    secret,
+                    secretStr,
+                    index,
+                    pending: this.#readPendingIndexes(jobId, secretStr),
+                    finalized: this.#readLastFinalizedIndex(jobId, secretStr)
+                };
+            });
+            // Await all reads together
+            const preTagData = await Promise.all(preTagReadPromises.map(async (item)=>({
+                    ...item,
+                    pendingData: await item.pending,
+                    finalizedIndex: await item.finalized
+                })));
+            // Process in memory and validate
+            for (const { secretStr, index, pendingData, finalizedIndex } of preTagData){
+                // First we check that for any secret the highest used index in tx is not further than window length from
+                // the highest finalized index.
+                if (index > (finalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) {
+                    throw new Error(`Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex ?? 0}.
+            Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team
+            to increase it!`);
+                }
+                // Throw if the new pending index is lower than or equal to the last finalized index
+                if (finalizedIndex !== undefined && index <= finalizedIndex) {
+                    throw new Error(`Cannot store pending index ${index} for secret ${secretStr}: ` + `it is lower than or equal to the last finalized index ${finalizedIndex}`);
                 }
-            // If it exists with the same index, ignore the update (no-op)
-            } else {
-                // If it doesn't exist, add it
-                await this.#pendingIndexes.set(secretStr, [
-                    ...existingForSecret,
-                    {
-                        index,
-                        txHash: txHashStr
+                // Check if this secret + txHash combination already exists
+                const existingForSecretAndTx = pendingData.find((entry)=>entry.txHash === txHashStr);
+                if (existingForSecretAndTx) {
+                    // If it exists with a different index, throw an error
+                    if (existingForSecretAndTx.index !== index) {
+                        throw new Error(`Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` + `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`);
                     }
-                ]);
+                // If it exists with the same index, ignore the update (no-op)
+                } else {
+                    // If it doesn't exist, add it
+                    this.#writePendingIndexes(jobId, secretStr, [
+                        ...pendingData,
+                        {
+                            index,
+                            txHash: txHashStr
+                        }
+                    ]);
+                }
             }
-        }
+        });
     }
     /**
    * Returns the transaction hashes of all pending transactions that contain indexes within a specified range
@@ -97,100 +191,158 @@ import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/index.js';
    * @param endIndex - The upper bound of the index range (exclusive).
    * @returns An array of unique transaction hashes for pending transactions that contain indexes in the range
    * [startIndex, endIndex). Returns an empty array if no pending indexes exist in the range.
-   */ async getTxHashesOfPendingIndexes(secret, startIndex, endIndex) {
-        const existing = await this.#pendingIndexes.getAsync(secret.toString()) ?? [];
-        const txHashes = existing.filter((entry)=>entry.index >= startIndex && entry.index < endIndex).map((entry)=>entry.txHash);
-        return Array.from(new Set(txHashes)).map(TxHash.fromString);
+   */ getTxHashesOfPendingIndexes(secret, startIndex, endIndex, jobId) {
+        return this.#store.transactionAsync(async ()=>{
+            const existing = await this.#readPendingIndexes(jobId, secret.toString());
+            const txHashes = existing.filter((entry)=>entry.index >= startIndex && entry.index < endIndex).map((entry)=>entry.txHash);
+            return Array.from(new Set(txHashes)).map(TxHash.fromString);
+        });
     }
     /**
    * Returns the last (highest) finalized index for a given secret.
    * @param secret - The secret to get the last finalized index for.
    * @returns The last (highest) finalized index for the given secret.
-   */ getLastFinalizedIndex(secret) {
-        return this.#lastFinalizedIndexes.getAsync(secret.toString());
+   */ getLastFinalizedIndex(secret, jobId) {
+        return this.#store.transactionAsync(()=>this.#readLastFinalizedIndex(jobId, secret.toString()));
     }
     /**
    * Returns the last used index for a given directional app tagging secret, considering both finalized and pending
    * indexes.
    * @param secret - The directional app tagging secret to query the last used index for.
    * @returns The last used index.
-   */ async getLastUsedIndex(secret) {
+   */ getLastUsedIndex(secret, jobId) {
         const secretStr = secret.toString();
-        const pendingTxScopedIndexes = await this.#pendingIndexes.getAsync(secretStr) ?? [];
-        const pendingIndexes = pendingTxScopedIndexes.map((entry)=>entry.index);
-        if (pendingTxScopedIndexes.length === 0) {
-            return this.#lastFinalizedIndexes.getAsync(secretStr);
-        }
-        // As the last used index we return the highest one from the pending indexes. Note that this value will be always
-        // higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
-        return Math.max(...pendingIndexes);
+        return this.#store.transactionAsync(async ()=>{
+            const pendingPromise = this.#readPendingIndexes(jobId, secretStr);
+            const finalizedPromise = this.#readLastFinalizedIndex(jobId, secretStr);
+            const [pendingTxScopedIndexes, lastFinalized] = await Promise.all([
+                pendingPromise,
+                finalizedPromise
+            ]);
+            const pendingIndexes = pendingTxScopedIndexes.map((entry)=>entry.index);
+            if (pendingTxScopedIndexes.length === 0) {
+                return lastFinalized;
+            }
+            // As the last used index we return the highest one from the pending indexes. Note that this value will be always
+            // higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
+            return Math.max(...pendingIndexes);
+        });
     }
     /**
    * Drops all pending indexes corresponding to the given transaction hashes.
-   */ async dropPendingIndexes(txHashes) {
+   */ dropPendingIndexes(txHashes, jobId) {
         if (txHashes.length === 0) {
-            return;
+            return Promise.resolve();
         }
-        const txHashStrs = new Set(txHashes.map((txHash)=>txHash.toString()));
-        const allSecrets = await toArray(this.#pendingIndexes.keysAsync());
-        for (const secret of allSecrets){
-            const pendingData = await this.#pendingIndexes.getAsync(secret);
-            if (pendingData) {
-                const filtered = pendingData.filter((item)=>!txHashStrs.has(item.txHash));
-                if (filtered.length === 0) {
-                    await this.#pendingIndexes.delete(secret);
-                } else if (filtered.length !== pendingData.length) {
-                    // Some items were filtered out, so update the pending data
-                    await this.#pendingIndexes.set(secret, filtered);
+        const txHashStrings = new Set(txHashes.map((txHash)=>txHash.toString()));
+        return this.#store.transactionAsync(async ()=>{
+            // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+            const secretReadPromises = new Map();
+            for await (const secret of this.#pendingIndexes.keysAsync()){
+                secretReadPromises.set(secret, this.#readPendingIndexes(jobId, secret));
+            }
+            // Add staged-only secrets (sync, no DB)
+            for (const secret of this.#getPendingIndexesForJob(jobId).keys()){
+                if (!secretReadPromises.has(secret)) {
+                    secretReadPromises.set(secret, Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []));
                 }
-            // else: No items were filtered out (txHashes not found for this secret) --> no-op
             }
-        }
+            // Await all reads together
+            const secrets = [
+                ...secretReadPromises.keys()
+            ];
+            const pendingDataResults = await Promise.all(secretReadPromises.values());
+            // Process in memory
+            for(let i = 0; i < secrets.length; i++){
+                const secret = secrets[i];
+                const pendingData = pendingDataResults[i];
+                if (pendingData && pendingData.length > 0) {
+                    const filtered = pendingData.filter((item)=>!txHashStrings.has(item.txHash));
+                    if (filtered.length === 0) {
+                        this.#writePendingIndexes(jobId, secret, []);
+                    } else if (filtered.length !== pendingData.length) {
+                        // Some items were filtered out, so update the pending data
+                        this.#writePendingIndexes(jobId, secret, filtered);
+                    }
+                // else: No items were filtered out (txHashes not found for this secret) --> no-op
+                }
+            }
+        });
     }
     /**
    * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending
    * indexes.
-   */ async finalizePendingIndexes(txHashes) {
+   */ finalizePendingIndexes(txHashes, jobId) {
         if (txHashes.length === 0) {
-            return;
+            return Promise.resolve();
         }
-        for (const txHash of txHashes){
-            const txHashStr = txHash.toString();
-            const allSecrets = await toArray(this.#pendingIndexes.keysAsync());
-            for (const secret of allSecrets){
-                const pendingData = await this.#pendingIndexes.getAsync(secret);
-                if (!pendingData) {
-                    continue;
+        const txHashStrings = new Set(txHashes.map((tx)=>tx.toString()));
+        return this.#store.transactionAsync(async ()=>{
+            // Prefetch all data, start reads during iteration to keep IndexedDB transaction alive
+            const secretDataPromises = new Map();
+            for await (const secret of this.#pendingIndexes.keysAsync()){
+                secretDataPromises.set(secret, {
+                    pending: this.#readPendingIndexes(jobId, secret),
+                    finalized: this.#readLastFinalizedIndex(jobId, secret)
+                });
+            }
+            // Add staged-only secrets (sync, no DB)
+            for (const secret of this.#getPendingIndexesForJob(jobId).keys()){
+                if (!secretDataPromises.has(secret)) {
+                    secretDataPromises.set(secret, {
+                        pending: Promise.resolve(this.#getPendingIndexesForJob(jobId).get(secret) ?? []),
+                        finalized: Promise.resolve(this.#getLastFinalizedIndexesForJob(jobId).get(secret))
+                    });
                 }
-                const matchingIndexes = pendingData.filter((item)=>item.txHash === txHashStr).map((item)=>item.index);
-                if (matchingIndexes.length === 0) {
+            }
+            // Await all reads together
+            const secrets = [
+                ...secretDataPromises.keys()
+            ];
+            const dataResults = await Promise.all(secrets.map(async (secret)=>({
+                    secret,
+                    pendingData: await secretDataPromises.get(secret).pending,
+                    lastFinalized: await secretDataPromises.get(secret).finalized
+                })));
+            // Process all txHashes for each secret in memory
+            for (const { secret, pendingData, lastFinalized } of dataResults){
+                if (!pendingData || pendingData.length === 0) {
                     continue;
                 }
-                if (matchingIndexes.length > 1) {
-                    // We should always just store the highest pending index for a given tx hash and secret because the lower
-                    // values are irrelevant.
-                    throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`);
+                let currentPending = pendingData;
+                let currentFinalized = lastFinalized;
+                // Process all txHashes for this secret
+                for (const txHashStr of txHashStrings){
+                    const matchingIndexes = currentPending.filter((item)=>item.txHash === txHashStr).map((item)=>item.index);
+                    if (matchingIndexes.length === 0) {
+                        continue;
+                    }
+                    if (matchingIndexes.length > 1) {
+                        // We should always just store the highest pending index for a given tx hash and secret because the lower
+                        // values are irrelevant.
+                        throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`);
+                    }
+                    const newFinalized = matchingIndexes[0];
+                    if (newFinalized < (currentFinalized ?? 0)) {
+                        // This should never happen because when last finalized index was finalized we should have pruned the lower
+                        // pending indexes.
+                        throw new Error(`New finalized index ${newFinalized} is smaller than the current last finalized index ${currentFinalized}`);
+                    }
+                    currentFinalized = newFinalized;
+                    // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we
+                    // cannot control the order in which transactions are finalized, there may be pending indexes that are now
+                    // obsolete because they are lower than the most recently finalized index. For this reason, we prune these
+                    // outdated pending indexes.
+                    currentPending = currentPending.filter((item)=>item.index > currentFinalized);
                 }
-                let lastFinalized = await this.#lastFinalizedIndexes.getAsync(secret);
-                const newFinalized = matchingIndexes[0];
-                if (newFinalized < (lastFinalized ?? 0)) {
-                    // This should never happen because when last finalized index was finalized we should have pruned the lower
-                    // pending indexes.
-                    throw new Error(`New finalized index ${newFinalized} is smaller than the current last finalized index ${lastFinalized}`);
+                // Write final state if changed
+                if (currentFinalized !== lastFinalized) {
+                    this.#writeLastFinalizedIndex(jobId, secret, currentFinalized);
                 }
-                await this.#lastFinalizedIndexes.set(secret, newFinalized);
-                lastFinalized = newFinalized;
-                // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we
-                // cannot control the order in which transactions are finalized, there may be pending indexes that are now
-                // obsolete because they are lower than the most recently finalized index. For this reason, we prune these
-                // outdated pending indexes.
-                const remainingItemsOfHigherIndex = pendingData.filter((item)=>item.index > (lastFinalized ?? 0));
-                if (remainingItemsOfHigherIndex.length === 0) {
-                    await this.#pendingIndexes.delete(secret);
-                } else {
-                    await this.#pendingIndexes.set(secret, remainingItemsOfHigherIndex);
+                if (currentPending !== pendingData) {
+                    this.#writePendingIndexes(jobId, secret, currentPending);
                 }
             }
-        }
+        });
     }
 }

package/dest/tagging/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uc3RhbnRzLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvdGFnZ2luZy9jb25zdGFudHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBU0EsZUFBTyxNQUFNLHNDQUFzQyxLQUFLLENBQUMifQ==

package/dest/tagging/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/tagging/constants.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,sCAAsC,KAAK,CAAC"}

package/dest/tagging/constants.js

@@ -0,0 +1,10 @@
+// This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app
+// tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function.
+// This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than
+// MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note
+// and the note is nullified in the same tx.
+//
+// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than
+// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use
+// cases.
+export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;

package/dest/tagging/get_all_logs_by_tags.d.ts

@@ -0,0 +1,24 @@
+import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { BlockHash } from '@aztec/stdlib/block';
+import type { AztecNode } from '@aztec/stdlib/interfaces/client';
+import type { SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs';
+/**
+ * Fetches all private logs for the given tags, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param tags - The siloed tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */
+export declare function getAllPrivateLogsByTags(aztecNode: AztecNode, tags: SiloedTag[], anchorBlockHash: BlockHash): Promise<TxScopedL2Log[][]>;
+/**
+ * Fetches all public logs for the given tags from a contract, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param contractAddress - The contract address to search logs for.
+ * @param tags - The tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */
+export declare function getAllPublicLogsByTagsFromContract(aztecNode: AztecNode, contractAddress: AztecAddress, tags: Tag[], anchorBlockHash: BlockHash): Promise<TxScopedL2Log[][]>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2V0X2FsbF9sb2dzX2J5X3RhZ3MuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90YWdnaW5nL2dldF9hbGxfbG9nc19ieV90YWdzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBRXJELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBQ2pFLE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsYUFBYSxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFpRHhFOzs7Ozs7O0dBT0c7QUFDSCx3QkFBZ0IsdUJBQXVCLENBQ3JDLFNBQVMsRUFBRSxTQUFTLEVBQ3BCLElBQUksRUFBRSxTQUFTLEVBQUUsRUFDakIsZUFBZSxFQUFFLFNBQVMsR0FDekIsT0FBTyxDQUFDLGFBQWEsRUFBRSxFQUFFLENBQUMsQ0FJNUI7QUFFRDs7Ozs7Ozs7R0FRRztBQUNILHdCQUFnQixrQ0FBa0MsQ0FDaEQsU0FBUyxFQUFFLFNBQVMsRUFDcEIsZUFBZSxFQUFFLFlBQVksRUFDN0IsSUFBSSxFQUFFLEdBQUcsRUFBRSxFQUNYLGVBQWUsRUFBRSxTQUFTLEdBQ3pCLE9BQU8sQ0FBQyxhQUFhLEVBQUUsRUFBRSxDQUFDLENBTTVCIn0=

package/dest/tagging/get_all_logs_by_tags.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get_all_logs_by_tags.d.ts","sourceRoot":"","sources":["../../src/tagging/get_all_logs_by_tags.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAiDxE;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,SAAS,EAAE,EACjB,eAAe,EAAE,SAAS,GACzB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAI5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,kCAAkC,CAChD,SAAS,EAAE,SAAS,EACpB,eAAe,EAAE,YAAY,EAC7B,IAAI,EAAE,GAAG,EAAE,EACX,eAAe,EAAE,SAAS,GACzB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAM5B"}

package/dest/tagging/get_all_logs_by_tags.js

@@ -0,0 +1,60 @@
+import { MAX_LOGS_PER_TAG, MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit';
+/**
+ * Generic pagination helper that fetches all pages of results.
+ * @param numTags - The number of tags being queried (determines result array size).
+ * @param fetchPage - Function that fetches a single page of results given a page number.
+ * @returns An array of arrays, one per tag, containing all results across all pages.
+ */ async function getAllPages(numTags, fetchPage) {
+    const allResultsPerTag = Array.from({
+        length: numTags
+    }, ()=>[]);
+    let page = 0;
+    let hasMore = true;
+    while(hasMore){
+        const resultsPage = await fetchPage(page);
+        hasMore = false;
+        for(let i = 0; i < resultsPage.length; i++){
+            allResultsPerTag[i].push(...resultsPage[i]);
+            if (resultsPage[i].length === MAX_LOGS_PER_TAG) {
+                hasMore = true;
+            }
+        }
+        page++;
+    }
+    return allResultsPerTag;
+}
+/**
+ * Splits tags into chunks of MAX_RPC_LEN, fetches logs for each chunk using getAllPages, then stitches the results
+ * back into a single array preserving the original tag order.
+ */ async function getAllPagesInBatches(tags, fetchAllPagesForBatch) {
+    if (tags.length <= MAX_RPC_LEN) {
+        return fetchAllPagesForBatch(tags);
+    }
+    const batches = [];
+    for(let i = 0; i < tags.length; i += MAX_RPC_LEN){
+        batches.push(tags.slice(i, i + MAX_RPC_LEN));
+    }
+    const batchResults = await Promise.all(batches.map(fetchAllPagesForBatch));
+    return batchResults.flat();
+}
+/**
+ * Fetches all private logs for the given tags, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param tags - The siloed tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */ export function getAllPrivateLogsByTags(aztecNode, tags, anchorBlockHash) {
+    return getAllPagesInBatches(tags, (batch)=>getAllPages(batch.length, (page)=>aztecNode.getPrivateLogsByTags(batch, page, anchorBlockHash)));
+}
+/**
+ * Fetches all public logs for the given tags from a contract, automatically paginating through all pages.
+ * @param aztecNode - The Aztec node to query.
+ * @param contractAddress - The contract address to search logs for.
+ * @param tags - The tags to search for.
+ * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
+ * because of reorgs).
+ * @returns An array of log arrays, one per tag, containing all logs across all pages.
+ */ export function getAllPublicLogsByTagsFromContract(aztecNode, contractAddress, tags, anchorBlockHash) {
+    return getAllPagesInBatches(tags, (batch)=>getAllPages(batch.length, (page)=>aztecNode.getPublicLogsByTagsFromContract(contractAddress, batch, page, anchorBlockHash)));
+}

package/dest/tagging/index.d.ts

@@ -10,7 +10,8 @@
  */
 export { loadPrivateLogsForSenderRecipientPair } from './recipient_sync/load_private_logs_for_sender_recipient_pair.js';
 export { syncSenderTaggingIndexes } from './sender_sync/sync_sender_tagging_indexes.js';
-export declare const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;
+export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from './constants.js';
+export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './get_all_logs_by_tags.js';
 export { DirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs';
 export { type PreTag } from '@aztec/stdlib/logs';
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90YWdnaW5nL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOzs7Ozs7Ozs7R0FTRztBQUVILE9BQU8sRUFBRSxxQ0FBcUMsRUFBRSxNQUFNLGlFQUFpRSxDQUFDO0FBQ3hILE9BQU8sRUFBRSx3QkFBd0IsRUFBRSxNQUFNLDhDQUE4QyxDQUFDO0FBV3hGLGVBQU8sTUFBTSxzQ0FBc0MsS0FBSyxDQUFDO0FBR3pELE9BQU8sRUFBRSwyQkFBMkIsRUFBRSxHQUFHLEVBQUUsU0FBUyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDakYsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFFLE1BQU0sb0JBQW9CLENBQUMifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90YWdnaW5nL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOzs7Ozs7Ozs7R0FTRztBQUVILE9BQU8sRUFBRSxxQ0FBcUMsRUFBRSxNQUFNLGlFQUFpRSxDQUFDO0FBQ3hILE9BQU8sRUFBRSx3QkFBd0IsRUFBRSxNQUFNLDhDQUE4QyxDQUFDO0FBQ3hGLE9BQU8sRUFBRSxzQ0FBc0MsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBQ3hFLE9BQU8sRUFBRSx1QkFBdUIsRUFBRSxrQ0FBa0MsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBR3hHLE9BQU8sRUFBRSwyQkFBMkIsRUFBRSxHQUFHLEVBQUUsU0FBUyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDakYsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFFLE1BQU0sb0JBQW9CLENBQUMifQ==

package/dest/tagging/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tagging/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,qCAAqC,EAAE,MAAM,iEAAiE,CAAC;AACxH,OAAO,EAAE,wBAAwB,EAAE,MAAM,8CAA8C,CAAC;AAWxF,eAAO,MAAM,sCAAsC,KAAK,CAAC;AAGzD,OAAO,EAAE,2BAA2B,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACjF,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tagging/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,qCAAqC,EAAE,MAAM,iEAAiE,CAAC;AACxH,OAAO,EAAE,wBAAwB,EAAE,MAAM,8CAA8C,CAAC;AACxF,OAAO,EAAE,sCAAsC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,kCAAkC,EAAE,MAAM,2BAA2B,CAAC;AAGxG,OAAO,EAAE,2BAA2B,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACjF,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC"}

package/dest/tagging/index.js

@@ -9,15 +9,7 @@
  * @module tagging
  */ export { loadPrivateLogsForSenderRecipientPair } from './recipient_sync/load_private_logs_for_sender_recipient_pair.js';
 export { syncSenderTaggingIndexes } from './sender_sync/sync_sender_tagging_indexes.js';
-// This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app
-// tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function.
-// This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than
-// MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note
-// and the note is nullified in the same tx.
-//
-// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than
-// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use
-// cases.
-export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;
+export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from './constants.js';
+export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './get_all_logs_by_tags.js';
 // Re-export tagging-related types from stdlib
 export { DirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs';

package/dest/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.d.ts

@@ -1,5 +1,6 @@
 import type { BlockNumber } from '@aztec/foundation/branded-types';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { BlockHash } from '@aztec/stdlib/block';
 import type { AztecNode } from '@aztec/stdlib/interfaces/client';
 import type { DirectionalAppTaggingSecret, TxScopedL2Log } from '@aztec/stdlib/logs';
 import type { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js';
@@ -10,5 +11,5 @@ import type { RecipientTaggingStore } from '../../storage/tagging_store/recipien
  * @dev This function can be safely executed "in parallel" for other sender-recipient pairs because the data in
  * in the tagging data provider is indexed by the secret and hence completely disjoint.
  */
-export declare function loadPrivateLogsForSenderRecipientPair(secret: DirectionalAppTaggingSecret, app: AztecAddress, aztecNode: AztecNode, taggingStore: RecipientTaggingStore, anchorBlockNumber: BlockNumber): Promise<TxScopedL2Log[]>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZF9wcml2YXRlX2xvZ3NfZm9yX3NlbmRlcl9yZWNpcGllbnRfcGFpci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3RhZ2dpbmcvcmVjaXBpZW50X3N5bmMvbG9hZF9wcml2YXRlX2xvZ3NfZm9yX3NlbmRlcl9yZWNpcGllbnRfcGFpci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUNuRSxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUNqRSxPQUFPLEtBQUssRUFBRSwyQkFBMkIsRUFBRSxhQUFhLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUVyRixPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLHdEQUF3RCxDQUFDO0FBS3BHOzs7Ozs7R0FNRztBQUNILHdCQUFzQixxQ0FBcUMsQ0FDekQsTUFBTSxFQUFFLDJCQUEyQixFQUNuQyxHQUFHLEVBQUUsWUFBWSxFQUNqQixTQUFTLEVBQUUsU0FBUyxFQUNwQixZQUFZLEVBQUUscUJBQXFCLEVBQ25DLGlCQUFpQixFQUFFLFdBQVcsR0FDN0IsT0FBTyxDQUFDLGFBQWEsRUFBRSxDQUFDLENBeUcxQiJ9
+export declare function loadPrivateLogsForSenderRecipientPair(secret: DirectionalAppTaggingSecret, app: AztecAddress, aztecNode: AztecNode, taggingStore: RecipientTaggingStore, anchorBlockNumber: BlockNumber, anchorBlockHash: BlockHash, jobId: string): Promise<TxScopedL2Log[]>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZF9wcml2YXRlX2xvZ3NfZm9yX3NlbmRlcl9yZWNpcGllbnRfcGFpci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3RhZ2dpbmcvcmVjaXBpZW50X3N5bmMvbG9hZF9wcml2YXRlX2xvZ3NfZm9yX3NlbmRlcl9yZWNpcGllbnRfcGFpci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUNuRSxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUNyRCxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUNqRSxPQUFPLEtBQUssRUFBRSwyQkFBMkIsRUFBRSxhQUFhLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUVyRixPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLHdEQUF3RCxDQUFDO0FBS3BHOzs7Ozs7R0FNRztBQUNILHdCQUFzQixxQ0FBcUMsQ0FDekQsTUFBTSxFQUFFLDJCQUEyQixFQUNuQyxHQUFHLEVBQUUsWUFBWSxFQUNqQixTQUFTLEVBQUUsU0FBUyxFQUNwQixZQUFZLEVBQUUscUJBQXFCLEVBQ25DLGlCQUFpQixFQUFFLFdBQVcsRUFDOUIsZUFBZSxFQUFFLFNBQVMsRUFDMUIsS0FBSyxFQUFFLE1BQU0sR0FDWixPQUFPLENBQUMsYUFBYSxFQUFFLENBQUMsQ0FvSDFCIn0=

package/dest/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"load_private_logs_for_sender_recipient_pair.d.ts","sourceRoot":"","sources":["../../../src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,KAAK,EAAE,2BAA2B,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAErF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,wDAAwD,CAAC;AAKpG;;;;;;GAMG;AACH,wBAAsB,qCAAqC,CACzD,MAAM,EAAE,2BAA2B,EACnC,GAAG,EAAE,YAAY,EACjB,SAAS,EAAE,SAAS,EACpB,YAAY,EAAE,qBAAqB,EACnC,iBAAiB,EAAE,WAAW,GAC7B,OAAO,CAAC,aAAa,EAAE,CAAC,CAyG1B"}
+{"version":3,"file":"load_private_logs_for_sender_recipient_pair.d.ts","sourceRoot":"","sources":["../../../src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,KAAK,EAAE,2BAA2B,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAErF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,wDAAwD,CAAC;AAKpG;;;;;;GAMG;AACH,wBAAsB,qCAAqC,CACzD,MAAM,EAAE,2BAA2B,EACnC,GAAG,EAAE,YAAY,EACjB,SAAS,EAAE,SAAS,EACpB,YAAY,EAAE,qBAAqB,EACnC,iBAAiB,EAAE,WAAW,EAC9B,eAAe,EAAE,SAAS,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,aAAa,EAAE,CAAC,CAoH1B"}