@aztec/pxe 0.0.1-commit.d3ec352c → 0.0.1-commit.d58ff9d0

package/src/tagging/reconcile_tagging_index_ranges.ts

@@ -0,0 +1,102 @@
+import { type AppTaggingSecret, SiloedTag, type TaggingIndexRange } from '@aztec/stdlib/logs';
+
+/**
+ * Reconciles tagging index ranges recorded by the PXE during private execution against the set of siloed tags whose
+ * private logs survived to the final kernel output.
+ *
+ * Each input range is the contiguous `[lowestIndex, highestIndex]` set of indexes that the tagging index cache
+ * reserved for a given tagging secret while a tx was being executed. The kernel may then squash some of those logs
+ * (e.g. when a note is created and nullified within the same tx, taking its log with it), so the actual on-chain
+ * footprint can be a strict subset of what was reserved.
+ *
+ * For each input range, this function shrinks `[lowestIndex, highestIndex]` so that both bounds correspond to indexes
+ * that actually survived, trimming any dropped indices at the front or the back. It is possible that some interior
+ * indexes were also dropped, leaving gaps between the lowest and highest indices, but we ignore those and accept the
+ * resulting inefficiency rather than complicate the tagging model with one in which transactions may use interleaved
+ * indices.
+ *
+ * Trimming tailing indices (e.g. `highestIndex`) is highly useful as it lets us reuse those indices in future
+ * transactions. Trimming indices at the front (e.g. `lowestIndex`) is not as impactful since we will not ever use a
+ * tagging index if a larger one exists (same as with gaps), but it does help in both making the consistency check
+ * between database and transaction effects cheaper (as there's fewer logs to test) and making the database more
+ * accurately reflect reality.
+ *
+ * Ranges with no surviving indexes are dropped from the output entirely.
+ *
+ * Example scenario:
+ *   - used indexes 3 to 7, kerel squashed indexes 3, 5, and 7:
+ *      - range: [3, 7]
+ *      - survived: set(3, 5, 7)
+ *      - output: [4, 6]  (trimmed from both ends, gap at 5 is tolerated)
+ *
+ * @param ranges - The tagging index ranges as recorded during private execution (pre-squash).
+ * @param survivingTags - The set of siloed tag values (as strings) of private logs that survived kernel squashing.
+ * @returns The reconciled ranges, with bounds tightened to surviving indexes and fully-squashed ranges removed.
+ */
+export async function reconcileTaggingIndexRangesAgainstSurvivingTags(
+  ranges: TaggingIndexRange[],
+  survivingTags: Set<string>,
+): Promise<TaggingIndexRange[]> {
+  const reconciled: TaggingIndexRange[] = [];
+
+  for (const range of ranges) {
+    const newLowestIndex = await findFirstSurvivingIndex(
+      range.extendedSecret,
+      range.lowestIndex,
+      range.highestIndex,
+      survivingTags,
+    );
+    if (newLowestIndex === undefined) {
+      // No index in this range corresponds to a surviving log: the entire range was squashed.
+      continue;
+    }
+
+    // newLowestIndex is itself a survivor, so the backward scan is guaranteed to find at least that index.
+    const newHighestIndex = await findLastSurvivingIndex(
+      range.extendedSecret,
+      newLowestIndex,
+      range.highestIndex,
+      survivingTags,
+    );
+
+    reconciled.push({
+      extendedSecret: range.extendedSecret,
+      lowestIndex: newLowestIndex,
+      highestIndex: newHighestIndex!,
+    });
+  }
+
+  return reconciled;
+}
+
+/** Scans `[start, end]` ascending and returns the first index whose siloed tag is in `survivingTags`. */
+async function findFirstSurvivingIndex(
+  secret: AppTaggingSecret,
+  start: number,
+  end: number,
+  survivingTags: Set<string>,
+): Promise<number | undefined> {
+  for (let index = start; index <= end; index++) {
+    const tag = await SiloedTag.compute({ extendedSecret: secret, index });
+    if (survivingTags.has(tag.value.toString())) {
+      return index;
+    }
+  }
+  return undefined;
+}
+
+/** Scans `[start, end]` descending and returns the first index whose siloed tag is in `survivingTags`. */
+async function findLastSurvivingIndex(
+  secret: AppTaggingSecret,
+  start: number,
+  end: number,
+  survivingTags: Set<string>,
+): Promise<number | undefined> {
+  for (let index = end; index >= start; index--) {
+    const tag = await SiloedTag.compute({ extendedSecret: secret, index });
+    if (survivingTags.has(tag.value.toString())) {
+      return index;
+    }
+  }
+  return undefined;
+}

package/src/tagging/sender_sync/sync_sender_tagging_indexes.ts

@@ -0,0 +1,134 @@
+import type { BlockHash } from '@aztec/stdlib/block';
+import type { AztecNode } from '@aztec/stdlib/interfaces/server';
+import type { AppTaggingSecret } from '@aztec/stdlib/logs';
+
+import type { SenderTaggingStore } from '../../storage/tagging_store/sender_tagging_store.js';
+import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../constants.js';
+import {
+  EMPTY_STATUS_CHANGE,
+  getStatusChangeOfPending,
+  mergeStatusChanges,
+} from './utils/get_status_change_of_pending.js';
+import { loadAndStoreNewTaggingIndexes } from './utils/load_and_store_new_tagging_indexes.js';
+
+/**
+ * Syncs tagging indexes. This function needs to be called whenever a private log is being sent.
+ *
+ * @param secret - The sender-side tagging `AppTaggingSecret`.
+ * @remarks When syncing the indexes as sender we don't care about the log contents - we only care about the highest
+ * pending and highest finalized indexes as that guides the next index choice when sending a log. The next index choice
+ * is simply the highest pending index plus one (or finalized if pending is undefined).
+ * @dev This function looks for new indexes, adds them to pending, then it checks status of each pending index and
+ * updates its status accordingly.
+ */
+export async function syncSenderTaggingIndexes(
+  secret: AppTaggingSecret,
+  aztecNode: AztecNode,
+  taggingStore: SenderTaggingStore,
+  anchorBlockHash: BlockHash,
+  jobId: string,
+): Promise<void> {
+  // # Explanation of how syncing works
+  //
+  // When choosing an index, we select: highest pending index + 1 (or highest finalized index + 1 if no pending).
+  // If the chosen index is more than WINDOW_LEN from the highest finalized index, we throw an error. By having this
+  // hard limit we give a guarantee to a recipient that he doesn't need to look further than WINDOW_LEN ahead of the
+  // highest finalized index.
+  //
+  // This function synchronizes the finalized and pending indexes by iteratively querying the node for a window of
+  // indexes at a time, storing all those indexes as pending, and then checking the status of each pending index to
+  // update its finalization status accordingly. If we stumble upon a window with no indexes, we stop the loop.
+  //
+  // Stopping at that point is safe because of the limit described above - there can never be an index that is more
+  // than WINDOW_LEN from the highest finalized index.
+  //
+  // # Note on performance
+  // Each window advance requires two queries (logs + tx status). For example, syncing indexes 0–500 with a window of
+  // 100 takes at least 10 round trips (5 windows × 2 queries).
+
+  const finalizedIndex = await taggingStore.getLastFinalizedIndex(secret, jobId);
+
+  let start = finalizedIndex === undefined ? 0 : finalizedIndex + 1;
+  let end = start + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN;
+
+  let previousFinalizedIndex = finalizedIndex;
+  let newFinalizedIndex = undefined;
+
+  while (true) {
+    // Pending tx hashes already in the store for this window (from prior syncs or txs this PXE itself sent).
+    // Reading these before issuing any RPC lets us fetch their receipts in parallel with the logs query below.
+    const knownPendingTxHashes = await taggingStore.getTxHashesOfPendingIndexes(secret, start, end, jobId);
+
+    // Fire the logs query and the known-pending receipts query in parallel
+    const [, statusOfKnown] = await Promise.all([
+      loadAndStoreNewTaggingIndexes(secret, start, end, aztecNode, taggingStore, anchorBlockHash, jobId),
+      knownPendingTxHashes.length > 0
+        ? getStatusChangeOfPending(knownPendingTxHashes, aztecNode)
+        : Promise.resolve(EMPTY_STATUS_CHANGE),
+    ]);
+
+    // Re-read pending tx hashes after the logs query writes any newly-discovered ones to the store.
+    const allPendingTxHashes = await taggingStore.getTxHashesOfPendingIndexes(secret, start, end, jobId);
+    if (allPendingTxHashes.length === 0) {
+      break;
+    }
+
+    // Receipts for pending tx hashes that the logs query just surfaced still need a sequential follow-up call.
+    // `mergePendingIndexes` is idempotent on (secret, txHash), so a re-discovered hash stays classified as known
+    // and is not re-fetched here.
+    const knownSet = new Set(knownPendingTxHashes.map(h => h.toString()));
+    const newPendingTxHashes = allPendingTxHashes.filter(h => !knownSet.has(h.toString()));
+
+    const statusOfNew =
+      newPendingTxHashes.length > 0
+        ? await getStatusChangeOfPending(newPendingTxHashes, aztecNode)
+        : EMPTY_STATUS_CHANGE;
+
+    const { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted } = mergeStatusChanges(
+      statusOfKnown,
+      statusOfNew,
+    );
+
+    await taggingStore.dropPendingIndexes(txHashesToDrop, jobId);
+    await taggingStore.finalizePendingIndexes(txHashesToFinalize, jobId);
+
+    if (txHashesWithExecutionReverted.length > 0) {
+      const receipts = await Promise.all(
+        txHashesWithExecutionReverted.map(txHash => aztecNode.getTxReceipt(txHash, { includeTxEffect: true })),
+      );
+      for (const receipt of receipts) {
+        if (!receipt.isMined() || !receipt.txEffect) {
+          throw new Error(
+            'TxEffect not found for execution-reverted tx. This is either a bug or a reorg has occurred.',
+          );
+        }
+
+        await taggingStore.finalizePendingIndexesOfAPartiallyRevertedTx(receipt.txEffect, jobId);
+      }
+    }
+
+    // We check if the finalized index has been updated.
+    newFinalizedIndex = await taggingStore.getLastFinalizedIndex(secret, jobId);
+    if (previousFinalizedIndex !== newFinalizedIndex) {
+      // A new finalized index was found, so we'll run the loop again. For example:
+      // - Previous finalized index: 10
+      // - New finalized index: 13
+      // - Window length: 10
+      //
+      // In the last iteration, we processed indexes 11-20. To avoid reprocessing the same logs,
+      // we'll only look at the new indexes 21-23:
+      //
+      //    Previous window: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+      //    New window:                                             [21, 22, 23]
+
+      const previousEnd = end;
+      // Add 1 because `end` is exclusive and the known finalized index is not included in the window.
+      end = newFinalizedIndex! + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1;
+      start = previousEnd;
+      previousFinalizedIndex = newFinalizedIndex;
+    } else {
+      // No new finalized index was found, so we don't need to process the next window.
+      break;
+    }
+  }
+}

package/src/tagging/sender_sync/utils/get_status_change_of_pending.ts

@@ -0,0 +1,66 @@
+import type { AztecNode } from '@aztec/stdlib/interfaces/server';
+import { TxHash, TxStatus } from '@aztec/stdlib/tx';
+
+/** Classification of a batch of pending tx hashes by the status change implied by their receipts. */
+export type StatusChange = {
+  txHashesToFinalize: TxHash[];
+  txHashesToDrop: TxHash[];
+  txHashesWithExecutionReverted: TxHash[];
+};
+
+export const EMPTY_STATUS_CHANGE: StatusChange = {
+  txHashesToFinalize: [],
+  txHashesToDrop: [],
+  txHashesWithExecutionReverted: [],
+};
+
+/** Concatenates two status changes field-by-field. */
+export function mergeStatusChanges(a: StatusChange, b: StatusChange): StatusChange {
+  return {
+    txHashesToFinalize: [...a.txHashesToFinalize, ...b.txHashesToFinalize],
+    txHashesToDrop: [...a.txHashesToDrop, ...b.txHashesToDrop],
+    txHashesWithExecutionReverted: [...a.txHashesWithExecutionReverted, ...b.txHashesWithExecutionReverted],
+  };
+}
+
+/**
+ * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized,
+ * dropped, or execution-reverted (but mined).
+ */
+export async function getStatusChangeOfPending(pending: TxHash[], aztecNode: AztecNode): Promise<StatusChange> {
+  // Get receipts for all pending tx hashes.
+  const receipts = await Promise.all(pending.map(pendingTxHash => aztecNode.getTxReceipt(pendingTxHash)));
+
+  const txHashesToFinalize: TxHash[] = [];
+  const txHashesToDrop: TxHash[] = [];
+  const txHashesWithExecutionReverted: TxHash[] = [];
+
+  for (let i = 0; i < receipts.length; i++) {
+    const receipt = receipts[i];
+    const txHash = pending[i];
+
+    if (receipt.status === TxStatus.FINALIZED) {
+      // Tx has been included in a block and the corresponding block is finalized
+      if (receipt.hasExecutionSucceeded()) {
+        // No part of execution reverted - we just finalize all the indexes.
+        txHashesToFinalize.push(txHash);
+      } else if (receipt.hasExecutionReverted()) {
+        // Tx was mined but execution reverted (app logic, teardown, or both). Some logs from the non-revertible
+        // phase may still be onchain. We check which tags made it onchain and finalize those; drop the rest.
+        txHashesWithExecutionReverted.push(txHash);
+      } else {
+        // Defensive check - this branch should never be triggered
+        throw new Error(
+          'Both hasExecutionSucceeded and hasExecutionReverted on the receipt returned false. This should never happen and it implies a bug. Please open an issue.',
+        );
+      }
+    } else if (receipt.isDropped()) {
+      // Tx was dropped from the mempool --> we drop the corresponding pending indexes.
+      txHashesToDrop.push(txHash);
+    } else {
+      // Tx is still pending, not yet finalized, or was mined successfully but not yet finalized --> we don't do anything.
+    }
+  }
+
+  return { txHashesToFinalize, txHashesToDrop, txHashesWithExecutionReverted };
+}

package/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts

@@ -0,0 +1,88 @@
+import type { BlockHash } from '@aztec/stdlib/block';
+import type { AztecNode } from '@aztec/stdlib/interfaces/server';
+import { type AppTaggingSecret, SiloedTag } from '@aztec/stdlib/logs';
+import { TxHash } from '@aztec/stdlib/tx';
+
+import type { SenderTaggingStore } from '../../../storage/tagging_store/sender_tagging_store.js';
+import { getAllPrivateLogsByTags } from '../../get_all_logs_by_tags.js';
+
+/**
+ * Loads tagging indexes from the Aztec node and stores them in the tagging data provider.
+ * @remarks This function is one of two places by which a pending index can get to the tagging data provider. The other
+ * place is when a tx is being sent from this PXE.
+ * @param extendedSecret - The app tagging secret whose indexes are being synced.
+ * @param start - The starting index (inclusive) of the window to process.
+ * @param end - The ending index (exclusive) of the window to process.
+ * @param aztecNode - The Aztec node instance to query for logs.
+ * @param taggingStore - The data provider to store pending indexes.
+ * @param anchorBlockHash - Hash of a block to use as reference block when querying node.
+ * @param jobId - Job identifier, used to keep writes in-memory until they can be persisted in a data integrity
+ * preserving way.
+ */
+export async function loadAndStoreNewTaggingIndexes(
+  extendedSecret: AppTaggingSecret,
+  start: number,
+  end: number,
+  aztecNode: AztecNode,
+  taggingStore: SenderTaggingStore,
+  anchorBlockHash: BlockHash,
+  jobId: string,
+) {
+  // We compute the tags for the current window of indexes
+  const siloedTagsForWindow = await Promise.all(
+    Array.from({ length: end - start }, (_, i) => SiloedTag.compute({ extendedSecret, index: start + i })),
+  );
+
+  const txsForTags = await getTxsContainingTags(siloedTagsForWindow, aztecNode, anchorBlockHash);
+  const txIndexesMap = getTxIndexesMap(txsForTags, start, siloedTagsForWindow.length);
+
+  // Now we iterate over the map, construct the tagging index ranges and store them in the db. A tx already tracked
+  // in the store is merged rather than range-checked: if this PXE sent the tx and it partially reverted, the chain
+  // only shows the surviving sub-range of the prove-time entry (the finalized receipt step of the sync owns
+  // resolving that difference), and a tx from another PXE may straddle a sync window boundary, in which case the
+  // entry is widened so the next index choice covers the full onchain range.
+  for (const [txHashStr, indexes] of txIndexesMap.entries()) {
+    const txHash = TxHash.fromString(txHashStr);
+    const ranges = [{ extendedSecret, lowestIndex: Math.min(...indexes), highestIndex: Math.max(...indexes) }];
+    await taggingStore.mergePendingIndexes(ranges, txHash, jobId);
+  }
+}
+
+// Returns txs that used the given tags. A tag might have been used in multiple txs and for this reason we return
+// an array for each tag.
+async function getTxsContainingTags(
+  tags: SiloedTag[],
+  aztecNode: AztecNode,
+  anchorBlockHash: BlockHash,
+): Promise<TxHash[][]> {
+  // We use the utility function below to retrieve all logs for the tags across all pages, so we don't need to handle
+  // pagination here. Sender sync only needs `txHash` from each log, so we leave `includeEffects` off.
+  const allLogs = await getAllPrivateLogsByTags(aztecNode, tags, anchorBlockHash);
+  return allLogs.map(logs => logs.map(log => log.txHash));
+}
+
+// Returns a map of txHash to all indexes for that txHash.
+function getTxIndexesMap(txHashesForTags: TxHash[][], start: number, count: number): Map<string, number[]> {
+  if (txHashesForTags.length !== count) {
+    throw new Error(`Number of tx hashes arrays does not match number of tags. ${txHashesForTags.length} !== ${count}`);
+  }
+
+  const indexesMap = new Map<string, number[]>();
+  // Iterate over indexes
+  for (let i = 0; i < txHashesForTags.length; i++) {
+    const taggingIndex = start + i;
+    const txHashesForTag = txHashesForTags[i];
+    // iterate over tx hashes that used that index (tag)
+    for (const txHash of txHashesForTag) {
+      const key = txHash.toString();
+      const existing = indexesMap.get(key);
+      // Add the index to the tx's indexes
+      if (existing) {
+        existing.push(taggingIndex);
+      } else {
+        indexesMap.set(key, [taggingIndex]);
+      }
+    }
+  }
+  return indexesMap;
+}

package/src/tagging/testing/tag_query_test_utils.ts

@@ -0,0 +1,11 @@
+import { type AppTaggingSecret, type PrivateLogsQuery, SiloedTag, type TagQuery } from '@aztec/stdlib/logs';
+
+/** Computes the siloed tag */
+export function computeSiloedTagForIndex(secret: AppTaggingSecret, index: number): Promise<SiloedTag> {
+  return SiloedTag.compute({ extendedSecret: secret, index });
+}
+
+/** Extracts the bare-tag set from a query */
+export function extractTags(query: PrivateLogsQuery): SiloedTag[] {
+  return query.tags.map((entry: TagQuery<SiloedTag>) => (entry instanceof SiloedTag ? entry : entry.tag));
+}