@aztec/pxe 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

package/src/storage/tagging_store/tagging_secret_sources_store.ts

@@ -0,0 +1,138 @@
+import { Point } from '@aztec/foundation/curves/grumpkin';
+import { toArray } from '@aztec/foundation/iterable';
+import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
+import { AztecAddress } from '@aztec/stdlib/aztec-address';
+
+/**
+ * The kinds of shared secret an entry can hold.
+ */
+export type SharedSecretKind = 'arbitrary-secret' | 'handshake';
+
+/** A shared secret registered for a recipient, with the kind that determines which tag streams it backs. */
+export type SharedSecret = { kind: SharedSecretKind; secret: Point };
+
+/**
+ * Stores the sources from which directional app tagging secrets are derived during recipient log synchronization.
+ *
+ * Two kinds of source are held:
+ * - Sender addresses: combined with a recipient to derive a shared tagging secret via ECDH. These are global (not
+ *   scoped to a recipient) because the per-recipient binding comes from re-mixing the recipient's keys during
+ *   derivation, so each account only ever derives secrets meant for it.
+ * - Pre-shared tagging secrets: shared secret points registered directly, bypassing ECDH. These are scoped to a
+ *   specific recipient, since the derivation of the directional app tagging secret does not require any secret
+ *   recipient data: given the original secret anyone can derive a recipient's app-siloed directional tagging secret,
+ *   and so these must not be reused across recipients to preserve privacy. Each carries the {@link SharedSecretKind}
+ *   it was registered through, which determines the tag streams it is scanned under.
+ */
+export class TaggingSecretSourcesStore {
+  #store: AztecAsyncKVStore;
+  #senders: AztecAsyncMap<string, true>;
+  #sharedSecretsByRecipient: AztecAsyncMultiMap<string, StoredSharedSecret>;
+
+  constructor(store: AztecAsyncKVStore) {
+    this.#store = store;
+
+    this.#senders = this.#store.openMap('senders');
+    this.#sharedSecretsByRecipient = this.#store.openMultiMap('recipient_shared_secrets');
+  }
+
+  addSender(address: AztecAddress): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      if (await this.#senders.hasAsync(address.toString())) {
+        return false;
+      }
+
+      await this.#senders.set(address.toString(), true);
+
+      return true;
+    });
+  }
+
+  getSenders(): Promise<AztecAddress[]> {
+    return this.#store.transactionAsync(async () => {
+      return (await toArray(this.#senders.keysAsync())).map(AztecAddress.fromStringUnsafe);
+    });
+  }
+
+  removeSender(address: AztecAddress): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      if (!(await this.#senders.hasAsync(address.toString()))) {
+        return false;
+      }
+
+      await this.#senders.delete(address.toString());
+
+      return true;
+    });
+  }
+
+  /**
+   * Registers a pre-shared tagging secret scoped to a recipient.
+   * @returns true if the secret was newly added, false if it was already registered for that recipient.
+   */
+  addSharedSecret(recipient: AztecAddress, kind: SharedSecretKind, secret: Point): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      const secretStr = secret.toString();
+      // MultiMap.set silently ignores an identical (key, value), so we scan to report whether this is a new secret.
+      for await (const existing of this.#sharedSecretsByRecipient.getValuesAsync(recipient.toString())) {
+        if (existing.secret === secretStr) {
+          if (existing.kind !== kind) {
+            throw new Error(
+              `Secret already registered for recipient with kind '${existing.kind}', cannot re-register it as '${kind}'.`,
+            );
+          }
+          return false;
+        }
+      }
+
+      await this.#sharedSecretsByRecipient.set(recipient.toString(), { kind, secret: secretStr });
+
+      return true;
+    });
+  }
+
+  /** Returns the pre-shared tagging secrets registered for a given recipient. */
+  getSharedSecretsForRecipient(recipient: AztecAddress): Promise<SharedSecret[]> {
+    return this.#store.transactionAsync(async () => {
+      return (await toArray(this.#sharedSecretsByRecipient.getValuesAsync(recipient.toString()))).map(
+        deserializeSharedSecret,
+      );
+    });
+  }
+
+  /** Returns every registered pre-shared tagging secret, each paired with the recipient it is scoped to. */
+  getAllSharedSecrets(): Promise<{ recipient: AztecAddress; kind: SharedSecretKind; secret: Point }[]> {
+    return this.#store.transactionAsync(async () => {
+      const entries = await toArray(this.#sharedSecretsByRecipient.entriesAsync());
+      return entries.map(([recipient, entry]) => ({
+        recipient: AztecAddress.fromStringUnsafe(recipient),
+        ...deserializeSharedSecret(entry),
+      }));
+    });
+  }
+
+  /**
+   * Removes a pre-shared tagging secret scoped to a recipient. Both the secret and its kind must match.
+   * @returns true if the secret was registered and removed, false if it was not registered for that recipient.
+   */
+  removeSharedSecret(recipient: AztecAddress, kind: SharedSecretKind, secret: Point): Promise<boolean> {
+    return this.#store.transactionAsync(async () => {
+      const secretStr = secret.toString();
+      for await (const existing of this.#sharedSecretsByRecipient.getValuesAsync(recipient.toString())) {
+        if (existing.kind === kind && existing.secret === secretStr) {
+          await this.#sharedSecretsByRecipient.deleteValue(recipient.toString(), existing);
+          return true;
+        }
+      }
+
+      return false;
+    });
+  }
+}
+
+/** The wire form a shared secret entry is persisted as. */
+type StoredSharedSecret = { kind: SharedSecretKind; secret: string };
+
+function deserializeSharedSecret(entry: StoredSharedSecret): SharedSecret {
+  return { kind: entry.kind, secret: Point.fromString(entry.secret) };
+}

package/src/tagging/constants.ts

@@ -1,10 +1,42 @@
-// This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app
-// tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function.
-// This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than
-// MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note
-// and the note is nullified in the same tx.
-//
-// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than
-// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use
-// cases.
-export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20;
+import { MAX_PRIVATE_LOGS_PER_TX } from '@aztec/constants';
+
+// This window bounds how far a sender's next tag index for a directional app tagging secret can run ahead of that
+// secret's highest finalized index before `PXE::proveTx` throws.
+//
+// MAX_PRIVATE_LOGS_PER_TX is a floor for ordinary transaction patterns: the private kernel accumulates every
+// surviving private log into a single tx-wide array of exactly this size, so a fresh secret's very first
+// ordinary tx can consume that many indexes before anything is finalized.
+//
+// The +20 on top is headroom for multiple pending ordinary txs to the same counterparty stacking up
+// before the first one is observed as mined. Only a run of near-MAX_PRIVATE_LOGS_PER_TX txs back-to-back would
+// exhaust this; ordinary transfers use far fewer logs per tx, so the same margin covers many more of those. Keep the
+// margin additive rather than another multiple of the floor. A larger window also makes recipient-side sync more
+// expensive, since discovery has to probe ahead of the last finalized index for every active secret.
+//
+// TODO(F-783): MAX_PRIVATE_LOGS_PER_TX is not a hard ceiling in general — a tx that creates and nullifies (squashes)
+// many notes/logs to the same secret can drive that secret's raw tag index arbitrarily higher, since indexes are
+// reserved at log emission time, before squashing is decided, and the kernel's reset/squash loop is not bounded by
+// MAX_PRIVATE_LOGS_PER_TX. No fixed window value closes that gap.
+export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = MAX_PRIVATE_LOGS_PER_TX + 20;
+
+/**
+ * Exclusive upper bound of the tag indexes that can exist for a secret whose highest finalized index is
+ * `finalizedIndex` (undefined when nothing is finalized yet). The sender store refuses pending indexes at or past it,
+ * and both sender and recipient sync scan exactly up to it. Every absolute window bound must come from this helper:
+ * a site with a wider or narrower bound lets a tx land at an index the syncs never scan, so two stores sharing the
+ * secret could later pick a colliding index.
+ */
+export function unfinalizedTaggingIndexesWindowEnd(finalizedIndex: number | undefined): number {
+  const windowStart = finalizedIndex === undefined ? 0 : finalizedIndex + 1;
+  return windowStart + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN;
+}
+
+// The number of tags probed per constrained secret in the first round.
+//
+// The probe doubles each round (2, 4, 8, ..., capped at UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) while every probed
+// index is a hit, stopping at the first missing tag. Constrained delivery is gapless, so a single missing tag proves
+// the stream has ended: at steady state this turns a full WINDOW_LEN probe into two tags. A secret K logs behind
+// catches up in ~log2(K) round-trips while the probe is still doubling (2, 4, 8, 16, 32), but once it saturates the cap
+// and advances WINDOW_LEN tags per round, deeper catch-up is linear at ~K/WINDOW_LEN rounds. Either way it beats both
+// the full window every round and one round per log.
+export const INITIAL_CONSTRAINED_PROBE_LEN = 2;

package/src/tagging/get_all_logs_by_tags.ts

@@ -1,68 +1,157 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { allToCompletion } from '@aztec/foundation/promise';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import type { BlockHash } from '@aztec/stdlib/block';
-import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit';
+import { MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit';
 import type { AztecNode } from '@aztec/stdlib/interfaces/client';
-import type { SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs';
+import {
+  type LogIncludeOptions,
+  type LogResult,
+  type SiloedTag,
+  type Tag,
+  queryAllPrivateLogsByTags,
+  queryAllPublicLogsByTags,
+} from '@aztec/stdlib/logs';
+import type { BlockHeader } from '@aztec/stdlib/tx';
 
 /**
- * Generic pagination helper that fetches all pages of results.
- * @param numTags - The number of tags being queried (determines result array size).
- * @param fetchPage - Function that fetches a single page of results given a page number.
- * @returns An array of arrays, one per tag, containing all results across all pages.
+ * The block a tag query is anchored to.
+ *
+ * Both fields come from one header (see {@link logQueryAnchorOf}), which is what keeps the bound and the reorg check
+ * talking about the same block.
  */
-async function getAllPages<T>(numTags: number, fetchPage: (page: number) => Promise<T[][]>): Promise<T[][]> {
-  const allResultsPerTag: T[][] = Array.from({ length: numTags }, () => []);
-  let page = 0;
-  let hasMore = true;
+export type LogQueryAnchor = {
+  /**
+   * Hash of the anchor block, naming the chain the query must be answered on: the node throws if that block is gone,
+   * which is how a reorg surfaces.
+   */
+  hash: BlockHash;
+  /** Height of the anchor block, bounding the query to blocks at or below it. */
+  number: BlockNumber;
+};
 
-  while (hasMore) {
-    const resultsPage = await fetchPage(page);
-    hasMore = false;
+/**
+ * The {@link LogQueryAnchor} naming `anchorBlockHeader`: its hash and its height. Callers build it once and reuse
+ * it across every query anchored to that block, so the header is hashed only once.
+ */
+export async function logQueryAnchorOf(anchorBlockHeader: BlockHeader): Promise<LogQueryAnchor> {
+  return { hash: await anchorBlockHeader.hash(), number: anchorBlockHeader.getBlockNumber() };
+}
+
+/** Optional block-range, effects opt-in, and pagination cap shared by both wrappers. */
+export type GetAllLogsByTagsOptions = {
+  /** Lower block bound, inclusive. */
+  fromBlock?: BlockNumber;
+  /** Upper block bound, exclusive. */
+  toBlock?: BlockNumber;
+  /**
+   * When set, each log also carries `noteHashes` and all `nullifiers` of its tx. Defaults to off — sender
+   * sync only needs `txHash`. The recipient-sync/log-service paths flip this on to build `PendingTaggedLog`
+   * / `LogRetrievalResponse` from note data.
+   */
+  includeEffects?: boolean;
+  /**
+   * Maximum number of logs returned per tag per round-trip. Capped at `MAX_LOGS_PER_TAG` on the node
+   * (rejected if higher). Defaults to `MAX_LOGS_PER_TAG`. Mainly useful for tests that need to force
+   * pagination at a small page size.
+   */
+  limitPerTag?: number;
+};
+
+/**
+ * Splits tags into chunks of MAX_RPC_LEN, paginates each chunk via the stdlib per-tag cursor helper,
+ * then stitches the results back into a single array preserving the original tag order.
+ */
+async function getAllPagesInBatches<T extends Tag | SiloedTag, Opts extends LogIncludeOptions>(
+  tags: T[],
+  fetchAllPagesForBatch: (batch: T[]) => Promise<LogResult<Opts>[][]>,
+): Promise<LogResult<Opts>[][]> {
+  if (tags.length === 0) {
+    return [];
+  }
 
-    for (let i = 0; i < resultsPage.length; i++) {
-      allResultsPerTag[i].push(...resultsPage[i]);
-      if (resultsPage[i].length === MAX_LOGS_PER_TAG) {
-        hasMore = true;
-      }
-    }
-    page++;
+  if (tags.length <= MAX_RPC_LEN) {
+    return fetchAllPagesForBatch(tags);
   }
 
-  return allResultsPerTag;
+  const batches: T[][] = [];
+  for (let i = 0; i < tags.length; i += MAX_RPC_LEN) {
+    batches.push(tags.slice(i, i + MAX_RPC_LEN));
+  }
+  const batchResults = await allToCompletion(batches.map(fetchAllPagesForBatch));
+  return batchResults.flat();
 }
 
 /**
- * Fetches all private logs for the given tags, automatically paginating through all pages.
+ * Fetches all private logs for the given tags, automatically paginating per-tag via `afterLog` cursors.
+ *
  * @param aztecNode - The Aztec node to query.
  * @param tags - The siloed tags to search for.
- * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
- * because of reorgs).
+ * @param anchor - Block the query is anchored to: results stop there, and the call throws if that block is not found
+ *   (typically because of reorgs).
+ * @param options - Optional `fromBlock`/`toBlock` range and `includeEffects` opt-in.
  * @returns An array of log arrays, one per tag, containing all logs across all pages.
  */
-export function getAllPrivateLogsByTags(
+export function getAllPrivateLogsByTags<Opts extends GetAllLogsByTagsOptions = GetAllLogsByTagsOptions>(
   aztecNode: AztecNode,
   tags: SiloedTag[],
-  anchorBlockHash: BlockHash,
-): Promise<TxScopedL2Log[][]> {
-  return getAllPages(tags.length, page => aztecNode.getPrivateLogsByTags(tags, page, anchorBlockHash));
+  anchor: LogQueryAnchor,
+  options: Opts = {} as Opts,
+): Promise<LogResult<Opts>[][]> {
+  return getAllPagesInBatches<SiloedTag, Opts>(
+    tags,
+    batch =>
+      queryAllPrivateLogsByTags(aztecNode, {
+        tags: batch,
+        referenceBlock: anchor.hash,
+        fromBlock: options.fromBlock,
+        toBlock: exclusiveUpperBound(anchor, options.toBlock),
+        includeEffects: options.includeEffects ?? false,
+        limitPerTag: options.limitPerTag,
+      }) as Promise<LogResult<Opts>[][]>,
+  );
 }
 
 /**
- * Fetches all public logs for the given tags from a contract, automatically paginating through all pages.
+ * Fetches all public logs for the given tags from a contract, automatically paginating per-tag via `afterLog` cursors.
+ *
  * @param aztecNode - The Aztec node to query.
  * @param contractAddress - The contract address to search logs for.
  * @param tags - The tags to search for.
- * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically
- * because of reorgs).
+ * @param anchor - Block the query is anchored to: results stop there, and the call throws if that block is not found
+ *   (typically because of reorgs).
+ * @param options - Optional `fromBlock`/`toBlock` range and `includeEffects` opt-in.
  * @returns An array of log arrays, one per tag, containing all logs across all pages.
  */
-export function getAllPublicLogsByTagsFromContract(
+export function getAllPublicLogsByTagsFromContract<Opts extends GetAllLogsByTagsOptions = GetAllLogsByTagsOptions>(
   aztecNode: AztecNode,
   contractAddress: AztecAddress,
   tags: Tag[],
-  anchorBlockHash: BlockHash,
-): Promise<TxScopedL2Log[][]> {
-  return getAllPages(tags.length, page =>
-    aztecNode.getPublicLogsByTagsFromContract(contractAddress, tags, page, anchorBlockHash),
+  anchor: LogQueryAnchor,
+  options: Opts = {} as Opts,
+): Promise<LogResult<Opts>[][]> {
+  return getAllPagesInBatches<Tag, Opts>(
+    tags,
+    batch =>
+      queryAllPublicLogsByTags(aztecNode, {
+        contractAddress,
+        tags: batch,
+        referenceBlock: anchor.hash,
+        fromBlock: options.fromBlock,
+        toBlock: exclusiveUpperBound(anchor, options.toBlock),
+        includeEffects: options.includeEffects ?? false,
+        limitPerTag: options.limitPerTag,
+      }) as Promise<LogResult<Opts>[][]>,
   );
 }
+
+/**
+ * Exclusive upper block bound for a query anchored at `anchor`: one past the anchor block, or the caller's own
+ * `toBlock` when that is tighter.
+ *
+ * The bound is spelled out in the request rather than left implicit in `referenceBlock`, so the request itself
+ * names a fixed block range that no later block can widen.
+ */
+function exclusiveUpperBound(anchor: LogQueryAnchor, toBlock: BlockNumber | undefined): BlockNumber {
+  return BlockNumber(Math.min(toBlock ?? Infinity, anchor.number + 1));
+}

package/src/tagging/index.ts

@@ -4,16 +4,22 @@
  * The objective of the sender sync algorithm is to determine which tags have already been used by a sender, thereby
  * deciding which tag should be used next.
  *
- * The objective of the recipient sync algorithm is to load and process the corresponding logs.
+ * The objective of the recipient sync algorithm is to fetch and sync the corresponding logs.
  *
  * @module tagging
  */
 
-export { loadPrivateLogsForSenderRecipientPair } from './recipient_sync/load_private_logs_for_sender_recipient_pair.js';
+export { syncTaggedPrivateLogs } from './recipient_sync/sync_tagged_private_logs.js';
 export { syncSenderTaggingIndexes } from './sender_sync/sync_sender_tagging_indexes.js';
-export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from './constants.js';
-export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './get_all_logs_by_tags.js';
+export { persistSenderTaggingIndexRangesForTx } from './persist_sender_tagging_index_ranges.js';
+export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, INITIAL_CONSTRAINED_PROBE_LEN } from './constants.js';
+export {
+  type LogQueryAnchor,
+  getAllPrivateLogsByTags,
+  getAllPublicLogsByTagsFromContract,
+  logQueryAnchorOf,
+} from './get_all_logs_by_tags.js';
 
 // Re-export tagging-related types from stdlib
-export { DirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs';
-export { type PreTag } from '@aztec/stdlib/logs';
+export { AppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs';
+export { type PreTag, type TaggingIndexRange } from '@aztec/stdlib/logs';

package/src/tagging/persist_sender_tagging_index_ranges.ts

@@ -0,0 +1,57 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { PrivateKernelTailCircuitPublicInputs } from '@aztec/stdlib/kernel';
+import type { TaggingIndexRange } from '@aztec/stdlib/logs';
+import type { TxHash } from '@aztec/stdlib/tx';
+
+import type { SenderTaggingStore } from '../storage/tagging_store/sender_tagging_store.js';
+import { reconcileTaggingIndexRangesAgainstSurvivingTags } from './reconcile_tagging_index_ranges.js';
+
+/**
+ * Persists the tagging index ranges that a tx used as a sender, after reconciling them against the kernel's surviving
+ * private logs.
+ *
+ * The tagging index cache reserves an index for every log emission attempted during private execution, but the kernel
+ * may then squash some of those logs (e.g. when a note is created and nullified within the same tx). This function
+ * shrinks the recorded ranges to match the actual on-chain footprint before writing them, so that what we persist is
+ * consistent with what the network will see.
+ *
+ * @remarks Storing the recorded ranges in the DB may not be seen as necessary because we sync from chain before sending
+ * new logs, but the sync can only see logs already included in blocks. If we sent another transaction before this one
+ * was included from the same PXE, and that transaction contained a log with a tag derived from the same secret, we
+ * would reuse the tag and the transactions would be linked. Persisting in the DB prevents that linkage.
+ *
+ * @param store - The sender tagging store.
+ * @param recordedRanges - The tagging index ranges as recorded during private execution (pre-squash).
+ * @param publicInputs - The final kernel public inputs, used to determine which private logs survived squashing.
+ * @param getTxHash - Lazy accessor for the tx hash. Called only when there is something to persist, since computing
+ * the tx hash is expensive.
+ * @param jobId - Job context for staged writes to the store. See `JobCoordinator` for more details.
+ * @param log - Logger.
+ */
+export async function persistSenderTaggingIndexRangesForTx(
+  store: SenderTaggingStore,
+  recordedRanges: TaggingIndexRange[],
+  publicInputs: PrivateKernelTailCircuitPublicInputs,
+  getTxHash: () => Promise<TxHash>,
+  jobId: string,
+  log: Logger,
+): Promise<void> {
+  if (recordedRanges.length === 0) {
+    log.debug(`No tagging index ranges used in the tx`);
+    return;
+  }
+
+  const survivingTags = new Set(
+    publicInputs.getNonEmptyPrivateLogs().map(privateLog => privateLog.fields[0].toString()),
+  );
+  const reconciledRanges = await reconcileTaggingIndexRangesAgainstSurvivingTags(recordedRanges, survivingTags);
+
+  if (reconciledRanges.length === 0) {
+    log.debug(`All tagging index ranges used in the tx were squashed by the kernel`, { recordedRanges });
+    return;
+  }
+
+  const txHash = await getTxHash();
+  await store.storePendingIndexes(reconciledRanges, txHash, jobId);
+  log.debug(`Stored used tagging index ranges as sender for the tx`, { recordedRanges, reconciledRanges });
+}