@aztec/archiver 3.0.0-nightly.20251222 → 3.0.0-nightly.20251224

package/src/archiver/kv_archiver_store/log_store.ts

@@ -4,6 +4,7 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import { createLogger } from '@aztec/foundation/log';
 import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
+import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { L2BlockHash, L2BlockNew } from '@aztec/stdlib/block';
 import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
 import {
@@ -13,6 +14,8 @@ import {
   type LogFilter,
   LogId,
   PublicLog,
+  type SiloedTag,
+  Tag,
   TxScopedL2Log,
 } from '@aztec/stdlib/logs';
 
@@ -22,8 +25,12 @@ import type { BlockStore } from './block_store.js';
  * A store for logs
  */
 export class LogStore {
-  #logsByTag: AztecAsyncMap<string, Buffer[]>;
-  #logTagsByBlock: AztecAsyncMap<number, string[]>;
+  // `tag` --> private logs
+  #privateLogsByTag: AztecAsyncMap<string, Buffer[]>;
+  // `{contractAddress}_${tag}` --> public logs
+  #publicLogsByContractAndTag: AztecAsyncMap<string, Buffer[]>;
+  #privateLogKeysByBlock: AztecAsyncMap<number, string[]>;
+  #publicLogKeysByBlock: AztecAsyncMap<number, string[]>;
   #publicLogsByBlock: AztecAsyncMap<number, Buffer>;
   #contractClassLogsByBlock: AztecAsyncMap<number, Buffer>;
   #logsMaxPageSize: number;
@@ -34,86 +41,173 @@ export class LogStore {
     private blockStore: BlockStore,
     logsMaxPageSize: number = 1000,
   ) {
-    this.#logsByTag = db.openMap('archiver_tagged_logs_by_tag');
-    this.#logTagsByBlock = db.openMap('archiver_log_tags_by_block');
+    this.#privateLogsByTag = db.openMap('archiver_private_tagged_logs_by_tag');
+    this.#publicLogsByContractAndTag = db.openMap('archiver_public_tagged_logs_by_tag');
+    this.#privateLogKeysByBlock = db.openMap('archiver_private_log_keys_by_block');
+    this.#publicLogKeysByBlock = db.openMap('archiver_public_log_keys_by_block');
     this.#publicLogsByBlock = db.openMap('archiver_public_logs_by_block');
     this.#contractClassLogsByBlock = db.openMap('archiver_contract_class_logs_by_block');
 
     this.#logsMaxPageSize = logsMaxPageSize;
   }
 
-  async #extractTaggedLogs(block: L2BlockNew) {
+  /**
+   * Extracts tagged logs from a single block, grouping them into private and public maps.
+   *
+   * @param block - The L2 block to extract logs from.
+   * @returns An object containing the private and public tagged logs for the block.
+   */
+  async #extractTaggedLogsFromBlock(block: L2BlockNew) {
     const blockHash = L2BlockHash.fromField(await block.hash());
-    const taggedLogs = new Map<string, Buffer[]>();
+    // SiloedTag (as string) -> array of log buffers.
+    const privateTaggedLogs = new Map<string, Buffer[]>();
+    // "{contractAddress}_{tag}" (as string) -> array of log buffers.
+    const publicTaggedLogs = new Map<string, Buffer[]>();
     const dataStartIndexForBlock =
       block.header.state.partial.noteHashTree.nextAvailableLeafIndex -
       block.body.txEffects.length * MAX_NOTE_HASHES_PER_TX;
+
     block.body.txEffects.forEach((txEffect, txIndex) => {
       const txHash = txEffect.txHash;
       const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NOTE_HASHES_PER_TX;
 
       txEffect.privateLogs.forEach((log, logIndex) => {
+        // Private logs use SiloedTag (already siloed by kernel)
         const tag = log.fields[0];
         this.#log.debug(`Found private log with tag ${tag.toString()} in block ${block.number}`);
 
-        const currentLogs = taggedLogs.get(tag.toString()) ?? [];
+        const currentLogs = privateTaggedLogs.get(tag.toString()) ?? [];
         currentLogs.push(
-          new TxScopedL2Log(txHash, dataStartIndexForTx, logIndex, block.number, blockHash, log).toBuffer(),
+          new TxScopedL2Log(
+            txHash,
+            dataStartIndexForTx,
+            logIndex,
+            block.number,
+            blockHash,
+            block.timestamp,
+            log,
+          ).toBuffer(),
         );
-        taggedLogs.set(tag.toString(), currentLogs);
+        privateTaggedLogs.set(tag.toString(), currentLogs);
       });
 
       txEffect.publicLogs.forEach((log, logIndex) => {
+        // Public logs use Tag directly (not siloed) and are stored with contract address
         const tag = log.fields[0];
-        this.#log.debug(`Found public log with tag ${tag.toString()} in block ${block.number}`);
+        const contractAddress = log.contractAddress;
+        const key = `${contractAddress.toString()}_${tag.toString()}`;
+        this.#log.debug(
+          `Found public log with tag ${tag.toString()} from contract ${contractAddress.toString()} in block ${block.number}`,
+        );
 
-        const currentLogs = taggedLogs.get(tag.toString()) ?? [];
+        const currentLogs = publicTaggedLogs.get(key) ?? [];
         currentLogs.push(
-          new TxScopedL2Log(txHash, dataStartIndexForTx, logIndex, block.number, blockHash, log).toBuffer(),
+          new TxScopedL2Log(
+            txHash,
+            dataStartIndexForTx,
+            logIndex,
+            block.number,
+            blockHash,
+            block.timestamp,
+            log,
+          ).toBuffer(),
         );
-        taggedLogs.set(tag.toString(), currentLogs);
+        publicTaggedLogs.set(key, currentLogs);
       });
     });
-    return taggedLogs;
+
+    return { privateTaggedLogs, publicTaggedLogs };
   }
 
   /**
-   * Append new logs to the store's list.
-   * @param blocks - The blocks for which to add the logs.
-   * @returns True if the operation is successful.
+   * Extracts and aggregates tagged logs from a list of blocks.
+   * @param blocks - The blocks to extract logs from.
+   * @returns A map from tag (as string) to an array of serialized private logs belonging to that tag, and a map from
+   * "{contractAddress}_{tag}" (as string) to an array of serialized public logs belonging to that key.
    */
-  async addLogs(blocks: L2BlockNew[]): Promise<boolean> {
-    const taggedLogsInBlocks = await Promise.all(blocks.map(block => this.#extractTaggedLogs(block)));
-    const taggedLogsToAdd = taggedLogsInBlocks.reduce((acc, taggedLogs) => {
-      for (const [tag, logs] of taggedLogs.entries()) {
+  async #extractTaggedLogs(
+    blocks: L2BlockNew[],
+  ): Promise<{ privateTaggedLogs: Map<string, Buffer[]>; publicTaggedLogs: Map<string, Buffer[]> }> {
+    const taggedLogsInBlocks = await Promise.all(blocks.map(block => this.#extractTaggedLogsFromBlock(block)));
+
+    // Now we merge the maps from each block into a single map.
+    const privateTaggedLogs = taggedLogsInBlocks.reduce((acc, { privateTaggedLogs }) => {
+      for (const [tag, logs] of privateTaggedLogs.entries()) {
         const currentLogs = acc.get(tag) ?? [];
         acc.set(tag, currentLogs.concat(logs));
       }
       return acc;
     }, new Map<string, Buffer[]>());
-    const tagsToUpdate = Array.from(taggedLogsToAdd.keys());
+
+    const publicTaggedLogs = taggedLogsInBlocks.reduce((acc, { publicTaggedLogs }) => {
+      for (const [key, logs] of publicTaggedLogs.entries()) {
+        const currentLogs = acc.get(key) ?? [];
+        acc.set(key, currentLogs.concat(logs));
+      }
+      return acc;
+    }, new Map<string, Buffer[]>());
+
+    return { privateTaggedLogs, publicTaggedLogs };
+  }
+
+  /**
+   * Append new logs to the store's list.
+   * @param blocks - The blocks for which to add the logs.
+   * @returns True if the operation is successful.
+   */
+  async addLogs(blocks: L2BlockNew[]): Promise<boolean> {
+    const { privateTaggedLogs, publicTaggedLogs } = await this.#extractTaggedLogs(blocks);
+
+    const keysOfPrivateLogsToUpdate = Array.from(privateTaggedLogs.keys());
+    const keysOfPublicLogsToUpdate = Array.from(publicTaggedLogs.keys());
 
     return this.db.transactionAsync(async () => {
-      const currentTaggedLogs = await Promise.all(
-        tagsToUpdate.map(async tag => ({ tag, logBuffers: await this.#logsByTag.getAsync(tag) })),
+      const currentPrivateTaggedLogs = await Promise.all(
+        keysOfPrivateLogsToUpdate.map(async key => ({
+          tag: key,
+          logBuffers: await this.#privateLogsByTag.getAsync(key),
+        })),
       );
-      currentTaggedLogs.forEach(taggedLogBuffer => {
+      currentPrivateTaggedLogs.forEach(taggedLogBuffer => {
         if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
-          taggedLogsToAdd.set(
+          privateTaggedLogs.set(
             taggedLogBuffer.tag,
-            taggedLogBuffer.logBuffers!.concat(taggedLogsToAdd.get(taggedLogBuffer.tag)!),
+            taggedLogBuffer.logBuffers!.concat(privateTaggedLogs.get(taggedLogBuffer.tag)!),
+          );
+        }
+      });
+
+      const currentPublicTaggedLogs = await Promise.all(
+        keysOfPublicLogsToUpdate.map(async key => ({
+          key,
+          logBuffers: await this.#publicLogsByContractAndTag.getAsync(key),
+        })),
+      );
+      currentPublicTaggedLogs.forEach(taggedLogBuffer => {
+        if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
+          publicTaggedLogs.set(
+            taggedLogBuffer.key,
+            taggedLogBuffer.logBuffers!.concat(publicTaggedLogs.get(taggedLogBuffer.key)!),
           );
         }
       });
+
       for (const block of blocks) {
         const blockHash = await block.hash();
 
-        const tagsInBlock = [];
-        for (const [tag, logs] of taggedLogsToAdd.entries()) {
-          await this.#logsByTag.set(tag, logs);
-          tagsInBlock.push(tag);
+        const privateTagsInBlock: string[] = [];
+        for (const [tag, logs] of privateTaggedLogs.entries()) {
+          await this.#privateLogsByTag.set(tag, logs);
+          privateTagsInBlock.push(tag);
         }
-        await this.#logTagsByBlock.set(block.number, tagsInBlock);
+        await this.#privateLogKeysByBlock.set(block.number, privateTagsInBlock);
+
+        const publicKeysInBlock: string[] = [];
+        for (const [key, logs] of publicTaggedLogs.entries()) {
+          await this.#publicLogsByContractAndTag.set(key, logs);
+          publicKeysInBlock.push(key);
+        }
+        await this.#publicLogKeysByBlock.set(block.number, publicKeysInBlock);
 
         const publicLogsInBlock = block.body.txEffects
           .map((txEffect, txIndex) =>
@@ -162,44 +256,55 @@ export class LogStore {
 
   deleteLogs(blocks: L2BlockNew[]): Promise<boolean> {
     return this.db.transactionAsync(async () => {
-      const tagsToDelete = (
-        await Promise.all(
-          blocks.map(async block => {
-            const tags = await this.#logTagsByBlock.getAsync(block.number);
-            return tags ?? [];
-          }),
-        )
-      ).flat();
+      await Promise.all(
+        blocks.map(async block => {
+          // Delete private logs
+          const privateKeys = (await this.#privateLogKeysByBlock.getAsync(block.number)) ?? [];
+          await Promise.all(privateKeys.map(tag => this.#privateLogsByTag.delete(tag)));
+
+          // Delete public logs
+          const publicKeys = (await this.#publicLogKeysByBlock.getAsync(block.number)) ?? [];
+          await Promise.all(publicKeys.map(key => this.#publicLogsByContractAndTag.delete(key)));
+        }),
+      );
 
       await Promise.all(
         blocks.map(block =>
           Promise.all([
             this.#publicLogsByBlock.delete(block.number),
-            this.#logTagsByBlock.delete(block.number),
+            this.#privateLogKeysByBlock.delete(block.number),
+            this.#publicLogKeysByBlock.delete(block.number),
             this.#contractClassLogsByBlock.delete(block.number),
           ]),
         ),
       );
 
-      await Promise.all(tagsToDelete.map(tag => this.#logsByTag.delete(tag.toString())));
       return true;
     });
   }
 
   /**
-   * Gets all logs that match any of the received tags (i.e. logs with their first field equal to a tag).
-   * @param tags - The tags to filter the logs by.
-   * @returns For each received tag, an array of matching logs is returned. An empty array implies no logs match
-   * that tag.
+   * Gets all private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty
+   * array implies no logs match that tag.
    */
-  async getLogsByTags(tags: Fr[], limitPerTag?: number): Promise<TxScopedL2Log[][]> {
-    if (limitPerTag !== undefined && limitPerTag <= 0) {
-      throw new TypeError('limitPerTag needs to be greater than 0');
-    }
-    const logs = await Promise.all(tags.map(tag => this.#logsByTag.getAsync(tag.toString())));
-    return logs.map(
-      logBuffers => logBuffers?.slice(0, limitPerTag).map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? [],
+  async getPrivateLogsByTags(tags: SiloedTag[]): Promise<TxScopedL2Log[][]> {
+    const logs = await Promise.all(tags.map(tag => this.#privateLogsByTag.getAsync(tag.toString())));
+
+    return logs.map(logBuffers => logBuffers?.map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
+  }
+
+  /**
+   * Gets all public logs that match any of the `tags` from the specified contract. For each tag, an array of matching
+   * logs is returned. An empty array implies no logs match that tag.
+   */
+  async getPublicLogsByTagsFromContract(contractAddress: AztecAddress, tags: Tag[]): Promise<TxScopedL2Log[][]> {
+    const logs = await Promise.all(
+      tags.map(tag => {
+        const key = `${contractAddress.toString()}_${tag.value.toString()}`;
+        return this.#publicLogsByContractAndTag.getAsync(key);
+      }),
     );
+    return logs.map(logBuffers => logBuffers?.map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
   }
 
   /**

package/src/archiver/l1/bin/retrieve-calldata.ts

@@ -76,7 +76,7 @@ async function main() {
     // Create viem public client
     const publicClient = createPublicClient({
       chain: mainnet,
-      transport: http(rpcUrl),
+      transport: http(rpcUrl, { batch: false }),
     });
 
     logger.info('Fetching transaction...');

package/src/archiver/l1/data_retrieval.ts

@@ -1,3 +1,4 @@
+import type { BlobClientInterface } from '@aztec/blob-client/client';
 import {
   BlobDeserializationError,
   type CheckpointBlobData,
@@ -5,7 +6,6 @@ import {
   decodeCheckpointBlobDataFromBlobs,
   encodeBlockBlobData,
 } from '@aztec/blob-lib';
-import type { BlobSinkClientInterface } from '@aztec/blob-sink/client';
 import type { EpochProofPublicInputArgs } from '@aztec/ethereum/contracts';
 import type { ViemClient, ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
 import { asyncPool } from '@aztec/foundation/async-pool';
@@ -140,7 +140,7 @@ export async function retrievedToPublishedCheckpoint({
  * @param rollup - The rollup contract instance.
  * @param publicClient - The viem public client to use for transaction retrieval.
  * @param debugClient - The viem debug client to use for trace/debug RPC methods (optional).
- * @param blobSinkClient - The blob sink client for fetching blob data.
+ * @param blobClient - The blob client client for fetching blob data.
  * @param searchStartBlock - The block number to use for starting the search.
  * @param searchEndBlock - The highest block number that we should search up to.
  * @param contractAddresses - The contract addresses (governanceProposerAddress, slashFactoryAddress, slashingProposerAddress).
@@ -153,7 +153,7 @@ export async function retrieveCheckpointsFromRollup(
   rollup: GetContractReturnType<typeof RollupAbi, ViemPublicClient>,
   publicClient: ViemPublicClient,
   debugClient: ViemPublicDebugClient,
-  blobSinkClient: BlobSinkClientInterface,
+  blobClient: BlobClientInterface,
   searchStartBlock: bigint,
   searchEndBlock: bigint,
   contractAddresses: {
@@ -209,7 +209,7 @@ export async function retrieveCheckpointsFromRollup(
       rollup,
       publicClient,
       debugClient,
-      blobSinkClient,
+      blobClient,
       checkpointProposedLogs,
       rollupConstants,
       contractAddresses,
@@ -230,7 +230,7 @@ export async function retrieveCheckpointsFromRollup(
  * @param rollup - The rollup contract instance.
  * @param publicClient - The viem public client to use for transaction retrieval.
  * @param debugClient - The viem debug client to use for trace/debug RPC methods (optional).
- * @param blobSinkClient - The blob sink client for fetching blob data.
+ * @param blobClient - The blob client client for fetching blob data.
  * @param logs - CheckpointProposed logs.
  * @param rollupConstants - The rollup constants (chainId, version, targetCommitteeSize).
  * @param contractAddresses - The contract addresses (governanceProposerAddress, slashFactoryAddress, slashingProposerAddress).
@@ -243,7 +243,7 @@ async function processCheckpointProposedLogs(
   rollup: GetContractReturnType<typeof RollupAbi, ViemPublicClient>,
   publicClient: ViemPublicClient,
   debugClient: ViemPublicDebugClient,
-  blobSinkClient: BlobSinkClientInterface,
+  blobClient: BlobClientInterface,
   logs: GetContractEventsReturnType<typeof RollupAbi, 'CheckpointProposed'>,
   { chainId, version, targetCommitteeSize }: { chainId: Fr; version: Fr; targetCommitteeSize: number },
   contractAddresses: {
@@ -286,7 +286,7 @@ async function processCheckpointProposedLogs(
         expectedHashes,
       );
       const checkpointBlobData = await getCheckpointBlobDataFromBlobs(
-        blobSinkClient,
+        blobClient,
         checkpoint.blockHash,
         blobHashes,
         checkpointNumber,
@@ -324,14 +324,14 @@ export async function getL1BlockTime(publicClient: ViemPublicClient, blockNumber
 }
 
 export async function getCheckpointBlobDataFromBlobs(
-  blobSinkClient: BlobSinkClientInterface,
+  blobClient: BlobClientInterface,
   blockHash: string,
   blobHashes: Buffer<ArrayBufferLike>[],
   checkpointNumber: CheckpointNumber,
   logger: Logger,
   isHistoricalSync: boolean,
 ): Promise<CheckpointBlobData> {
-  const blobBodies = await blobSinkClient.getBlobSidecar(blockHash, blobHashes, undefined, { isHistoricalSync });
+  const blobBodies = await blobClient.getBlobSidecar(blockHash, blobHashes, { isHistoricalSync });
   if (blobBodies.length === 0) {
     throw new NoBlobBodiesFoundError(checkpointNumber);
   }
@@ -339,7 +339,7 @@ export async function getCheckpointBlobDataFromBlobs(
   let checkpointBlobData: CheckpointBlobData;
   try {
     // Attempt to decode the checkpoint blob data.
-    checkpointBlobData = decodeCheckpointBlobDataFromBlobs(blobBodies.map(b => b.blob));
+    checkpointBlobData = decodeCheckpointBlobDataFromBlobs(blobBodies);
   } catch (err: any) {
     if (err instanceof BlobDeserializationError) {
       logger.fatal(err.message);

package/src/factory.ts

@@ -28,7 +28,7 @@ export async function createArchiverStore(
 /**
  * Creates a local archiver.
  * @param config - The archiver configuration.
- * @param blobSinkClient - The blob sink client.
+ * @param blobClient - The blob client client.
  * @param opts - The options.
  * @param telemetry - The telemetry client.
  * @returns The local archiver.