@aztec/archiver 0.0.1-commit.f295ac2 → 0.0.1-commit.f2ce05ee

package/src/store/log_store.ts

@@ -6,7 +6,8 @@ import { createLogger } from '@aztec/foundation/log';
 import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
-import { L2BlockHash, L2BlockNew } from '@aztec/stdlib/block';
+import { BlockHash, L2Block } from '@aztec/stdlib/block';
+import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit';
 import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
 import {
   ContractClassLog,
@@ -19,6 +20,7 @@ import {
   Tag,
   TxScopedL2Log,
 } from '@aztec/stdlib/logs';
+import { TxHash } from '@aztec/stdlib/tx';
 
 import type { BlockStore } from './block_store.js';
 
@@ -58,7 +60,7 @@ export class LogStore {
    * @param block - The L2 block to extract logs from.
    * @returns An object containing the private and public tagged logs for the block.
    */
-  #extractTaggedLogsFromBlock(block: L2BlockNew) {
+  #extractTaggedLogsFromBlock(block: L2Block) {
     // SiloedTag (as string) -> array of log buffers.
     const privateTaggedLogs = new Map<string, Buffer[]>();
     // "{contractAddress}_{tag}" (as string) -> array of log buffers.
@@ -119,7 +121,7 @@ export class LogStore {
    * @returns A map from tag (as string) to an array of serialized private logs belonging to that tag, and a map from
    * "{contractAddress}_{tag}" (as string) to an array of serialized public logs belonging to that key.
    */
-  #extractTaggedLogs(blocks: L2BlockNew[]): {
+  #extractTaggedLogs(blocks: L2Block[]): {
     privateTaggedLogs: Map<string, Buffer[]>;
     publicTaggedLogs: Map<string, Buffer[]>;
   } {
@@ -145,7 +147,7 @@ export class LogStore {
     return { privateTaggedLogs, publicTaggedLogs };
   }
 
-  async #addPrivateLogs(blocks: L2BlockNew[]): Promise<void> {
+  async #addPrivateLogs(blocks: L2Block[]): Promise<void> {
     const newBlocks = await filterAsync(
       blocks,
       async block => !(await this.#privateLogKeysByBlock.hasAsync(block.number)),
@@ -180,7 +182,7 @@ export class LogStore {
     }
   }
 
-  async #addPublicLogs(blocks: L2BlockNew[]): Promise<void> {
+  async #addPublicLogs(blocks: L2Block[]): Promise<void> {
     const newBlocks = await filterAsync(
       blocks,
       async block => !(await this.#publicLogKeysByBlock.hasAsync(block.number)),
@@ -218,6 +220,7 @@ export class LogStore {
         .map((txEffect, txIndex) =>
           [
             numToUInt32BE(txIndex),
+            txEffect.txHash.toBuffer(),
             numToUInt32BE(txEffect.publicLogs.length),
             txEffect.publicLogs.map(log => log.toBuffer()),
           ].flat(),
@@ -228,7 +231,7 @@ export class LogStore {
     }
   }
 
-  async #addContractClassLogs(blocks: L2BlockNew[]): Promise<void> {
+  async #addContractClassLogs(blocks: L2Block[]): Promise<void> {
     const newBlocks = await filterAsync(
       blocks,
       async block => !(await this.#contractClassLogsByBlock.hasAsync(block.number)),
@@ -241,6 +244,7 @@ export class LogStore {
         .map((txEffect, txIndex) =>
           [
             numToUInt32BE(txIndex),
+            txEffect.txHash.toBuffer(),
             numToUInt32BE(txEffect.contractClassLogs.length),
             txEffect.contractClassLogs.map(log => log.toBuffer()),
           ].flat(),
@@ -259,7 +263,7 @@ export class LogStore {
    * @param blocks - The blocks for which to add the logs.
    * @returns True if the operation is successful.
    */
-  addLogs(blocks: L2BlockNew[]): Promise<boolean> {
+  addLogs(blocks: L2Block[]): Promise<boolean> {
     return this.db.transactionAsync(async () => {
       await Promise.all([
         this.#addPrivateLogs(blocks),
@@ -270,21 +274,21 @@ export class LogStore {
     });
   }
 
-  #packWithBlockHash(blockHash: Fr, data: Buffer<ArrayBufferLike>[]): Buffer<ArrayBufferLike> {
+  #packWithBlockHash(blockHash: BlockHash, data: Buffer<ArrayBufferLike>[]): Buffer<ArrayBufferLike> {
     return Buffer.concat([blockHash.toBuffer(), ...data]);
   }
 
-  #unpackBlockHash(reader: BufferReader): L2BlockHash {
+  #unpackBlockHash(reader: BufferReader): BlockHash {
     const blockHash = reader.remainingBytes() > 0 ? reader.readObject(Fr) : undefined;
 
     if (!blockHash) {
       throw new Error('Failed to read block hash from log entry buffer');
     }
 
-    return L2BlockHash.fromField(blockHash);
+    return new BlockHash(blockHash);
   }
 
-  deleteLogs(blocks: L2BlockNew[]): Promise<boolean> {
+  deleteLogs(blocks: L2Block[]): Promise<boolean> {
     return this.db.transactionAsync(async () => {
       await Promise.all(
         blocks.map(async block => {
@@ -314,27 +318,49 @@ export class LogStore {
   }
 
   /**
-   * Gets all private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty
+   * Gets private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty
    * array implies no logs match that tag.
+   * @param tags - The tags to search for.
+   * @param page - The page number (0-indexed) for pagination.
+   * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
+   * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
    */
-  async getPrivateLogsByTags(tags: SiloedTag[]): Promise<TxScopedL2Log[][]> {
+  async getPrivateLogsByTags(tags: SiloedTag[], page: number = 0): Promise<TxScopedL2Log[][]> {
     const logs = await Promise.all(tags.map(tag => this.#privateLogsByTag.getAsync(tag.toString())));
+    const start = page * MAX_LOGS_PER_TAG;
+    const end = start + MAX_LOGS_PER_TAG;
 
-    return logs.map(logBuffers => logBuffers?.map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
+    return logs.map(
+      logBuffers => logBuffers?.slice(start, end).map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? [],
+    );
   }
 
   /**
-   * Gets all public logs that match any of the `tags` from the specified contract. For each tag, an array of matching
+   * Gets public logs that match any of the `tags` from the specified contract. For each tag, an array of matching
    * logs is returned. An empty array implies no logs match that tag.
+   * @param contractAddress - The contract address to search logs for.
+   * @param tags - The tags to search for.
+   * @param page - The page number (0-indexed) for pagination.
+   * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
+   * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
    */
-  async getPublicLogsByTagsFromContract(contractAddress: AztecAddress, tags: Tag[]): Promise<TxScopedL2Log[][]> {
+  async getPublicLogsByTagsFromContract(
+    contractAddress: AztecAddress,
+    tags: Tag[],
+    page: number = 0,
+  ): Promise<TxScopedL2Log[][]> {
     const logs = await Promise.all(
       tags.map(tag => {
         const key = `${contractAddress.toString()}_${tag.value.toString()}`;
         return this.#publicLogsByContractAndTag.getAsync(key);
       }),
     );
-    return logs.map(logBuffers => logBuffers?.map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
+    const start = page * MAX_LOGS_PER_TAG;
+    const end = start + MAX_LOGS_PER_TAG;
+
+    return logs.map(
+      logBuffers => logBuffers?.slice(start, end).map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? [],
+    );
   }
 
   /**
@@ -363,24 +389,33 @@ export class LogStore {
     }
 
     const buffer = (await this.#publicLogsByBlock.getAsync(blockNumber)) ?? Buffer.alloc(0);
-    const publicLogsInBlock: [PublicLog[]] = [[]];
+    const publicLogsInBlock: { txHash: TxHash; logs: PublicLog[] }[] = [];
     const reader = new BufferReader(buffer);
 
     const blockHash = this.#unpackBlockHash(reader);
 
     while (reader.remainingBytes() > 0) {
       const indexOfTx = reader.readNumber();
+      const txHash = reader.readObject(TxHash);
       const numLogsInTx = reader.readNumber();
-      publicLogsInBlock[indexOfTx] = [];
+      publicLogsInBlock[indexOfTx] = { txHash, logs: [] };
       for (let i = 0; i < numLogsInTx; i++) {
-        publicLogsInBlock[indexOfTx].push(reader.readObject(PublicLog));
+        publicLogsInBlock[indexOfTx].logs.push(reader.readObject(PublicLog));
       }
     }
 
-    const txLogs = publicLogsInBlock[txIndex];
+    const txData = publicLogsInBlock[txIndex];
 
     const logs: ExtendedPublicLog[] = [];
-    const maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+    const maxLogsHit = this.#accumulatePublicLogs(
+      logs,
+      blockNumber,
+      blockHash,
+      txIndex,
+      txData.txHash,
+      txData.logs,
+      filter,
+    );
 
     return { logs, maxLogsHit };
   }
@@ -401,22 +436,31 @@ export class LogStore {
 
     let maxLogsHit = false;
     loopOverBlocks: for await (const [blockNumber, logBuffer] of this.#publicLogsByBlock.entriesAsync({ start, end })) {
-      const publicLogsInBlock: [PublicLog[]] = [[]];
+      const publicLogsInBlock: { txHash: TxHash; logs: PublicLog[] }[] = [];
       const reader = new BufferReader(logBuffer);
 
       const blockHash = this.#unpackBlockHash(reader);
 
       while (reader.remainingBytes() > 0) {
         const indexOfTx = reader.readNumber();
+        const txHash = reader.readObject(TxHash);
         const numLogsInTx = reader.readNumber();
-        publicLogsInBlock[indexOfTx] = [];
+        publicLogsInBlock[indexOfTx] = { txHash, logs: [] };
         for (let i = 0; i < numLogsInTx; i++) {
-          publicLogsInBlock[indexOfTx].push(reader.readObject(PublicLog));
+          publicLogsInBlock[indexOfTx].logs.push(reader.readObject(PublicLog));
         }
       }
       for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < publicLogsInBlock.length; txIndex++) {
-        const txLogs = publicLogsInBlock[txIndex];
-        maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+        const txData = publicLogsInBlock[txIndex];
+        maxLogsHit = this.#accumulatePublicLogs(
+          logs,
+          blockNumber,
+          blockHash,
+          txIndex,
+          txData.txHash,
+          txData.logs,
+          filter,
+        );
         if (maxLogsHit) {
           this.#log.debug(`Max logs hit at block ${blockNumber}`);
           break loopOverBlocks;
@@ -452,24 +496,33 @@ export class LogStore {
       return { logs: [], maxLogsHit: false };
     }
     const contractClassLogsBuffer = (await this.#contractClassLogsByBlock.getAsync(blockNumber)) ?? Buffer.alloc(0);
-    const contractClassLogsInBlock: [ContractClassLog[]] = [[]];
+    const contractClassLogsInBlock: { txHash: TxHash; logs: ContractClassLog[] }[] = [];
 
     const reader = new BufferReader(contractClassLogsBuffer);
     const blockHash = this.#unpackBlockHash(reader);
 
     while (reader.remainingBytes() > 0) {
       const indexOfTx = reader.readNumber();
+      const txHash = reader.readObject(TxHash);
       const numLogsInTx = reader.readNumber();
-      contractClassLogsInBlock[indexOfTx] = [];
+      contractClassLogsInBlock[indexOfTx] = { txHash, logs: [] };
       for (let i = 0; i < numLogsInTx; i++) {
-        contractClassLogsInBlock[indexOfTx].push(reader.readObject(ContractClassLog));
+        contractClassLogsInBlock[indexOfTx].logs.push(reader.readObject(ContractClassLog));
       }
     }
 
-    const txLogs = contractClassLogsInBlock[txIndex];
+    const txData = contractClassLogsInBlock[txIndex];
 
     const logs: ExtendedContractClassLog[] = [];
-    const maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+    const maxLogsHit = this.#accumulateContractClassLogs(
+      logs,
+      blockNumber,
+      blockHash,
+      txIndex,
+      txData.txHash,
+      txData.logs,
+      filter,
+    );
 
     return { logs, maxLogsHit };
   }
@@ -493,20 +546,29 @@ export class LogStore {
       start,
       end,
     })) {
-      const contractClassLogsInBlock: [ContractClassLog[]] = [[]];
+      const contractClassLogsInBlock: { txHash: TxHash; logs: ContractClassLog[] }[] = [];
       const reader = new BufferReader(logBuffer);
       const blockHash = this.#unpackBlockHash(reader);
       while (reader.remainingBytes() > 0) {
         const indexOfTx = reader.readNumber();
+        const txHash = reader.readObject(TxHash);
         const numLogsInTx = reader.readNumber();
-        contractClassLogsInBlock[indexOfTx] = [];
+        contractClassLogsInBlock[indexOfTx] = { txHash, logs: [] };
         for (let i = 0; i < numLogsInTx; i++) {
-          contractClassLogsInBlock[indexOfTx].push(reader.readObject(ContractClassLog));
+          contractClassLogsInBlock[indexOfTx].logs.push(reader.readObject(ContractClassLog));
         }
       }
       for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < contractClassLogsInBlock.length; txIndex++) {
-        const txLogs = contractClassLogsInBlock[txIndex];
-        maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+        const txData = contractClassLogsInBlock[txIndex];
+        maxLogsHit = this.#accumulateContractClassLogs(
+          logs,
+          blockNumber,
+          blockHash,
+          txIndex,
+          txData.txHash,
+          txData.logs,
+          filter,
+        );
         if (maxLogsHit) {
           this.#log.debug(`Max logs hit at block ${blockNumber}`);
           break loopOverBlocks;
@@ -517,12 +579,13 @@ export class LogStore {
     return { logs, maxLogsHit };
   }
 
-  #accumulateLogs(
-    results: (ExtendedContractClassLog | ExtendedPublicLog)[],
+  #accumulatePublicLogs(
+    results: ExtendedPublicLog[],
     blockNumber: number,
-    blockHash: L2BlockHash,
+    blockHash: BlockHash,
     txIndex: number,
-    txLogs: (ContractClassLog | PublicLog)[],
+    txHash: TxHash,
+    txLogs: PublicLog[],
     filter: LogFilter = {},
   ): boolean {
     let maxLogsHit = false;
@@ -530,15 +593,37 @@ export class LogStore {
     for (; logIndex < txLogs.length; logIndex++) {
       const log = txLogs[logIndex];
       if (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) {
-        if (log instanceof ContractClassLog) {
-          results.push(
-            new ExtendedContractClassLog(new LogId(BlockNumber(blockNumber), blockHash, txIndex, logIndex), log),
-          );
-        } else if (log instanceof PublicLog) {
-          results.push(new ExtendedPublicLog(new LogId(BlockNumber(blockNumber), blockHash, txIndex, logIndex), log));
-        } else {
-          throw new Error('Unknown log type');
+        results.push(
+          new ExtendedPublicLog(new LogId(BlockNumber(blockNumber), blockHash, txHash, txIndex, logIndex), log),
+        );
+
+        if (results.length >= this.#logsMaxPageSize) {
+          maxLogsHit = true;
+          break;
         }
+      }
+    }
+
+    return maxLogsHit;
+  }
+
+  #accumulateContractClassLogs(
+    results: ExtendedContractClassLog[],
+    blockNumber: number,
+    blockHash: BlockHash,
+    txIndex: number,
+    txHash: TxHash,
+    txLogs: ContractClassLog[],
+    filter: LogFilter = {},
+  ): boolean {
+    let maxLogsHit = false;
+    let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0;
+    for (; logIndex < txLogs.length; logIndex++) {
+      const log = txLogs[logIndex];
+      if (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) {
+        results.push(
+          new ExtendedContractClassLog(new LogId(BlockNumber(blockNumber), blockHash, txHash, txIndex, logIndex), log),
+        );
 
         if (results.length >= this.#logsMaxPageSize) {
           maxLogsHit = true;

package/src/test/fake_l1_state.ts

@@ -10,7 +10,7 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import { RollupAbi } from '@aztec/l1-artifacts';
-import { CommitteeAttestation, CommitteeAttestationsAndSigners, L2BlockNew } from '@aztec/stdlib/block';
+import { CommitteeAttestation, CommitteeAttestationsAndSigners, L2Block } from '@aztec/stdlib/block';
 import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
 import { InboxLeaf } from '@aztec/stdlib/messaging';
@@ -51,7 +51,7 @@ type AddCheckpointOptions = {
   /** Number of L2 blocks in the checkpoint. Default: 1 */
   numBlocks?: number;
   /** Or the actual blocks for the checkpoint */
-  blocks?: L2BlockNew[];
+  blocks?: L2Block[];
   /** Number of transactions per block. Default: 4 */
   txsPerBlock?: number;
   /** Max number of effects per tx (for generating large blobs). Default: undefined */

package/src/test/index.ts

@@ -2,3 +2,6 @@ export * from './mock_structs.js';
 export * from './mock_l2_block_source.js';
 export * from './mock_l1_to_l2_message_source.js';
 export * from './mock_archiver.js';
+// NOTE: noop_l1_archiver.js is intentionally NOT exported here because it imports
+// jest-mock-extended, which depends on @jest/globals and can only run inside Jest.
+// Import it directly: import { NoopL1Archiver } from '@aztec/archiver/test/noop-l1';

package/src/test/mock_archiver.ts

@@ -57,7 +57,7 @@ export class MockPrefilledArchiver extends MockArchiver {
 
     const fromBlock = this.l2Blocks.length;
     // TODO: Add L2 blocks and checkpoints separately once archiver has the apis for that.
-    this.addBlocks(this.prefilled.slice(fromBlock, fromBlock + numBlocks).flatMap(c => c.blocks));
+    this.addProposedBlocks(this.prefilled.slice(fromBlock, fromBlock + numBlocks).flatMap(c => c.blocks));
     return Promise.resolve();
   }
 }