@aztec/archiver 0.65.2 → 0.67.0

package/src/archiver/archiver_store.ts

@@ -1,23 +1,21 @@
 import {
-  type FromLogType,
   type GetUnencryptedLogsResponse,
   type InBlock,
   type InboxLeaf,
   type L2Block,
-  type L2BlockL2Logs,
   type LogFilter,
-  type LogType,
   type TxEffect,
   type TxHash,
   type TxReceipt,
   type TxScopedL2Log,
 } from '@aztec/circuit-types';
 import {
+  type BlockHeader,
   type ContractClassPublic,
   type ContractInstanceWithAddress,
   type ExecutablePrivateFunctionWithMembershipProof,
   type Fr,
-  type Header,
+  type PrivateLog,
   type UnconstrainedFunctionWithMembershipProof,
 } from '@aztec/circuits.js';
 import { type ContractArtifact, type FunctionSelector } from '@aztec/foundation/abi';
@@ -73,7 +71,7 @@ export interface ArchiverDataStore {
    * @param limit - The number of blocks to return.
    * @returns The requested L2 block headers.
    */
-  getBlockHeaders(from: number, limit: number): Promise<Header[]>;
+  getBlockHeaders(from: number, limit: number): Promise<BlockHeader[]>;
 
   /**
    * Gets a tx effect.
@@ -142,17 +140,12 @@ export interface ArchiverDataStore {
   getTotalL1ToL2MessageCount(): Promise<bigint>;
 
   /**
-   * Gets up to `limit` amount of logs starting from `from`.
-   * @param from - Number of the L2 block to which corresponds the first logs to be returned.
-   * @param limit - The number of logs to return.
-   * @param logType - Specifies whether to return encrypted or unencrypted logs.
-   * @returns The requested logs.
+   * Retrieves all private logs from up to `limit` blocks, starting from the block number `from`.
+   * @param from - The block number from which to begin retrieving logs.
+   * @param limit - The maximum number of blocks to retrieve logs from.
+   * @returns An array of private logs from the specified range of blocks.
    */
-  getLogs<TLogType extends LogType>(
-    from: number,
-    limit: number,
-    logType: TLogType,
-  ): Promise<L2BlockL2Logs<FromLogType<TLogType>>[]>;
+  getPrivateLogs(from: number, limit: number): Promise<PrivateLog[]>;
 
   /**
    * Gets all logs that match any of the received tags (i.e. logs with their first field equal to a tag).

package/src/archiver/archiver_store_test_suite.ts

@@ -1,4 +1,14 @@
-import { InboxLeaf, L2Block, LogId, LogType, TxHash, wrapInBlock } from '@aztec/circuit-types';
+import {
+  InboxLeaf,
+  L2Block,
+  LogId,
+  TxEffect,
+  TxHash,
+  UnencryptedFunctionL2Logs,
+  UnencryptedL2Log,
+  UnencryptedTxL2Logs,
+  wrapInBlock,
+} from '@aztec/circuit-types';
 import '@aztec/circuit-types/jest';
 import {
   AztecAddress,
@@ -8,6 +18,8 @@ import {
   INITIAL_L2_BLOCK_NUM,
   L1_TO_L2_MSG_SUBTREE_HEIGHT,
   MAX_NULLIFIERS_PER_TX,
+  PRIVATE_LOG_SIZE_IN_FIELDS,
+  PrivateLog,
   SerializableContractInstance,
   computePublicBytecodeCommitment,
 } from '@aztec/circuits.js';
@@ -16,7 +28,6 @@ import {
   makeExecutablePrivateFunctionWithMembershipProof,
   makeUnconstrainedFunctionWithMembershipProof,
 } from '@aztec/circuits.js/testing';
-import { toBufferBE } from '@aztec/foundation/bigint-buffer';
 import { times } from '@aztec/foundation/collection';
 import { randomBytes, randomInt } from '@aztec/foundation/crypto';
 
@@ -155,55 +166,41 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch
     });
 
     describe('addLogs', () => {
-      it('adds encrypted & unencrypted logs', async () => {
+      it('adds private & unencrypted logs', async () => {
         const block = blocks[0].data;
         await expect(store.addLogs([block])).resolves.toEqual(true);
       });
     });
 
     describe('deleteLogs', () => {
-      it('deletes encrypted & unencrypted logs', async () => {
+      it('deletes private & unencrypted logs', async () => {
         const block = blocks[0].data;
         await store.addBlocks([blocks[0]]);
         await expect(store.addLogs([block])).resolves.toEqual(true);
 
-        expect((await store.getLogs(1, 1, LogType.NOTEENCRYPTED))[0]).toEqual(block.body.noteEncryptedLogs);
-        expect((await store.getLogs(1, 1, LogType.ENCRYPTED))[0]).toEqual(block.body.encryptedLogs);
-        expect((await store.getLogs(1, 1, LogType.UNENCRYPTED))[0]).toEqual(block.body.unencryptedLogs);
+        expect((await store.getPrivateLogs(1, 1)).length).toEqual(
+          block.body.txEffects.map(txEffect => txEffect.privateLogs).flat().length,
+        );
+        expect((await store.getUnencryptedLogs({ fromBlock: 1 })).logs.length).toEqual(
+          block.body.unencryptedLogs.getTotalLogCount(),
+        );
 
         // This one is a pain for memory as we would never want to just delete memory in the middle.
         await store.deleteLogs([block]);
 
-        expect((await store.getLogs(1, 1, LogType.NOTEENCRYPTED))[0]).toEqual(undefined);
-        expect((await store.getLogs(1, 1, LogType.ENCRYPTED))[0]).toEqual(undefined);
-        expect((await store.getLogs(1, 1, LogType.UNENCRYPTED))[0]).toEqual(undefined);
+        expect((await store.getPrivateLogs(1, 1)).length).toEqual(0);
+        expect((await store.getUnencryptedLogs({ fromBlock: 1 })).logs.length).toEqual(0);
       });
     });
 
-    describe.each([
-      ['note_encrypted', LogType.NOTEENCRYPTED],
-      ['encrypted', LogType.ENCRYPTED],
-      ['unencrypted', LogType.UNENCRYPTED],
-    ])('getLogs (%s)', (_, logType) => {
-      beforeEach(async () => {
-        await store.addBlocks(blocks);
-        await store.addLogs(blocks.map(b => b.data));
-      });
+    describe('getPrivateLogs', () => {
+      it('gets added private logs', async () => {
+        const block = blocks[0].data;
+        await store.addBlocks([blocks[0]]);
+        await store.addLogs([block]);
 
-      it.each(blockTests)('retrieves previously stored logs', async (from, limit, getExpectedBlocks) => {
-        const expectedLogs = getExpectedBlocks().map(block => {
-          switch (logType) {
-            case LogType.ENCRYPTED:
-              return block.data.body.encryptedLogs;
-            case LogType.NOTEENCRYPTED:
-              return block.data.body.noteEncryptedLogs;
-            case LogType.UNENCRYPTED:
-            default:
-              return block.data.body.unencryptedLogs;
-          }
-        });
-        const actualLogs = await store.getLogs(from, limit, logType);
-        expect(actualLogs[0].txLogs[0]).toEqual(expectedLogs[0].txLogs[0]);
+        const privateLogs = await store.getPrivateLogs(1, 1);
+        expect(privateLogs).toEqual(block.body.txEffects.map(txEffect => txEffect.privateLogs).flat());
       });
     });
 
@@ -373,178 +370,155 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch
     });
 
     describe('getLogsByTags', () => {
-      const txsPerBlock = 4;
-      const numPrivateFunctionCalls = 3;
-      const numPublicFunctionCalls = 1;
-      const numEncryptedLogsPerFn = 2;
-      const numUnencryptedLogsPerFn = 1;
-      const numBlocks = 10;
-      let blocks: L1Published<L2Block>[];
-      let encryptedLogTags: { [i: number]: { [j: number]: Buffer[] } } = {};
-      let unencryptedLogTags: { [i: number]: { [j: number]: Buffer[] } } = {};
-
-      beforeEach(async () => {
-        blocks = times(numBlocks, (index: number) => ({
-          data: L2Block.random(
-            index + 1,
-            txsPerBlock,
-            numPrivateFunctionCalls,
-            numPublicFunctionCalls,
-            numEncryptedLogsPerFn,
-            numUnencryptedLogsPerFn,
-          ),
-          l1: { blockNumber: BigInt(index), blockHash: `0x${index}`, timestamp: BigInt(index) },
-        }));
-        // Last block has the note encrypted log tags of the first tx copied from the previous block
-        blocks[numBlocks - 1].data.body.noteEncryptedLogs.txLogs[0].functionLogs.forEach((fnLogs, fnIndex) => {
-          fnLogs.logs.forEach((log, logIndex) => {
-            const previousLogData =
-              blocks[numBlocks - 2].data.body.noteEncryptedLogs.txLogs[0].functionLogs[fnIndex].logs[logIndex].data;
-            previousLogData.copy(log.data, 0, 0, 32);
-          });
-        });
-        // Last block has invalid tags in the second tx
-        const tooBig = toBufferBE(Fr.MODULUS, 32);
-        blocks[numBlocks - 1].data.body.noteEncryptedLogs.txLogs[1].functionLogs.forEach(fnLogs => {
-          fnLogs.logs.forEach(log => {
-            tooBig.copy(log.data, 0, 0, 32);
-          });
-        });
+      const numBlocks = 3;
+      const numTxsPerBlock = 4;
+      const numPrivateLogsPerTx = 3;
+      const numUnencryptedLogsPerTx = 2;
 
-        await store.addBlocks(blocks);
-        await store.addLogs(blocks.map(b => b.data));
+      let blocks: L1Published<L2Block>[];
 
-        encryptedLogTags = {};
-        unencryptedLogTags = {};
-        blocks.forEach((b, blockIndex) => {
-          if (!encryptedLogTags[blockIndex]) {
-            encryptedLogTags[blockIndex] = {};
-          }
-          if (!unencryptedLogTags[blockIndex]) {
-            unencryptedLogTags[blockIndex] = {};
-          }
-          b.data.body.noteEncryptedLogs.txLogs.forEach((txLogs, txIndex) => {
-            if (!encryptedLogTags[blockIndex][txIndex]) {
-              encryptedLogTags[blockIndex][txIndex] = [];
-            }
-            encryptedLogTags[blockIndex][txIndex].push(...txLogs.unrollLogs().map(log => log.data.subarray(0, 32)));
-          });
-          b.data.body.unencryptedLogs.txLogs.forEach((txLogs, txIndex) => {
-            if (!unencryptedLogTags[blockIndex][txIndex]) {
-              unencryptedLogTags[blockIndex][txIndex] = [];
-            }
-            unencryptedLogTags[blockIndex][txIndex].push(...txLogs.unrollLogs().map(log => log.data.subarray(0, 32)));
-          });
-        });
-      });
+      const makeTag = (blockNumber: number, txIndex: number, logIndex: number, isPublic = false) =>
+        new Fr((blockNumber * 100 + txIndex * 10 + logIndex) * (isPublic ? 123 : 1));
 
-      it('is possible to batch request encrypted logs of a tx via tags', async () => {
-        // get random tx from any block that's not the last one
-        const targetBlockIndex = randomInt(numBlocks - 2);
-        const targetTxIndex = randomInt(txsPerBlock);
+      const makePrivateLog = (tag: Fr) =>
+        PrivateLog.fromFields([tag, ...times(PRIVATE_LOG_SIZE_IN_FIELDS - 1, i => new Fr(tag.toNumber() + i))]);
 
-        const logsByTags = await store.getLogsByTags(
-          encryptedLogTags[targetBlockIndex][targetTxIndex].map(buffer => new Fr(buffer)),
-        );
+      const makePublicLog = (tag: Fr) =>
+        Buffer.concat([tag.toBuffer(), ...times(tag.toNumber() % 60, i => new Fr(tag.toNumber() + i).toBuffer())]);
 
-        const expectedResponseSize = numPrivateFunctionCalls * numEncryptedLogsPerFn;
-        expect(logsByTags.length).toEqual(expectedResponseSize);
-
-        logsByTags.forEach((logsByTag, logIndex) => {
-          expect(logsByTag).toHaveLength(1);
-          const [scopedLog] = logsByTag;
-          expect(scopedLog.txHash).toEqual(blocks[targetBlockIndex].data.body.txEffects[targetTxIndex].txHash);
-          expect(scopedLog.logData).toEqual(
-            blocks[targetBlockIndex].data.body.noteEncryptedLogs.txLogs[targetTxIndex].unrollLogs()[logIndex].data,
-          );
+      const mockPrivateLogs = (blockNumber: number, txIndex: number) => {
+        return times(numPrivateLogsPerTx, (logIndex: number) => {
+          const tag = makeTag(blockNumber, txIndex, logIndex);
+          return makePrivateLog(tag);
         });
-      });
-
-      // TODO: Allow this test when #9835 is fixed and tags can be correctly decoded
-      it.skip('is possible to batch request all logs (encrypted and unencrypted) of a tx via tags', async () => {
-        // get random tx from any block that's not the last one
-        const targetBlockIndex = randomInt(numBlocks - 2);
-        const targetTxIndex = randomInt(txsPerBlock);
-
-        const logsByTags = await store.getLogsByTags(
-          encryptedLogTags[targetBlockIndex][targetTxIndex]
-            .concat(unencryptedLogTags[targetBlockIndex][targetTxIndex])
-            .map(buffer => new Fr(buffer)),
-        );
+      };
 
-        const expectedResponseSize =
-          numPrivateFunctionCalls * numEncryptedLogsPerFn + numPublicFunctionCalls * numUnencryptedLogsPerFn;
-        expect(logsByTags.length).toEqual(expectedResponseSize);
-
-        const encryptedLogsByTags = logsByTags.slice(0, numPrivateFunctionCalls * numEncryptedLogsPerFn);
-        const unencryptedLogsByTags = logsByTags.slice(numPrivateFunctionCalls * numEncryptedLogsPerFn);
-        encryptedLogsByTags.forEach((logsByTag, logIndex) => {
-          expect(logsByTag).toHaveLength(1);
-          const [scopedLog] = logsByTag;
-          expect(scopedLog.txHash).toEqual(blocks[targetBlockIndex].data.body.txEffects[targetTxIndex].txHash);
-          expect(scopedLog.logData).toEqual(
-            blocks[targetBlockIndex].data.body.noteEncryptedLogs.txLogs[targetTxIndex].unrollLogs()[logIndex].data,
-          );
+      const mockUnencryptedLogs = (blockNumber: number, txIndex: number) => {
+        const logs = times(numUnencryptedLogsPerTx, (logIndex: number) => {
+          const tag = makeTag(blockNumber, txIndex, logIndex, /* isPublic */ true);
+          const log = makePublicLog(tag);
+          return new UnencryptedL2Log(AztecAddress.fromNumber(txIndex), log);
         });
-        unencryptedLogsByTags.forEach((logsByTag, logIndex) => {
-          expect(logsByTag).toHaveLength(1);
-          const [scopedLog] = logsByTag;
-          expect(scopedLog.logData).toEqual(
-            blocks[targetBlockIndex].data.body.unencryptedLogs.txLogs[targetTxIndex].unrollLogs()[logIndex].data,
-          );
+        return new UnencryptedTxL2Logs([new UnencryptedFunctionL2Logs(logs)]);
+      };
+
+      const mockBlockWithLogs = (blockNumber: number): L1Published<L2Block> => {
+        const block = L2Block.random(blockNumber);
+        block.header.globalVariables.blockNumber = new Fr(blockNumber);
+
+        block.body.txEffects = times(numTxsPerBlock, (txIndex: number) => {
+          const txEffect = TxEffect.random();
+          txEffect.privateLogs = mockPrivateLogs(blockNumber, txIndex);
+          txEffect.unencryptedLogs = mockUnencryptedLogs(blockNumber, txIndex);
+          return txEffect;
         });
-      });
 
-      it('is possible to batch request logs of different blocks via tags', async () => {
-        // get first tx of first block and second tx of second block
-        const logsByTags = await store.getLogsByTags(
-          [...encryptedLogTags[0][0], ...encryptedLogTags[1][1]].map(buffer => new Fr(buffer)),
-        );
+        return {
+          data: block,
+          l1: { blockNumber: BigInt(blockNumber), blockHash: `0x${blockNumber}`, timestamp: BigInt(blockNumber) },
+        };
+      };
 
-        const expectedResponseSize = 2 * numPrivateFunctionCalls * numEncryptedLogsPerFn;
-        expect(logsByTags.length).toEqual(expectedResponseSize);
+      beforeEach(async () => {
+        blocks = times(numBlocks, (index: number) => mockBlockWithLogs(index));
 
-        logsByTags.forEach(logsByTag => expect(logsByTag).toHaveLength(1));
+        await store.addBlocks(blocks);
+        await store.addLogs(blocks.map(b => b.data));
       });
 
-      it('is possible to batch request logs that have the same tag but different content', async () => {
-        // get first tx of last block
-        const logsByTags = await store.getLogsByTags(encryptedLogTags[numBlocks - 1][0].map(buffer => new Fr(buffer)));
+      it('is possible to batch request private logs via tags', async () => {
+        const tags = [makeTag(1, 1, 2), makeTag(0, 2, 0)];
+
+        const logsByTags = await store.getLogsByTags(tags);
+
+        expect(logsByTags).toEqual([
+          [
+            expect.objectContaining({
+              blockNumber: 1,
+              logData: makePrivateLog(tags[0]).toBuffer(),
+              isFromPublic: false,
+            }),
+          ],
+          [
+            expect.objectContaining({
+              blockNumber: 0,
+              logData: makePrivateLog(tags[1]).toBuffer(),
+              isFromPublic: false,
+            }),
+          ],
+        ]);
+      });
 
-        const expectedResponseSize = numPrivateFunctionCalls * numEncryptedLogsPerFn;
-        expect(logsByTags.length).toEqual(expectedResponseSize);
+      // TODO: Allow this test when #9835 is fixed and tags can be correctly decoded
+      it.skip('is possible to batch request all logs (private and unencrypted) via tags', async () => {
+        // Tag(0, 0, 0) is shared with the first private log and the first unencrypted log.
+        const tags = [makeTag(0, 0, 0)];
+
+        const logsByTags = await store.getLogsByTags(tags);
+
+        expect(logsByTags).toEqual([
+          [
+            expect.objectContaining({
+              blockNumber: 0,
+              logData: makePrivateLog(tags[0]).toBuffer(),
+              isFromPublic: false,
+            }),
+            expect.objectContaining({
+              blockNumber: 0,
+              logData: makePublicLog(tags[0]),
+              isFromPublic: true,
+            }),
+          ],
+        ]);
+      });
 
-        logsByTags.forEach(logsByTag => {
-          expect(logsByTag).toHaveLength(2);
-          const [tag0, tag1] = logsByTag.map(scopedLog => new Fr(scopedLog.logData.subarray(0, 32)));
-          expect(tag0).toEqual(tag1);
-        });
+      it('is possible to batch request logs that have the same tag but different content', async () => {
+        const tags = [makeTag(1, 2, 1)];
+
+        // Create a block containing logs that have the same tag as the blocks before.
+        const newBlockNumber = numBlocks;
+        const newBlock = mockBlockWithLogs(newBlockNumber);
+        const newLog = newBlock.data.body.txEffects[1].privateLogs[1];
+        newLog.fields[0] = tags[0];
+        newBlock.data.body.txEffects[1].privateLogs[1] = newLog;
+        await store.addBlocks([newBlock]);
+        await store.addLogs([newBlock.data]);
+
+        const logsByTags = await store.getLogsByTags(tags);
+
+        expect(logsByTags).toEqual([
+          [
+            expect.objectContaining({
+              blockNumber: 1,
+              logData: makePrivateLog(tags[0]).toBuffer(),
+              isFromPublic: false,
+            }),
+            expect.objectContaining({
+              blockNumber: newBlockNumber,
+              logData: newLog.toBuffer(),
+              isFromPublic: false,
+            }),
+          ],
+        ]);
       });
 
       it('is possible to request logs for non-existing tags and determine their position', async () => {
-        // get random tx from any block that's not the last one
-        const targetBlockIndex = randomInt(numBlocks - 2);
-        const targetTxIndex = randomInt(txsPerBlock);
-
-        const logsByTags = await store.getLogsByTags([
-          Fr.random(),
-          ...encryptedLogTags[targetBlockIndex][targetTxIndex].slice(1).map(buffer => new Fr(buffer)),
+        const tags = [makeTag(99, 88, 77), makeTag(1, 1, 1)];
+
+        const logsByTags = await store.getLogsByTags(tags);
+
+        expect(logsByTags).toEqual([
+          [
+            // No logs for the first tag.
+          ],
+          [
+            expect.objectContaining({
+              blockNumber: 1,
+              logData: makePrivateLog(tags[1]).toBuffer(),
+              isFromPublic: false,
+            }),
+          ],
         ]);
-
-        const expectedResponseSize = numPrivateFunctionCalls * numEncryptedLogsPerFn;
-        expect(logsByTags.length).toEqual(expectedResponseSize);
-
-        const [emptyLogsByTag, ...populatedLogsByTags] = logsByTags;
-        expect(emptyLogsByTag).toHaveLength(0);
-
-        populatedLogsByTags.forEach((logsByTag, logIndex) => {
-          expect(logsByTag).toHaveLength(1);
-          const [scopedLog] = logsByTag;
-          expect(scopedLog.txHash).toEqual(blocks[targetBlockIndex].data.body.txEffects[targetTxIndex].txHash);
-          expect(scopedLog.logData).toEqual(
-            blocks[targetBlockIndex].data.body.noteEncryptedLogs.txLogs[targetTxIndex].unrollLogs()[logIndex + 1].data,
-          );
-        });
       });
     });
 
@@ -557,7 +531,7 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch
 
       beforeEach(async () => {
         blocks = times(numBlocks, (index: number) => ({
-          data: L2Block.random(index + 1, txsPerBlock, 2, numPublicFunctionCalls, 2, numUnencryptedLogs),
+          data: L2Block.random(index + 1, txsPerBlock, numPublicFunctionCalls, numUnencryptedLogs),
           l1: { blockNumber: BigInt(index), blockHash: `0x${index}`, timestamp: BigInt(index) },
         }));
 

package/src/archiver/config.ts

@@ -18,24 +18,19 @@ import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } fr
  * The archiver configuration.
  */
 export type ArchiverConfig = {
-  /**
-   * URL for an archiver service. If set, will return an archiver client as opposed to starting a new one.
-   */
+  /** URL for an archiver service. If set, will return an archiver client as opposed to starting a new one. */
   archiverUrl?: string;
 
-  /**
-   * The polling interval in ms for retrieving new L2 blocks and encrypted logs.
-   */
+  /** The polling interval in ms for retrieving new L2 blocks and encrypted logs. */
   archiverPollingIntervalMS?: number;
 
-  /**
-   * The polling interval viem uses in ms
-   */
+  /** The number of L2 blocks the archiver will attempt to download at a time. */
+  archiverBatchSize?: number;
+
+  /** The polling interval viem uses in ms */
   viemPollingIntervalMS?: number;
 
-  /**
-   * The deployed L1 contract addresses
-   */
+  /** The deployed L1 contract addresses */
   l1Contracts: L1ContractAddresses;
 
   /** The max number of logs that can be obtained in 1 "getUnencryptedLogs" call. */
@@ -54,6 +49,11 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
     description: 'The polling interval in ms for retrieving new L2 blocks and encrypted logs.',
     ...numberConfigHelper(1_000),
   },
+  archiverBatchSize: {
+    env: 'ARCHIVER_BATCH_SIZE',
+    description: 'The number of L2 blocks the archiver will attempt to download at a time.',
+    ...numberConfigHelper(100),
+  },
   maxLogs: {
     env: 'ARCHIVER_MAX_LOGS',
     description: 'The max number of logs that can be obtained in 1 "getUnencryptedLogs" call.',