@aztec/archiver 0.0.1-commit.b33fc05d0 → 0.0.1-commit.b3d3157a

package/src/store/log_store_codec.ts

@@ -0,0 +1,132 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { BlockHash } from '@aztec/stdlib/block';
+import { TxHash } from '@aztec/stdlib/tx';
+
+export const NUMERIC_HEX_LEN = 8;
+export const SEP = '-';
+
+/**
+ * Sentinel appended after a numeric hex segment to build an end bound strictly greater than any
+ * real key for that namespace. `'g'` sorts lexicographically after every hex digit (`0`-`9`, `a`-`f`),
+ * so `prefix + '-g'` is a clean exclusive upper bound.
+ */
+export const HEX_SENTINEL = 'g';
+
+export type ParsedKeyTail = {
+  blockNumber: BlockNumber;
+  txIndexWithinBlock: number;
+  logIndexWithinTx: number;
+};
+
+/**
+ * Per-kind stored value layout (no msgpackr):
+ *   txHash(32) ++ blockHash(32) ++ blockTimestamp(u64 BE = 8) ++ logDataLen(u32 BE = 4) ++ logData[i].toBuffer()...
+ * `blockNumber`, `txIndexWithinBlock`, and `logIndexWithinTx` are decoded from the composite key.
+ */
+export type StoredLogValue = {
+  txHash: TxHash;
+  blockHash: BlockHash;
+  blockTimestamp: bigint;
+  logData: Fr[];
+};
+
+/** Returns the 64-char lowercase hex representation of a field, stripping the `0x` prefix. */
+export function fieldHex(value: Fr | { toString: () => string }): string {
+  // Fr.toString() and AztecAddress.toString() both return `0x` + 64 lowercase hex chars.
+  return value.toString().slice(2);
+}
+
+/** Encodes a number as 8-char zero-padded lowercase hex (matches a u32 big-endian byte buffer's lex order). */
+export function u32Hex(n: number): string {
+  return n.toString(16).padStart(NUMERIC_HEX_LEN, '0');
+}
+
+/**
+ * Encodes the composite primary key as `prefix-block-txIdx-logIdx` where `prefix` is the leading
+ * segment (`tag` for private; `contract-tag` for public) and the trailing triple is fixed-width
+ * 8-char zero-padded hex so byte-order matches `(blockNumber, txIndexWithinBlock, logIndexWithinTx)`.
+ */
+export function encodeKey(prefix: string, blockNumber: number, txIndex: number, logIndex: number): string {
+  return `${prefix}${SEP}${u32Hex(blockNumber)}${SEP}${u32Hex(txIndex)}${SEP}${u32Hex(logIndex)}`;
+}
+
+/**
+ * Decodes the trailing `(blockNumber, txIndexWithinBlock, logIndexWithinTx)` triple from a composite
+ * key. The leading prefix segments are ignored — we only ever read them off the input query, never
+ * back off the key.
+ */
+export function decodeKeyTail(key: string): ParsedKeyTail {
+  const parts = key.split(SEP);
+  const len = parts.length;
+  return {
+    blockNumber: BlockNumber(parseInt(parts[len - 3], 16)),
+    txIndexWithinBlock: parseInt(parts[len - 2], 16),
+    logIndexWithinTx: parseInt(parts[len - 1], 16),
+  };
+}
+
+/**
+ * Exclusive end bound for a `(contract, tag)`-prefix scan. With an `upperBlockExclusive` we cut at
+ * `(prefix, upper, 0, 0)`. With no bound we use `prefix + '-' + HEX_SENTINEL`, which sorts strictly
+ * after every real key under `prefix` (`g` is greater than any hex digit).
+ */
+export function endOfTagRange(prefix: string, upperBlockExclusive: number | undefined): string {
+  if (upperBlockExclusive === undefined) {
+    return `${prefix}${SEP}${HEX_SENTINEL}`;
+  }
+  return encodeKey(prefix, upperBlockExclusive, 0, 0);
+}
+
+/**
+ * Exclusive end bound for a tx-strict scan: every key strictly inside `(prefix, txBlk, txIdx, *)`.
+ * `prefix-block-txIdx-` followed by the hex sentinel is the first key past every real logIndex for
+ * this tx and strictly less than the next tx's first key.
+ */
+export function endOfTxRange(prefix: string, txBlk: number, txIdx: number): string {
+  return `${prefix}${SEP}${u32Hex(txBlk)}${SEP}${u32Hex(txIdx)}${SEP}${HEX_SENTINEL}`;
+}
+
+/**
+ * Returns the smallest string strictly greater than a fully-encoded composite key. The encoded key
+ * ends in a hex digit, and `'g'` sorts strictly after any hex digit, so appending `'g'` is the
+ * smallest possible successor in our key alphabet. Used to turn an inclusive cursor into an
+ * exclusive `start`.
+ */
+export function incKey(key: string): string {
+  return key + HEX_SENTINEL;
+}
+
+export function encodeValue(value: StoredLogValue): Buffer {
+  const head = Buffer.allocUnsafe(32 + 32 + 8 + 4);
+  value.txHash.toBuffer().copy(head, 0);
+  value.blockHash.toBuffer().copy(head, 32);
+  head.writeBigUInt64BE(value.blockTimestamp, 64);
+  head.writeUInt32BE(value.logData.length, 72);
+  const fieldBufs = value.logData.map(f => f.toBuffer());
+  return Buffer.concat([head, ...fieldBufs]);
+}
+
+export function decodeValue(buffer: Buffer | Uint8Array): StoredLogValue {
+  const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  let off = 0;
+  const txHash = TxHash.fromBuffer(buf.subarray(off, off + 32));
+  off += 32;
+  const blockHash = BlockHash.fromBuffer(buf.subarray(off, off + 32));
+  off += 32;
+  const blockTimestamp = buf.readBigUInt64BE(off);
+  off += 8;
+  const logDataLen = buf.readUInt32BE(off);
+  off += 4;
+  const logData: Fr[] = new Array(logDataLen);
+  for (let i = 0; i < logDataLen; i++) {
+    logData[i] = Fr.fromBuffer(buf.subarray(off, off + 32));
+    off += 32;
+  }
+  return { txHash, blockHash, blockTimestamp, logData };
+}
+
+/** Encodes the public-log key prefix as `contractHex-tagHex`. */
+export function encodePublicPrefix(contractHex: string, tagHex: string): string {
+  return `${contractHex}${SEP}${tagHex}`;
+}

package/src/store/message_store.ts

@@ -14,6 +14,7 @@ import {
 } from '@aztec/kv-store';
 import { InboxLeaf } from '@aztec/stdlib/messaging';
 
+import { L1ToL2MessagesNotReadyError } from '../errors.js';
 import {
   type InboxMessage,
   deserializeInboxMessage,
@@ -40,6 +41,10 @@ export class MessageStore {
   #lastSynchedL1Block: AztecAsyncSingleton<Buffer>;
   /** Stores total messages stored */
   #totalMessageCount: AztecAsyncSingleton<bigint>;
+  /** Stores the checkpoint number whose message tree is currently being filled on L1. */
+  #inboxTreeInProgress: AztecAsyncSingleton<bigint>;
+  /** Stores the L1 finalized block as of the last successful message sync. */
+  #messagesFinalizedL1Block: AztecAsyncSingleton<Buffer>;
 
   #log = createLogger('archiver:message_store');
 
@@ -48,6 +53,8 @@ export class MessageStore {
     this.#l1ToL2MessageIndices = db.openMap('archiver_l1_to_l2_message_indices');
     this.#lastSynchedL1Block = db.openSingleton('archiver_last_l1_block_id');
     this.#totalMessageCount = db.openSingleton('archiver_l1_to_l2_message_count');
+    this.#inboxTreeInProgress = db.openSingleton('archiver_inbox_tree_in_progress');
+    this.#messagesFinalizedL1Block = db.openSingleton('archiver_messages_finalized_l1_block');
   }
 
   public async getTotalL1ToL2MessageCount(): Promise<bigint> {
@@ -71,6 +78,26 @@ export class MessageStore {
     await this.#lastSynchedL1Block.set(buffer);
   }
 
+  /** Gets the L1 finalized block as of the last successful message sync. */
+  public async getMessagesFinalizedL1Block(): Promise<L1BlockId | undefined> {
+    const buffer = await this.#messagesFinalizedL1Block.getAsync();
+    if (!buffer) {
+      return undefined;
+    }
+    const reader = BufferReader.asReader(buffer);
+    return { l1BlockNumber: reader.readUInt256(), l1BlockHash: Buffer32.fromBuffer(reader.readBytes(Buffer32.SIZE)) };
+  }
+
+  /** Monotonically advances the persisted L1 finalized block for message sync. Never regresses. */
+  private async maybeAdvanceFinalizedL1Block(l1Block: L1BlockId): Promise<void> {
+    const existing = await this.getMessagesFinalizedL1Block();
+    if (existing && l1Block.l1BlockNumber <= existing.l1BlockNumber) {
+      return;
+    }
+    const buffer = serializeToBuffer([l1Block.l1BlockNumber, l1Block.l1BlockHash]);
+    await this.#messagesFinalizedL1Block.set(buffer);
+  }
+
   /**
    * Append L1 to L2 messages to the store.
    * Requires new messages to be in order and strictly after the last message added.
@@ -157,15 +184,6 @@ export class MessageStore {
         lastMessage = message;
       }
 
-      // Update the L1 sync point to that of the last message added.
-      const currentSyncPoint = await this.getSynchedL1Block();
-      if (!currentSyncPoint || currentSyncPoint.l1BlockNumber < lastMessage!.l1BlockNumber) {
-        await this.setSynchedL1Block({
-          l1BlockNumber: lastMessage!.l1BlockNumber,
-          l1BlockHash: lastMessage!.l1BlockHash,
-        });
-      }
-
       // Update total message count with the number of inserted messages.
       await this.increaseTotalMessageCount(messageCount);
     });
@@ -185,7 +203,39 @@ export class MessageStore {
     return msg ? deserializeInboxMessage(msg) : undefined;
   }
 
+  /** Returns the inbox tree-in-progress checkpoint number from L1, or undefined if not yet set. */
+  public getInboxTreeInProgress(): Promise<bigint | undefined> {
+    return this.#inboxTreeInProgress.getAsync();
+  }
+
+  /**
+   * Atomically updates the message sync state: the L1 sync point, the inbox tree-in-progress marker, and
+   * (optionally) the L1 finalized block as of this sync. The finalized block is advanced monotonically.
+   */
+  public setMessageSyncState(
+    l1Block: L1BlockId,
+    treeInProgress: bigint | undefined,
+    finalizedL1Block?: L1BlockId,
+  ): Promise<void> {
+    return this.db.transactionAsync(async () => {
+      await this.setSynchedL1Block(l1Block);
+      if (treeInProgress !== undefined) {
+        await this.#inboxTreeInProgress.set(treeInProgress);
+      } else {
+        await this.#inboxTreeInProgress.delete();
+      }
+      if (finalizedL1Block !== undefined) {
+        await this.maybeAdvanceFinalizedL1Block(finalizedL1Block);
+      }
+    });
+  }
+
   public async getL1ToL2Messages(checkpointNumber: CheckpointNumber): Promise<Fr[]> {
+    const treeInProgress = await this.#inboxTreeInProgress.getAsync();
+    if (treeInProgress !== undefined && BigInt(checkpointNumber) >= treeInProgress) {
+      throw new L1ToL2MessagesNotReadyError(checkpointNumber, treeInProgress);
+    }
+
     const messages: Fr[] = [];
 
     const [startIndex, endIndex] = InboxLeaf.indexRangeForCheckpoint(checkpointNumber);

package/src/structs/inbox_message.ts

@@ -8,7 +8,7 @@ export type InboxMessage = {
   index: bigint;
   leaf: Fr;
   checkpointNumber: CheckpointNumber;
-  l1BlockNumber: bigint; // L1 block number - NOT Aztec L2
+  l1BlockNumber: bigint;
   l1BlockHash: Buffer32;
   rollingHash: Buffer16;
 };

package/src/test/fake_l1_state.ts

@@ -1,5 +1,6 @@
 import type { BlobClientInterface } from '@aztec/blob-client/client';
 import { type Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
+import { INITIAL_CHECKPOINT_NUMBER } from '@aztec/constants';
 import type { CheckpointProposedLog, InboxContract, MessageSentLog, RollupContract } from '@aztec/ethereum/contracts';
 import { MULTI_CALL_3_ADDRESS } from '@aztec/ethereum/contracts';
 import type { ViemPublicClient } from '@aztec/ethereum/types';
@@ -14,12 +15,8 @@ import { CommitteeAttestation, CommitteeAttestationsAndSigners, L2Block } from '
 import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
 import { InboxLeaf } from '@aztec/stdlib/messaging';
-import { ConsensusPayload, SignatureDomainSeparator } from '@aztec/stdlib/p2p';
-import {
-  makeAndSignCommitteeAttestationsAndSigners,
-  makeCheckpointAttestationFromCheckpoint,
-  mockCheckpointAndMessages,
-} from '@aztec/stdlib/testing';
+import { ConsensusPayload, getHashedSignaturePayloadTypedData } from '@aztec/stdlib/p2p';
+import { mockCheckpointAndMessages } from '@aztec/stdlib/testing';
 import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
 
 import { type MockProxy, mock } from 'jest-mock-extended';
@@ -150,8 +147,10 @@ export class FakeL1State {
   // Computed from checkpoints based on L1 block visibility
   private pendingCheckpointNumber: CheckpointNumber = CheckpointNumber(0);
 
-  // The L1 block number reported as "finalized" (defaults to the start block)
-  private finalizedL1BlockNumber: bigint;
+  // The L1 block number reported as "finalized" (defaults to the start block).
+  // `undefined` simulates the startup window on a fresh devnet where
+  // `getBlock({ blockTag: 'finalized' })` fails with "finalized block not found".
+  private finalizedL1BlockNumber: bigint | undefined;
 
   constructor(private readonly config: FakeL1StateConfig) {
     this.l1BlockNumber = config.l1StartBlock;
@@ -287,8 +286,11 @@ export class FakeL1State {
     this.updatePendingCheckpointNumber();
   }
 
-  /** Sets the L1 block number that will be reported as "finalized". */
-  setFinalizedL1BlockNumber(blockNumber: bigint): void {
+  /**
+   * Sets the L1 block number that will be reported as "finalized". Pass `undefined` to
+   * simulate a chain that does not yet have a finalized block (devnet startup).
+   */
+  setFinalizedL1BlockNumber(blockNumber: bigint | undefined): void {
     this.finalizedL1BlockNumber = blockNumber;
   }
 
@@ -331,6 +333,21 @@ export class FakeL1State {
     this.updatePendingCheckpointNumber();
   }
 
+  /**
+   * Moves a checkpoint to a different L1 block number (simulates L1 reorg that
+   * re-includes the same checkpoint transaction in a different block).
+   * The checkpoint content stays the same — only the L1 metadata changes.
+   * Auto-updates pending status.
+   */
+  moveCheckpointToL1Block(checkpointNumber: CheckpointNumber, newL1BlockNumber: bigint): void {
+    for (const cpData of this.checkpoints) {
+      if (cpData.checkpointNumber === checkpointNumber) {
+        cpData.l1BlockNumber = newL1BlockNumber;
+      }
+    }
+    this.updatePendingCheckpointNumber();
+  }
+
   /**
    * Removes messages after a given total index (simulates L1 reorg).
    * Auto-updates rolling hash.
@@ -450,21 +467,44 @@ export class FakeL1State {
   createMockInboxContract(_publicClient: MockProxy<ViemPublicClient>): MockProxy<InboxContract> {
     const mockInbox = mock<InboxContract>();
 
-    mockInbox.getState.mockImplementation(() =>
-      Promise.resolve({
-        messagesRollingHash: this.messagesRollingHash,
-        totalMessagesInserted: BigInt(this.messages.length),
-        treeInProgress: 0n,
-      }),
-    );
+    mockInbox.getState.mockImplementation((opts: { blockTag?: string; blockNumber?: bigint } = {}) => {
+      // Filter messages visible at the given block number (or all if not specified)
+      const blockNumber = opts.blockNumber ?? this.l1BlockNumber;
+      const visibleMessages = this.messages.filter(m => m.l1BlockNumber <= blockNumber);
+
+      // treeInProgress must be > any sealed checkpoint. On L1, a checkpoint can only be proposed
+      // after its messages are sealed, so treeInProgress > checkpointNumber for all published checkpoints.
+      const maxFromMessages =
+        visibleMessages.length > 0 ? Math.max(...visibleMessages.map(m => Number(m.checkpointNumber))) + 1 : 0;
+      const maxFromCheckpoints =
+        this.checkpoints.length > 0
+          ? Math.max(
+              ...this.checkpoints
+                .filter(cp => !cp.pruned && cp.l1BlockNumber <= blockNumber)
+                .map(cp => Number(cp.checkpointNumber)),
+              0,
+            ) + 1
+          : 0;
+      const treeInProgress = Math.max(maxFromMessages, maxFromCheckpoints, INITIAL_CHECKPOINT_NUMBER);
+
+      // Compute rolling hash only for visible messages
+      const rollingHash =
+        visibleMessages.length > 0 ? visibleMessages[visibleMessages.length - 1].rollingHash : Buffer16.ZERO;
+
+      return Promise.resolve({
+        messagesRollingHash: rollingHash,
+        totalMessagesInserted: BigInt(visibleMessages.length),
+        treeInProgress: BigInt(treeInProgress),
+      });
+    });
 
     // Mock the wrapper methods for fetching message events
     mockInbox.getMessageSentEvents.mockImplementation((fromBlock: bigint, toBlock: bigint) =>
       Promise.resolve(this.getMessageSentLogs(fromBlock, toBlock)),
     );
 
-    mockInbox.getMessageSentEventByHash.mockImplementation((hash: string, fromBlock: bigint, toBlock: bigint) =>
-      Promise.resolve(this.getMessageSentLogs(fromBlock, toBlock, hash)),
+    mockInbox.getMessageSentEventByHash.mockImplementation((msgHash: string, aroundL1BlockNumber: bigint) =>
+      Promise.resolve(this.getMessageSentLogByHash(msgHash, aroundL1BlockNumber) as MessageSentLog),
     );
 
     return mockInbox;
@@ -475,11 +515,19 @@ export class FakeL1State {
     const publicClient = mock<ViemPublicClient>();
 
     publicClient.getChainId.mockResolvedValue(1);
+    // Several consumers (CalldataRetriever, ArchiverL1Synchronizer) derive the EIP-712 signing
+    // context from `publicClient.chain.id`. Pin it so it matches `getSignatureContext()` below.
+    (publicClient as unknown as { chain: { id: number } }).chain = { id: 1 };
     publicClient.getBlockNumber.mockImplementation(() => Promise.resolve(this.l1BlockNumber));
 
     publicClient.getBlock.mockImplementation((async (args: { blockNumber?: bigint; blockTag?: string } = {}) => {
       let blockNum: bigint;
       if (args.blockTag === 'finalized') {
+        if (this.finalizedL1BlockNumber === undefined) {
+          throw Object.assign(new Error('finalized block not found'), {
+            details: 'finalized block not found',
+          });
+        }
         blockNum = this.finalizedL1BlockNumber;
       } else {
         blockNum = args.blockNumber ?? (await publicClient.getBlockNumber());
@@ -505,10 +553,10 @@ export class FakeL1State {
   createMockBlobClient(): MockProxy<BlobClientInterface> {
     const blobClient = mock<BlobClientInterface>();
 
-    // The blockId is the transaction's blockHash, which we set to the checkpoint's archive root
+    // The blockId is the L1 block hash, which we derive from the L1 block number
     blobClient.getBlobSidecar.mockImplementation((blockId: `0x${string}`) =>
       Promise.resolve(
-        this.checkpoints.find(cpData => cpData.checkpoint.archive.root.toString() === blockId)?.blobs ?? [],
+        this.checkpoints.find(cpData => Buffer32.fromBigInt(cpData.l1BlockNumber).toString() === blockId)?.blobs ?? [],
       ),
     );
 
@@ -584,13 +632,38 @@ export class FakeL1State {
       }));
   }
 
+  private getMessageSentLogByHash(msgHash: string, aroundL1BlockNumber: bigint): MessageSentLog | undefined {
+    const msg = this.messages.find(
+      msg =>
+        msg.leaf.toString() === msgHash &&
+        msg.l1BlockNumber >= aroundL1BlockNumber - 5n &&
+        msg.l1BlockNumber <= aroundL1BlockNumber + 5n,
+    );
+    if (!msg) {
+      return undefined;
+    }
+    return {
+      l1BlockNumber: msg.l1BlockNumber,
+      l1BlockHash: Buffer32.fromBigInt(msg.l1BlockNumber),
+      l1TransactionHash: `0x${msg.l1BlockNumber.toString(16)}` as `0x${string}`,
+      args: {
+        checkpointNumber: msg.checkpointNumber,
+        index: msg.index,
+        leaf: msg.leaf,
+        rollingHash: msg.rollingHash,
+      },
+    };
+  }
+
   private async makeRollupTx(
     checkpoint: Checkpoint,
     signers: Secp256k1Signer[],
   ): Promise<{ tx: Transaction; attestationsHash: Buffer32; payloadDigest: Buffer32 }> {
+    const signatureContext = this.getSignatureContext();
+    const consensusPayload = ConsensusPayload.fromCheckpoint(checkpoint, signatureContext);
+    const attestationDigest = getHashedSignaturePayloadTypedData(consensusPayload);
     const attestations = signers
-      .map(signer => makeCheckpointAttestationFromCheckpoint(checkpoint, signer))
-      .map(attestation => CommitteeAttestation.fromSignature(attestation.signature))
+      .map(signer => CommitteeAttestation.fromSignature(signer.sign(attestationDigest)))
       .map(committeeAttestation => committeeAttestation.toViem());
 
     const header = checkpoint.header.toViem();
@@ -598,11 +671,15 @@ export class FakeL1State {
     const archive = toHex(checkpoint.archive.root.toBuffer());
     const attestationsAndSigners = new CommitteeAttestationsAndSigners(
       attestations.map(attestation => CommitteeAttestation.fromViem(attestation)),
+      signatureContext,
     );
 
-    const attestationsAndSignersSignature = makeAndSignCommitteeAttestationsAndSigners(
-      attestationsAndSigners,
-      signers[0],
+    // Fall back to a random signer when no attesters are provided, so tests that
+    // don't care about the proposer identity (e.g. sync tests) still produce a
+    // valid-looking signature for the attestationsAndSigners struct.
+    const proposerSigner = signers[0] ?? Secp256k1Signer.random();
+    const attestationsAndSignersSignature = proposerSigner.sign(
+      getHashedSignaturePayloadTypedData(attestationsAndSigners),
     );
 
     const packedAttestations = attestationsAndSigners.getPackedAttestations();
@@ -643,9 +720,7 @@ export class FakeL1State {
     );
 
     // Compute payloadDigest (same logic as CalldataRetriever)
-    const consensusPayload = ConsensusPayload.fromCheckpoint(checkpoint);
-    const payloadToSign = consensusPayload.getPayloadToSign(SignatureDomainSeparator.checkpointAttestation);
-    const payloadDigest = Buffer32.fromString(keccak256(payloadToSign));
+    const payloadDigest = getHashedSignaturePayloadTypedData(consensusPayload);
 
     const tx = {
       input: multiCallInput,
@@ -657,6 +732,13 @@ export class FakeL1State {
     return { tx, attestationsHash, payloadDigest };
   }
 
+  private getSignatureContext() {
+    return {
+      chainId: 1,
+      rollupAddress: this.config.rollupAddress,
+    };
+  }
+
   /** Extracts the CommitteeAttestations struct definition from RollupAbi for hash computation. */
   private getCommitteeAttestationsStructDef(): AbiParameter {
     const proposeFunction = RollupAbi.find(item => item.type === 'function' && item.name === 'propose') as

package/src/test/mock_l1_to_l2_message_source.ts

@@ -44,6 +44,7 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource {
       checkpointed: tip,
       proven: tip,
       finalized: tip,
+      proposedCheckpoint: tip,
     });
   }
 }