@forestrie/merklelog 0.0.1

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@forestrie/merklelog",
+  "version": "0.0.1",
+  "license": "MIT",
+  "type": "module",
+  "description": "MMR (merkle mountain range) proofs and massif storage format for Forestrie transparency logs",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/forestrie/canopy.git",
+    "directory": "packages/merklelog"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {},
+  "devDependencies": {
+    "typescript": "^5.9.2",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest --run",
+    "test:watch": "vitest",
+    "test:debug": "vitest --inspect --no-file-parallelism"
+  }
+}

package/src/index.ts

@@ -0,0 +1,68 @@
+/**
+ * @forestrie/merklelog - TypeScript implementation of MMR Merklelog format
+ *
+ * This package provides TypeScript implementations of the MMR (Merkle Mountain Range)
+ * merklelog format as defined by the go-merklelog project.
+ *
+ * @packageDocumentation
+ */
+
+// Export uint64 module
+export { Uint64 } from "./uint64/index.js";
+
+// Export massifs module
+export { Massif } from "./massifs/massif.js";
+export type {
+  MassifStart,
+  LeafComponent,
+  LeafEnumeratorSpec,
+  LeafEntry,
+} from "./massifs/types.js";
+export { MassifStartFmt } from "./massifs/types.js";
+export {
+  LogFormat,
+  Urkle,
+  Bloom,
+  IndexV2,
+  leafCountForMassifHeight,
+  leafTableBytes,
+  nodeCountMax,
+  nodeStoreBytes,
+  bloomMBits,
+  bloomBitsetBytes,
+  bloomRegionBytes,
+  indexDataBytesV2,
+  peakStackEnd,
+  massifLogEntries,
+  urkleLeafTableStartFieldIndex,
+  urkleLeafTableStartByteOffset,
+  createLeafEnumerator,
+  leafComponentByteOffset,
+  leafComponentSize,
+  computeLastMMRIndex,
+  isMassifFull,
+} from "./massifs/types.js";
+
+export { parseV2StorageObjectPath } from "./massifs/v2storagepaths.js";
+
+// Export mmr module
+export { mmrIndex, massifFirstLeaf, leafMinusSpurSum } from "./mmr/index.js";
+export {
+  heightIndex,
+  height,
+  mmrIndexFromLeafIndex,
+  leafIndex,
+  mmrPosition,
+  mmrSizeFromHeightIndex,
+  leafCount,
+  leafCountFromHeightIndex,
+} from "./mmr/math.js";
+export {
+  bagPeaks,
+  calculateRoot,
+  verifyInclusion,
+  verifyConsistency,
+} from "./mmr/algorithms.js";
+export { createSyncHasher } from "./mmr/algorithms-sync.js";
+export type { Proof, Peak } from "./mmr/types.js";
+export type { Hasher } from "./mmr/types.js";

package/src/massifs/indexformat.ts

@@ -0,0 +1,217 @@
+/**
+ * Index format constants for v2 massif format
+ *
+ * Constants for the urkle trie and bloom filter index regions.
+ * Based on the Go implementations in:
+ * - go-merklelog/urkle/types.go
+ * - go-merklelog/bloom/types.go
+ * - go-merklelog/massifs/indexformat_v2.go
+ */
+
+/**
+ * Urkle - Constants for urkle trie format
+ */
+export namespace Urkle {
+  /** Fixed width of hashes and values (32 bytes) */
+  export const HashBytes = 32;
+
+  /** Byte width of leafOrdinal in proofs and leaf hashes */
+  export const LeafOrdinalBytes = 4;
+
+  /**
+   * LeafRecordBytes is the fixed byte width of a leaf table record.
+   *
+   * v1 layout (extended for Forestrie v2 index needs):
+   *   - key_be8 (uint64, 8 bytes)
+   *   - valueBytes[32] (committed by trie hash)
+   *   - extra1[24] (auxiliary; not committed by the trie hash)
+   *   - extra2[32] (auxiliary; not committed by the trie hash)
+   *   - extra3[32] (auxiliary; not committed by the trie hash)
+   *
+   * NOTE: Record size is intentionally a multiple of 32 bytes. To achieve this
+   * without truncating valueBytes, we sacrifice 8 bytes from the first extra
+   * field.
+   */
+  export const LeafRecordBytes = 128; // 8 + 32 + 24 + 32 + 32
+
+  /** Fixed byte width of a node store record */
+  export const NodeRecordBytes = 64;
+
+  /** Size of FrontierStateV1 snapshot for resuming append-only construction */
+  export const FrontierStateV1Bytes = 544; // 32 + 64*8
+
+  // Leaf record field offsets
+  export const LeafKeyBytes = 8;
+  export const LeafValueBytes = HashBytes;
+  export const LeafExtra1Bytes = HashBytes - 8; // 24
+  export const LeafExtraBytes = HashBytes; // 32
+
+  export const LeafValueOffset = LeafKeyBytes; // 8
+  export const LeafExtraOffset = LeafValueOffset + LeafValueBytes; // 40
+  export const LeafExtra1Offset = LeafExtraOffset; // 40
+  export const LeafExtra2Offset = LeafExtra1Offset + LeafExtra1Bytes; // 64
+  export const LeafExtra3Offset = LeafExtra2Offset + LeafExtraBytes; // 96
+}
+
+/**
+ * Bloom - Constants for bloom filter format
+ */
+export namespace Bloom {
+  /** Fixed element width (Forestrie log value width) */
+  export const ValueBytes = 32;
+
+  /** Number of parallel Bloom filters in this format */
+  export const Filters = 4;
+
+  /** Fixed header size for BloomHeaderV1 */
+  export const HeaderBytesV1 = 32;
+
+  /** Magic string for BloomHeaderV1 */
+  export const MagicV1 = "BLM1";
+
+  /** Version number for v1 */
+  export const VersionV1 = 1;
+
+  /** BitOrderLSB0: bit 0 is the least-significant bit of byte 0 */
+  export const BitOrderLSB0 = 0;
+}
+
+/**
+ * IndexV2 - V2 massif index configuration constants
+ */
+export namespace IndexV2 {
+  /**
+   * BloomBitsPerElement is the fixed sizing knob for the v2 massif BloomRegion.
+   * mBits = bitsPerElement * leafCount, per filter.
+   */
+  export const BloomBitsPerElement = 10;
+
+  /**
+   * BloomK is the number of hash-derived bit positions set per inserted element.
+   * For b=10, k≈round(0.693*b)=7.
+   */
+  export const BloomK = 7;
+}
+
+/**
+ * Returns the fixed leaf capacity N for a massif height (one-based).
+ *
+ * In massifs, massifHeight is the one-based height h, so leaf capacity is:
+ *   N = 2^(h-1)
+ *
+ * @param massifHeight - One-based massif height
+ * @returns Leaf capacity for the massif
+ */
+export function leafCountForMassifHeight(massifHeight: number): bigint {
+  if (massifHeight === 0) {
+    return 0n;
+  }
+  return 1n << BigInt(massifHeight - 1);
+}
+
+/**
+ * Returns the required leaf table bytes for leafCount leaves.
+ *
+ * @param leafCount - Number of leaves
+ * @returns Byte size of the leaf table
+ */
+export function leafTableBytes(leafCount: bigint): bigint {
+  return leafCount * BigInt(Urkle.LeafRecordBytes);
+}
+
+/**
+ * Returns the maximum number of nodes in a binary trie with leafCount keys.
+ * For a Patricia/crit-bit style binary trie, node count is <= 2N-1.
+ *
+ * @param leafCount - Number of leaves
+ * @returns Maximum node count
+ */
+export function nodeCountMax(leafCount: bigint): bigint {
+  if (leafCount === 0n) {
+    return 0n;
+  }
+  return 2n * leafCount - 1n;
+}
+
+/**
+ * Returns the required node store bytes for leafCount leaves.
+ *
+ * @param leafCount - Number of leaves
+ * @returns Byte size of the node store
+ */
+export function nodeStoreBytes(leafCount: bigint): bigint {
+  return nodeCountMax(leafCount) * BigInt(Urkle.NodeRecordBytes);
+}
+
+/**
+ * Computes mBits = bitsPerElement * leafCount, returning 0 if overflow.
+ *
+ * @param leafCount - Number of leaves
+ * @param bitsPerElement - Bits per element (default: IndexV2.BloomBitsPerElement)
+ * @returns mBits value, or 0 if overflow
+ */
+export function bloomMBits(
+  leafCount: bigint,
+  bitsPerElement: bigint = BigInt(IndexV2.BloomBitsPerElement),
+): bigint {
+  const mBits = bitsPerElement * leafCount;
+  // Check overflow - mBits should fit in uint32
+  if (mBits > 0xffffffffn) {
+    return 0n;
+  }
+  return mBits;
+}
+
+/**
+ * Returns ceil(mBits/8) - the number of bytes for a single filter bitset.
+ *
+ * @param mBits - Number of bits
+ * @returns Byte size of one filter bitset
+ */
+export function bloomBitsetBytes(mBits: bigint): bigint {
+  return (mBits + 7n) / 8n;
+}
+
+/**
+ * Returns the required byte length for a 4-way BloomRegion given mBits:
+ *   HeaderBytesV1 + 4 * ceil(mBits/8)
+ *
+ * @param mBits - Number of bits per filter
+ * @returns Total byte size of bloom region (header + 4 bitsets)
+ */
+export function bloomRegionBytes(mBits: bigint): bigint {
+  const bitsetBytes = bloomBitsetBytes(mBits);
+  return BigInt(Bloom.HeaderBytesV1) + BigInt(Bloom.Filters) * bitsetBytes;
+}
+
+/**
+ * Returns the byte size of the v2 index data region, excluding the fixed 32B
+ * index header.
+ *
+ * v2 index header (32B) is BloomHeaderV1, and the index data is:
+ *   bloom bitsets || urkle frontier || urkle leaf table || urkle node store
+ *
+ * @param leafCount - Number of leaves
+ * @returns Byte size of index data region
+ * @throws Error if mBits overflows
+ */
+export function indexDataBytesV2(leafCount: bigint): bigint {
+  const mBits = bloomMBits(leafCount);
+  if (mBits === 0n && leafCount > 0n) {
+    throw new Error("bloom mBits overflow");
+  }
+  if (leafCount === 0n) {
+    return 0n;
+  }
+
+  // Bloom region bytes includes the 32B header; we exclude that here because
+  // the massif index header is a fixed 32B already accounted for.
+  const bloomRegion = bloomRegionBytes(mBits);
+  const bloomBitsetsOnly = bloomRegion - BigInt(Bloom.HeaderBytesV1);
+
+  const frontierBytes = BigInt(Urkle.FrontierStateV1Bytes);
+  const ltBytes = leafTableBytes(leafCount);
+  const nsBytes = nodeStoreBytes(leafCount);
+
+  return bloomBitsetsOnly + frontierBytes + ltBytes + nsBytes;
+}

package/src/massifs/logformat.ts

@@ -0,0 +1,54 @@
+/**
+ * Massif log format constants
+ *
+ * Constants for calculating offsets and sizes in the massif blob format.
+ * Based on the Go implementation in go-merklelog/massifs/logformat.go
+ *
+ * Reference: https://github.com/datatrails/epic-8120-scalable-proof-mechanisms/blob/main/mmr/forestrie-mmrblobs.md#massif-basic-file-format
+ */
+
+/**
+ * LogFormat - Constants for massif data format
+ *
+ * Matching the Go implementation in go-merklelog/massifs/logformat.go
+ */
+export namespace LogFormat {
+  /**
+   * ValueBytes defines the width of ALL entries in the log. This fixed width
+   * makes it possible to compute mmr current sizes based on knowing only the
+   * massif height and the number of bytes in the file.
+   */
+  export const ValueBytes = 32;
+
+  /**
+   * ReservedHeaderSlots reserves a place to put the urkle trie root, used for
+   * data recovery and proofs of exclusion, and any related material. And it
+   * gives us a little flex in the data format for the initial launch of
+   * forestrie. It would be frustrating to need a data migration for want of a
+   * few bytes.
+   */
+  export const ReservedHeaderSlots = 7; // reserves n * ValueBytes at the front of the blob
+
+  /**
+   * StartHeaderSize - Size of the start header including reserved slots
+   */
+  export const StartHeaderSize = ValueBytes + ValueBytes * ReservedHeaderSlots; // 256
+
+  /**
+   * StartHeaderEnd - End offset of the start header
+   */
+  export const StartHeaderEnd = StartHeaderSize;
+
+  /**
+   * MaxMmrHeight - Maximum MMR height - no single log can be taller than this
+   * Matches the allowable bit size of an mmrIndex.
+   * Note that the max height *index* is 63
+   */
+  export const MaxMmrHeight = 64;
+
+  /**
+   * IndexHeaderBytes - Index header bytes - reserved space for index header.
+   * In v2, this 32B region is the BloomHeaderV1.
+   */
+  export const IndexHeaderBytes = 32;
+}

package/src/massifs/massif.ts

@@ -0,0 +1,182 @@
+import type { MassifStart } from "./massifstart.js";
+import { MassifStartFmt } from "./massifstart.js";
+import { LogFormat } from "./logformat.js";
+import { massifFirstLeaf, leafMinusSpurSum } from "../mmr/index.js";
+
+/**
+ * Massif - Efficient buffer-based access to massif data
+ *
+ * Provides efficient access to massif start data based on a view of an
+ * in-memory buffer which holds the data. All operations work directly with
+ * the buffer without copying.
+ */
+export class Massif {
+  private readonly buffer: Uint8Array;
+
+  /**
+   * Creates a new Massif instance from buffer data
+   * @param data - The massif data as ArrayBuffer, Uint8Array, or Buffer
+   */
+  constructor(
+    data:
+      | ArrayBuffer
+      | Uint8Array
+      | Buffer
+      | { buffer: ArrayBuffer; byteOffset: number; byteLength: number },
+  ) {
+    if (data instanceof ArrayBuffer) {
+      this.buffer = new Uint8Array(data);
+    } else if (data instanceof Uint8Array) {
+      this.buffer = data;
+    } else if (
+      "buffer" in data &&
+      "byteOffset" in data &&
+      "byteLength" in data
+    ) {
+      // Node.js Buffer or similar
+      this.buffer = new Uint8Array(
+        data.buffer,
+        data.byteOffset,
+        data.byteLength,
+      );
+    } else {
+      throw new Error("Unsupported buffer type");
+    }
+  }
+
+  /**
+   * Returns a MassifStart instance populated by reading directly from the buffer
+   *
+   * Computes firstIndex using massif height and index (calls MMR function)
+   * Computes peakStackLen using massif index (calls MMR function)
+   */
+  getStart(): MassifStart {
+    const massifHeight = this.massifHeight;
+    const massifIndex = this.massifIndex;
+
+    // Compute firstIndex using massif height and index
+    const firstIndex = massifFirstLeaf(massifHeight, massifIndex);
+
+    // Compute peakStackLen using massif index
+    const peakStackLen = leafMinusSpurSum(BigInt(massifIndex));
+
+    return {
+      reserved: this.reserved,
+      lastID: this.lastID,
+      version: this.version,
+      commitmentEpoch: this.commitmentEpoch,
+      massifHeight,
+      massifIndex,
+      firstIndex,
+      peakStackLen,
+    };
+  }
+
+  /**
+   * Returns a buffer view of the corresponding field at zero-based index
+   *
+   * Each field is 32 bytes (LogFormat.ValueBytes). Zero-based indexing, without regard
+   * to peakstack or trieindex structure.
+   *
+   * @param index - Zero-based field index (Number or BigInt)
+   * @param count - Number of consecutive fields to include in the view (defaults to 1)
+   * @returns Uint8Array view of the field(s) (32 bytes per field)
+   */
+  fieldref(index: number | bigint, count: number = 1): Uint8Array {
+    const idx = typeof index === "bigint" ? Number(index) : index;
+    const offset = idx * LogFormat.ValueBytes;
+    return this.buffer.slice(offset, offset + count * LogFormat.ValueBytes);
+  }
+
+  /**
+   * Reads bytes from the buffer at the specified offset
+   *
+   * Returns a view of the buffer without copying. Efficient for reading
+   * arbitrary byte ranges that may span multiple 32-byte fields.
+   *
+   * @param offset - Byte offset into the buffer
+   * @param length - Number of bytes to read
+   * @returns Uint8Array view of the requested bytes
+   */
+  readBytes(offset: number, length: number): Uint8Array {
+    return this.buffer.slice(offset, offset + length);
+  }
+
+  /**
+   * Reads a big-endian uint64 from the buffer
+   */
+  private readUint64BE(offset: number): bigint {
+    const view = new DataView(
+      this.buffer.buffer,
+      this.buffer.byteOffset + offset,
+      8,
+    );
+    return view.getBigUint64(0, false); // false = big-endian
+  }
+
+  /**
+   * Reads a big-endian uint32 from the buffer
+   */
+  private readUint32BE(offset: number): number {
+    const view = new DataView(
+      this.buffer.buffer,
+      this.buffer.byteOffset + offset,
+      4,
+    );
+    return view.getUint32(0, false); // false = big-endian
+  }
+
+  /**
+   * Reads a big-endian uint16 from the buffer
+   */
+  private readUint16BE(offset: number): number {
+    const view = new DataView(
+      this.buffer.buffer,
+      this.buffer.byteOffset + offset,
+      2,
+    );
+    return view.getUint16(0, false); // false = big-endian
+  }
+
+  /**
+   * Reserved field (bytes 0-7)
+   */
+  get reserved(): bigint {
+    return this.readUint64BE(0);
+  }
+
+  /**
+   * Last ID timestamp (bytes 8-15)
+   */
+  get lastID(): bigint {
+    return this.readUint64BE(MassifStartFmt.LastIdFirstByte);
+  }
+
+  /**
+   * Version (bytes 21-22)
+   */
+  get version(): number {
+    return this.readUint16BE(MassifStartFmt.VersionFirstByte);
+  }
+
+  /**
+   * Commitment epoch (bytes 23-26)
+   */
+  get commitmentEpoch(): number {
+    return this.readUint32BE(MassifStartFmt.EpochFirstByte);
+  }
+
+  /**
+   * Massif height (byte 27)
+   */
+  get massifHeight(): number {
+    return this.buffer[MassifStartFmt.MassifHeightFirstByte];
+  }
+
+  /**
+   * Massif index (bytes 28-31)
+   */
+  get massifIndex(): number {
+    return this.readUint32BE(MassifStartFmt.MassifFirstByte);
+  }
+}

package/src/massifs/massiffull.ts

@@ -0,0 +1,28 @@
+/**
+ * Massif fullness determination functions
+ *
+ * Functions for determining if a massif is full based on its height and log entries.
+ */
+
+/**
+ * Determine if a massif is full.
+ *
+ * @param massifHeight - Massif height (1-based)
+ * @param logEntries - Number of log entries (nodes) in the massif
+ * @returns True if the massif is full
+ */
+export function isMassifFull(
+  massifHeight: number,
+  logEntries: bigint,
+): boolean {
+  // Convert 1-based height to 0-based height index
+  const heightIndex = massifHeight - 1;
+
+  // Expected number of leaves for a full massif: f = 2^g = 1 << heightIndex
+  const expectedLeaves = BigInt(1 << heightIndex);
+
+  // Actual number of leaves: f = (n + 1) / 2
+  const actualLeaves = (logEntries + 1n) >> 1n;
+
+  return actualLeaves >= expectedLeaves;
+}

package/src/massifs/massiflogentries.ts

@@ -0,0 +1,40 @@
+/**
+ * MassifLogEntries calculation for v2 massif format
+ */
+
+import { LogFormat } from "./logformat.js";
+import { peakStackEnd } from "./peakstackend.js";
+
+/**
+ * MassifLogEntries calculates the number of log entries (nodes) in a blob from
+ * the length of the blob in bytes.
+ *
+ * It does this by accounting for the index data and other header data.
+ * If you know the FirstIndex from the massif start header you can get the
+ * overall mmr size by direct addition.
+ *
+ * Note: this function exists so we can compute the mmrSize from just the blob
+ * store metadata: we store the FirstIndex on a blob tag, and the blob
+ * metadata includes ContentLength. This means when we are checking if a root
+ * seal covers the current log head, we don't need to fetch the log massif blob.
+ *
+ * @param dataLen - Total length of the massif blob in bytes
+ * @param massifHeight - One-based massif height
+ * @returns Number of log entries (nodes) in the blob
+ * @throws Error if the data length is too short to contain the required headers
+ */
+export function massifLogEntries(
+  dataLen: number,
+  massifHeight: number,
+): bigint {
+  const stackEnd = peakStackEnd(massifHeight);
+
+  if (BigInt(dataLen) < stackEnd) {
+    throw new Error(
+      `Massif data length ${dataLen} is too short. Minimum required: ${stackEnd} bytes`,
+    );
+  }
+
+  const mmrByteCount = BigInt(dataLen) - stackEnd;
+  return mmrByteCount / BigInt(LogFormat.ValueBytes);
+}

package/src/massifs/massifstart.ts

@@ -0,0 +1,67 @@
+/**
+ * MassifStart - Header information for a massif blob
+ *
+ * The massif start is a 32-byte field encoding bookkeeping information
+ * required in a blob to allow for efficient correctness checks.
+ */
+export interface MassifStart {
+  /** Reserved field (bytes 0-7) */
+  reserved: bigint;
+  /** Last ID timestamp (bytes 8-15) */
+  lastID: bigint;
+  /** Version (bytes 21-22) */
+  version: number;
+  /** Commitment epoch (bytes 23-26) */
+  commitmentEpoch: number;
+  /** Massif height (byte 27) */
+  massifHeight: number;
+  /** Massif index (bytes 28-31) */
+  massifIndex: number;
+  /** First MMR index in this massif (computed) */
+  firstIndex: bigint;
+  /** Peak stack length (computed) */
+  peakStackLen: bigint;
+}
+
+/**
+ * MassifStartFmt - Format constants for MassifStart fields
+ *
+ * Byte offsets and sizes for MassifStart fields.
+ * Matching the Go implementation in go-merklelog/massifs/massifstart.go
+ */
+export namespace MassifStartFmt {
+  /** Last ID timestamp field - first byte offset */
+  export const LastIdFirstByte = 8;
+  /** Last ID timestamp field - size in bytes */
+  export const LastIdSize = 8;
+  /** Last ID timestamp field - end byte offset */
+  export const LastIdEnd = LastIdFirstByte + LastIdSize;
+
+  /** Version field - first byte offset */
+  export const VersionFirstByte = 21;
+  /** Version field - size in bytes */
+  export const VersionSize = 2;
+  /** Version field - end byte offset */
+  export const VersionEnd = VersionFirstByte + VersionSize;
+
+  /** Commitment epoch field - first byte offset */
+  export const EpochFirstByte = VersionEnd; // 23
+  /** Commitment epoch field - size in bytes */
+  export const EpochSize = 4;
+  /** Commitment epoch field - end byte offset */
+  export const EpochEnd = EpochFirstByte + EpochSize;
+
+  /** Massif height field - first byte offset */
+  export const MassifHeightFirstByte = EpochEnd; // 27
+  /** Massif height field - size in bytes */
+  export const MassifHeightSize = 1;
+  /** Massif height field - end byte offset */
+  export const MassifHeightEnd = MassifHeightFirstByte + MassifHeightSize;
+
+  /** Massif index field - first byte offset */
+  export const MassifFirstByte = MassifHeightEnd; // 28
+  /** Massif index field - size in bytes */
+  export const MassifSize = 4;
+  /** Massif index field - end byte offset */
+  export const MassifEnd = MassifFirstByte + MassifSize;
+}