@lodestar/beacon-node 1.48.0-dev.b71ab45af3 → 1.48.0-dev.bb3ffba9e9

package/src/db/flatFileStore/dcolFormat.ts

@@ -0,0 +1,389 @@
+/**
+ * `.dcol` binary format for multi-column data column files.
+ *
+ * Layout:
+ *   HEADER (149 bytes):
+ *     [version:     1B = 0x01]
+ *     [_reserved:   4B]       — zero-filled
+ *     [bitmap:     16B]       — 128-bit bitmap of which columns are present
+ *     [blockRoot:  32B]
+ *     [slot:        8B LE]       — only low 4 bytes used (matching SSZ convention)
+ *     [reserved:   88B]       — zero-filled, for future use
+ *
+ *   OFFSET TABLE ((N+1) * 4 bytes, where N = popcount(bitmap)):
+ *     [offset_0:4B BE]   start of column 0's compressed data, relative to data region start
+ *     [offset_1:4B BE]   ...
+ *     [offset_N:4B BE]   end of last column = total data region size (sentinel)
+ *
+ *   DATA REGION:
+ *     [snappy(column_0)][snappy(column_1)]...[snappy(column_N-1)]
+ *
+ * Each column is independently Snappy block-compressed. To read column at bitmap position `p`:
+ *   dataStart = HEADER_SIZE + (N+1)*4
+ *   colStart  = dataStart + offsets[p]
+ *   colEnd    = dataStart + offsets[p+1]
+ *   column    = snappy.uncompress(file[colStart:colEnd])
+ */
+
+import {getSlotFromOffset} from "../../util/sszBytes.js";
+import {DataColumnStoreError, DataColumnStoreErrorCode} from "./errors.js";
+import {compress, uncompress} from "./snappy.js";
+
+export const DCOL_VERSION = 0x01;
+export const DCOL_HEADER_SIZE = 149;
+const BITMAP_BYTES = 16;
+const BITMAP_BITS = BITMAP_BYTES * 8;
+const BITMAP_OFFSET = 5;
+const BLOCK_ROOT_OFFSET = 21;
+const BLOCK_ROOT_BYTES = 32;
+const SLOT_OFFSET = 53;
+const MAX_SLOT = 0xffffffff;
+
+// --- Bitmap helpers ---
+
+/** Get bit `i` from a 16-byte Uint8Array bitmap (little-endian bit order within each byte). */
+export function getBit(bitmap: Uint8Array, i: number): boolean {
+  const byteIdx = Math.floor(i / 8);
+  const bitIdx = i % 8;
+  return (bitmap[byteIdx] & (1 << bitIdx)) !== 0;
+}
+
+/** Set bit `i` in a 16-byte Uint8Array bitmap. */
+export function setBit(bitmap: Uint8Array, i: number): void {
+  const byteIdx = Math.floor(i / 8);
+  const bitIdx = i % 8;
+  bitmap[byteIdx] |= 1 << bitIdx;
+}
+
+/** Count set bits in bitmap for positions 0..i-1 (exclusive of i). */
+export function popcount(bitmap: Uint8Array, upTo: number): number {
+  let count = 0;
+  for (let j = 0; j < upTo; j++) {
+    if (getBit(bitmap, j)) count++;
+  }
+  return count;
+}
+
+/** Count total set bits in bitmap. */
+export function totalBits(bitmap: Uint8Array): number {
+  let count = 0;
+  for (let byteIdx = 0; byteIdx < BITMAP_BYTES; byteIdx++) {
+    let b = bitmap[byteIdx];
+    while (b) {
+      count += b & 1;
+      b >>= 1;
+    }
+  }
+  return count;
+}
+
+// --- Header encode/decode ---
+
+export interface DcolHeader {
+  version: number;
+  bitmap: Uint8Array;
+  blockRoot: Uint8Array;
+  slot: number;
+}
+
+export function encodeDcolHeader(header: DcolHeader): Uint8Array {
+  // Defensive checks at the persistence boundary. Normal sidecars are already validated, but silently truncating
+  // malformed metadata here would turn an upstream regression into durable on-disk corruption.
+  if (header.bitmap.length !== BITMAP_BYTES) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.INVALID_BITMAP_LENGTH, length: header.bitmap.length},
+      `Invalid dcol bitmap length: ${header.bitmap.length}`
+    );
+  }
+  if (header.blockRoot.length !== BLOCK_ROOT_BYTES) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.INVALID_BLOCK_ROOT_LENGTH, length: header.blockRoot.length},
+      `Invalid dcol block root length: ${header.blockRoot.length}`
+    );
+  }
+  if (!Number.isInteger(header.slot) || header.slot < 0 || header.slot > MAX_SLOT) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.INVALID_SLOT, slot: header.slot},
+      `Invalid dcol slot: ${header.slot}`
+    );
+  }
+
+  const buf = new Uint8Array(DCOL_HEADER_SIZE);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+
+  buf[0] = header.version;
+  // bytes 1-4 reserved (zero)
+  buf.set(header.bitmap, BITMAP_OFFSET);
+  buf.set(header.blockRoot, BLOCK_ROOT_OFFSET);
+
+  // Slot as 8-byte LE (matching SSZ convention used throughout Lodestar).
+  // High 4 bytes are always zero for practical slot values (4B covers ~1634 years).
+  view.setUint32(SLOT_OFFSET, header.slot >>> 0, true);
+  // bytes SLOT_OFFSET+4..+7 are already zero
+
+  // reserved bytes are already zero
+  return buf;
+}
+
+export function parseDcolHeader(data: Uint8Array): DcolHeader {
+  if (data.length < DCOL_HEADER_SIZE) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.FILE_TOO_SMALL, actual: data.length, minimum: DCOL_HEADER_SIZE},
+      `dcol file too small: ${data.length} < ${DCOL_HEADER_SIZE}`
+    );
+  }
+
+  const version = data[0];
+  if (version !== DCOL_VERSION) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.UNSUPPORTED_VERSION, version},
+      `Unsupported dcol version: ${version}`
+    );
+  }
+
+  const bitmap = Uint8Array.prototype.slice.call(data, BITMAP_OFFSET, BITMAP_OFFSET + BITMAP_BYTES) as Uint8Array;
+  const blockRoot = Uint8Array.prototype.slice.call(data, BLOCK_ROOT_OFFSET, BLOCK_ROOT_OFFSET + 32) as Uint8Array;
+
+  const slot = getSlotFromOffset(data, SLOT_OFFSET);
+  if (slot === null) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.SLOT_OUT_OF_RANGE},
+      "dcol slot exceeds 4-byte range"
+    );
+  }
+
+  return {version, bitmap, blockRoot, slot};
+}
+
+// --- Column read helpers ---
+
+/**
+ * Read a single column by its column index from a dcol file buffer.
+ * Returns the uncompressed column data, or null if column is not present.
+ *
+ * Note: Production reads use getColumnByteRange() + fd.read() for targeted I/O.
+ * This function is a convenience for tests and mergeDcolColumns.
+ */
+export function readColumn(fileData: Uint8Array, header: DcolHeader, index: number): Uint8Array | null {
+  const {dataStart, offsets} = parseDcolOffsetsFromFile(fileData, header);
+  if (!getBit(header.bitmap, index)) return null;
+
+  const p = popcount(header.bitmap, index);
+  const colStart = dataStart + offsets[p];
+  const colEnd = dataStart + offsets[p + 1];
+  const compressed = fileData.subarray(colStart, colEnd);
+
+  return uncompress(compressed);
+}
+
+/**
+ * Read all columns from a dcol file buffer.
+ * Returns an array of {index, data} for each present column.
+ */
+export function readAllColumns(fileData: Uint8Array, header: DcolHeader): {index: number; data: Uint8Array}[] {
+  const result: {index: number; data: Uint8Array}[] = [];
+  const {dataStart, offsets} = parseDcolOffsetsFromFile(fileData, header);
+
+  let pos = 0;
+  for (let i = 0; i < 128; i++) {
+    if (getBit(header.bitmap, i)) {
+      const colStart = dataStart + offsets[pos];
+      const colEnd = dataStart + offsets[pos + 1];
+      const compressed = fileData.subarray(colStart, colEnd);
+      result.push({index: i, data: uncompress(compressed)});
+      pos++;
+    }
+  }
+
+  return result;
+}
+
+// --- Targeted read helpers (for fd.read()-based access) ---
+
+export interface ColumnByteRange {
+  /** Absolute file offset to start reading */
+  offset: number;
+  /** Number of compressed bytes to read */
+  length: number;
+}
+
+/**
+ * Compute the file byte range for a specific column, using the header
+ * and offset table bytes. Returns null if column is absent.
+ */
+export function getColumnByteRange(header: DcolHeader, offsets: number[], index: number): ColumnByteRange | null {
+  if (!getBit(header.bitmap, index)) return null;
+
+  // popcount(bitmap, index) counts bits in [0, index) — the number of columns
+  // stored before this one, giving the correct offset table position.
+  const p = popcount(header.bitmap, index);
+  const N = totalBits(header.bitmap);
+  const dataStart = DCOL_HEADER_SIZE + offsetTableSize(N);
+  const colStart = offsets[p];
+  const colEnd = offsets[p + 1];
+
+  return {
+    offset: dataStart + colStart,
+    length: colEnd - colStart,
+  };
+}
+
+export function parseDcolOffsets(offsetTable: Uint8Array, columnCount: number, dataLength: number): number[] {
+  const expectedTableSize = offsetTableSize(columnCount);
+  if (offsetTable.length !== expectedTableSize || dataLength < 0) {
+    throw invalidOffsetTable("size");
+  }
+
+  const view = new DataView(offsetTable.buffer, offsetTable.byteOffset, offsetTable.byteLength);
+  const offsets = Array.from({length: columnCount + 1}, (_, index) => view.getUint32(index * 4, false));
+  if (offsets[0] !== 0) {
+    throw invalidOffsetTable("first offset is not zero");
+  }
+  for (let i = 1; i < offsets.length; i++) {
+    if (offsets[i] < offsets[i - 1]) {
+      throw invalidOffsetTable("offsets are not monotonic");
+    }
+  }
+  if (offsets[columnCount] !== dataLength) {
+    throw invalidOffsetTable("terminal offset does not match data length");
+  }
+
+  return offsets;
+}
+
+function parseDcolOffsetsFromFile(fileData: Uint8Array, header: DcolHeader): {dataStart: number; offsets: number[]} {
+  const columnCount = totalBits(header.bitmap);
+  const tableSize = offsetTableSize(columnCount);
+  const dataStart = DCOL_HEADER_SIZE + tableSize;
+  const offsetTable = fileData.subarray(DCOL_HEADER_SIZE, dataStart);
+  return {
+    dataStart,
+    offsets: parseDcolOffsets(offsetTable, columnCount, fileData.length - dataStart),
+  };
+}
+
+function invalidOffsetTable(reason: string): DataColumnStoreError {
+  return new DataColumnStoreError(
+    {code: DataColumnStoreErrorCode.INVALID_OFFSET_TABLE, reason},
+    `Invalid dcol offset table: ${reason}`
+  );
+}
+
+/** Size of the offset table in bytes for N present columns. */
+export function offsetTableSize(N: number): number {
+  return (N + 1) * 4;
+}
+
+// --- Full file encode/decode ---
+
+/**
+ * Encode a complete dcol file from columns.
+ * Each column is independently Snappy block-compressed.
+ */
+export function encodeDcolFile(
+  blockRoot: Uint8Array,
+  slot: number,
+  columns: {index: number; data: Uint8Array}[]
+): Uint8Array {
+  if (columns.length === 0) {
+    throw new DataColumnStoreError(
+      {code: DataColumnStoreErrorCode.EMPTY_COLUMNS},
+      "Cannot encode dcol file with zero columns"
+    );
+  }
+
+  // Defensive validation keeps bitmap membership and offset table entries in one-to-one correspondence.
+  const seenIndices = new Set<number>();
+  for (const {index} of columns) {
+    if (!Number.isInteger(index) || index < 0 || index >= BITMAP_BITS) {
+      throw new DataColumnStoreError(
+        {code: DataColumnStoreErrorCode.INVALID_COLUMN_INDEX, index},
+        `Invalid dcol column index: ${index}`
+      );
+    }
+    if (seenIndices.has(index)) {
+      throw new DataColumnStoreError(
+        {code: DataColumnStoreErrorCode.DUPLICATE_COLUMN_INDEX, index},
+        `Duplicate dcol column index: ${index}`
+      );
+    }
+    seenIndices.add(index);
+  }
+
+  const bitmap = new Uint8Array(BITMAP_BYTES);
+
+  // Sort by index for deterministic ordering
+  const sorted = [...columns].sort((a, b) => a.index - b.index);
+
+  for (const col of sorted) {
+    setBit(bitmap, col.index);
+  }
+
+  // Compress each column
+  const compressed: Uint8Array[] = [];
+  for (const col of sorted) {
+    compressed.push(compress(col.data));
+  }
+
+  // Build offset table
+  const N = sorted.length;
+  const oTableSize = offsetTableSize(N);
+  const offsets = new Uint8Array(oTableSize);
+  const offsetView = new DataView(offsets.buffer, offsets.byteOffset, offsets.byteLength);
+
+  let cumulative = 0;
+  for (let i = 0; i < N; i++) {
+    offsetView.setUint32(i * 4, cumulative, false); // BE
+    cumulative += compressed[i].length;
+  }
+  offsetView.setUint32(N * 4, cumulative, false); // sentinel: end of last column
+
+  // Assemble: header + offset table + compressed data
+  const header = encodeDcolHeader({version: DCOL_VERSION, bitmap, blockRoot, slot});
+  const totalSize = DCOL_HEADER_SIZE + oTableSize + cumulative;
+  const result = new Uint8Array(totalSize);
+
+  result.set(header, 0);
+  result.set(offsets, DCOL_HEADER_SIZE);
+
+  let dataOffset = DCOL_HEADER_SIZE + oTableSize;
+  for (const chunk of compressed) {
+    result.set(chunk, dataOffset);
+    dataOffset += chunk.length;
+  }
+
+  return result;
+}
+
+/**
+ * Merge new columns into an existing dcol file buffer.
+ */
+export function mergeDcolColumns(existing: Uint8Array, newColumns: {index: number; data: Uint8Array}[]): Uint8Array {
+  const header = parseDcolHeader(existing);
+
+  const existingCols = readAllColumns(existing, header);
+
+  const allColumns: Map<number, Uint8Array> = new Map();
+  for (const col of existingCols) {
+    allColumns.set(col.index, col.data);
+  }
+
+  // Defensively reject ambiguous batches while still allowing a new column to overwrite its previously stored value.
+  const newIndices = new Set<number>();
+  for (const col of newColumns) {
+    if (newIndices.has(col.index)) {
+      throw new DataColumnStoreError(
+        {code: DataColumnStoreErrorCode.DUPLICATE_COLUMN_INDEX, index: col.index},
+        `Duplicate dcol column index: ${col.index}`
+      );
+    }
+    newIndices.add(col.index);
+    allColumns.set(col.index, col.data);
+  }
+
+  const cols = Array.from(allColumns.entries())
+    .sort(([a], [b]) => a - b)
+    .map(([index, data]) => ({index, data}));
+
+  return encodeDcolFile(header.blockRoot, header.slot, cols);
+}

package/src/db/flatFileStore/errors.ts

@@ -0,0 +1,53 @@
+import {LodestarError} from "@lodestar/utils";
+
+export enum DataColumnStoreErrorCode {
+  INVALID_ROOT = "DATA_COLUMN_STORE_INVALID_ROOT",
+  INVALID_SLOT = "DATA_COLUMN_STORE_INVALID_SLOT",
+  SLOT_OUT_OF_RANGE = "DATA_COLUMN_STORE_SLOT_OUT_OF_RANGE",
+  INVALID_BITMAP_LENGTH = "DATA_COLUMN_STORE_INVALID_BITMAP_LENGTH",
+  INVALID_BLOCK_ROOT_LENGTH = "DATA_COLUMN_STORE_INVALID_BLOCK_ROOT_LENGTH",
+  FILE_TOO_SMALL = "DATA_COLUMN_STORE_FILE_TOO_SMALL",
+  UNSUPPORTED_VERSION = "DATA_COLUMN_STORE_UNSUPPORTED_VERSION",
+  EMPTY_COLUMNS = "DATA_COLUMN_STORE_EMPTY_COLUMNS",
+  INVALID_COLUMN_INDEX = "DATA_COLUMN_STORE_INVALID_COLUMN_INDEX",
+  DUPLICATE_COLUMN_INDEX = "DATA_COLUMN_STORE_DUPLICATE_COLUMN_INDEX",
+  INVALID_OFFSET_TABLE = "DATA_COLUMN_STORE_INVALID_OFFSET_TABLE",
+  SLOT_PRUNED = "DATA_COLUMN_STORE_SLOT_PRUNED",
+  SLOT_MISMATCH = "DATA_COLUMN_STORE_SLOT_MISMATCH",
+  ROOT_MISMATCH = "DATA_COLUMN_STORE_ROOT_MISMATCH",
+  UNEXPECTED_EOF = "DATA_COLUMN_STORE_UNEXPECTED_EOF",
+  OPERATION_FAILED = "DATA_COLUMN_STORE_OPERATION_FAILED",
+  BATCH_DELETE_FAILED = "DATA_COLUMN_STORE_BATCH_DELETE_FAILED",
+  STARTUP_FAILED = "DATA_COLUMN_STORE_STARTUP_FAILED",
+}
+
+export type DataColumnStoreErrorType =
+  | {code: DataColumnStoreErrorCode.INVALID_ROOT; root: string}
+  | {code: DataColumnStoreErrorCode.INVALID_SLOT; slot: number}
+  | {code: DataColumnStoreErrorCode.SLOT_OUT_OF_RANGE}
+  | {code: DataColumnStoreErrorCode.INVALID_BITMAP_LENGTH; length: number}
+  | {code: DataColumnStoreErrorCode.INVALID_BLOCK_ROOT_LENGTH; length: number}
+  | {code: DataColumnStoreErrorCode.FILE_TOO_SMALL; actual: number; minimum: number}
+  | {code: DataColumnStoreErrorCode.UNSUPPORTED_VERSION; version: number}
+  | {code: DataColumnStoreErrorCode.EMPTY_COLUMNS}
+  | {code: DataColumnStoreErrorCode.INVALID_COLUMN_INDEX; index: number}
+  | {code: DataColumnStoreErrorCode.DUPLICATE_COLUMN_INDEX; index: number}
+  | {code: DataColumnStoreErrorCode.INVALID_OFFSET_TABLE; reason: string}
+  | {code: DataColumnStoreErrorCode.SLOT_PRUNED; slot: number; minRetainedSlot: number}
+  | {code: DataColumnStoreErrorCode.SLOT_MISMATCH; headerSlot: number; pathSlot: number}
+  | {code: DataColumnStoreErrorCode.ROOT_MISMATCH; headerRoot: string; pathRoot: string}
+  | {code: DataColumnStoreErrorCode.UNEXPECTED_EOF; offset: number}
+  | {code: DataColumnStoreErrorCode.OPERATION_FAILED; operation: string}
+  | {code: DataColumnStoreErrorCode.BATCH_DELETE_FAILED; failures: number}
+  | {code: DataColumnStoreErrorCode.STARTUP_FAILED};
+
+export class DataColumnStoreError extends LodestarError<DataColumnStoreErrorType> {
+  constructor(type: DataColumnStoreErrorType, message?: string, cause?: unknown) {
+    super(type, message);
+    this.cause = cause;
+  }
+}
+
+export function isFsNotFoundError(error: unknown): boolean {
+  return error instanceof Error && "code" in error && error.code === "ENOENT";
+}