@lodestar/beacon-node 1.48.0-dev.fd70e5fc15 → 1.48.0-dev.ff9749be2c

package/lib/db/flatFileStore/errors.d.ts

@@ -0,0 +1,82 @@
+import { LodestarError } from "@lodestar/utils";
+export declare enum DataColumnStoreErrorCode {
+    INVALID_ROOT = "DATA_COLUMN_STORE_INVALID_ROOT",
+    INVALID_SLOT = "DATA_COLUMN_STORE_INVALID_SLOT",
+    SLOT_OUT_OF_RANGE = "DATA_COLUMN_STORE_SLOT_OUT_OF_RANGE",
+    INVALID_BITMAP_LENGTH = "DATA_COLUMN_STORE_INVALID_BITMAP_LENGTH",
+    INVALID_BLOCK_ROOT_LENGTH = "DATA_COLUMN_STORE_INVALID_BLOCK_ROOT_LENGTH",
+    FILE_TOO_SMALL = "DATA_COLUMN_STORE_FILE_TOO_SMALL",
+    UNSUPPORTED_VERSION = "DATA_COLUMN_STORE_UNSUPPORTED_VERSION",
+    EMPTY_COLUMNS = "DATA_COLUMN_STORE_EMPTY_COLUMNS",
+    INVALID_COLUMN_INDEX = "DATA_COLUMN_STORE_INVALID_COLUMN_INDEX",
+    DUPLICATE_COLUMN_INDEX = "DATA_COLUMN_STORE_DUPLICATE_COLUMN_INDEX",
+    INVALID_OFFSET_TABLE = "DATA_COLUMN_STORE_INVALID_OFFSET_TABLE",
+    SLOT_PRUNED = "DATA_COLUMN_STORE_SLOT_PRUNED",
+    SLOT_MISMATCH = "DATA_COLUMN_STORE_SLOT_MISMATCH",
+    ROOT_MISMATCH = "DATA_COLUMN_STORE_ROOT_MISMATCH",
+    UNEXPECTED_EOF = "DATA_COLUMN_STORE_UNEXPECTED_EOF",
+    OPERATION_FAILED = "DATA_COLUMN_STORE_OPERATION_FAILED",
+    BATCH_DELETE_FAILED = "DATA_COLUMN_STORE_BATCH_DELETE_FAILED",
+    STARTUP_FAILED = "DATA_COLUMN_STORE_STARTUP_FAILED"
+}
+export type DataColumnStoreErrorType = {
+    code: DataColumnStoreErrorCode.INVALID_ROOT;
+    root: string;
+} | {
+    code: DataColumnStoreErrorCode.INVALID_SLOT;
+    slot: number;
+} | {
+    code: DataColumnStoreErrorCode.SLOT_OUT_OF_RANGE;
+} | {
+    code: DataColumnStoreErrorCode.INVALID_BITMAP_LENGTH;
+    length: number;
+} | {
+    code: DataColumnStoreErrorCode.INVALID_BLOCK_ROOT_LENGTH;
+    length: number;
+} | {
+    code: DataColumnStoreErrorCode.FILE_TOO_SMALL;
+    actual: number;
+    minimum: number;
+} | {
+    code: DataColumnStoreErrorCode.UNSUPPORTED_VERSION;
+    version: number;
+} | {
+    code: DataColumnStoreErrorCode.EMPTY_COLUMNS;
+} | {
+    code: DataColumnStoreErrorCode.INVALID_COLUMN_INDEX;
+    index: number;
+} | {
+    code: DataColumnStoreErrorCode.DUPLICATE_COLUMN_INDEX;
+    index: number;
+} | {
+    code: DataColumnStoreErrorCode.INVALID_OFFSET_TABLE;
+    reason: string;
+} | {
+    code: DataColumnStoreErrorCode.SLOT_PRUNED;
+    slot: number;
+    minRetainedSlot: number;
+} | {
+    code: DataColumnStoreErrorCode.SLOT_MISMATCH;
+    headerSlot: number;
+    pathSlot: number;
+} | {
+    code: DataColumnStoreErrorCode.ROOT_MISMATCH;
+    headerRoot: string;
+    pathRoot: string;
+} | {
+    code: DataColumnStoreErrorCode.UNEXPECTED_EOF;
+    offset: number;
+} | {
+    code: DataColumnStoreErrorCode.OPERATION_FAILED;
+    operation: string;
+} | {
+    code: DataColumnStoreErrorCode.BATCH_DELETE_FAILED;
+    failures: number;
+} | {
+    code: DataColumnStoreErrorCode.STARTUP_FAILED;
+};
+export declare class DataColumnStoreError extends LodestarError<DataColumnStoreErrorType> {
+    constructor(type: DataColumnStoreErrorType, message?: string, cause?: unknown);
+}
+export declare function isFsNotFoundError(error: unknown): boolean;
+//# sourceMappingURL=errors.d.ts.map

package/lib/db/flatFileStore/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,oBAAY,wBAAwB;IAClC,YAAY,mCAAmC;IAC/C,YAAY,mCAAmC;IAC/C,iBAAiB,wCAAwC;IACzD,qBAAqB,4CAA4C;IACjE,yBAAyB,gDAAgD;IACzE,cAAc,qCAAqC;IACnD,mBAAmB,0CAA0C;IAC7D,aAAa,oCAAoC;IACjD,oBAAoB,2CAA2C;IAC/D,sBAAsB,6CAA6C;IACnE,oBAAoB,2CAA2C;IAC/D,WAAW,kCAAkC;IAC7C,aAAa,oCAAoC;IACjD,aAAa,oCAAoC;IACjD,cAAc,qCAAqC;IACnD,gBAAgB,uCAAuC;IACvD,mBAAmB,0CAA0C;IAC7D,cAAc,qCAAqC;CACpD;AAED,MAAM,MAAM,wBAAwB,GAChC;IAAC,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAC,GAC3D;IAAC,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAC,GAC3D;IAAC,IAAI,EAAE,wBAAwB,CAAC,iBAAiB,CAAA;CAAC,GAClD;IAAC,IAAI,EAAE,wBAAwB,CAAC,qBAAqB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,GACtE;IAAC,IAAI,EAAE,wBAAwB,CAAC,yBAAyB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,GAC1E;IAAC,IAAI,EAAE,wBAAwB,CAAC,cAAc,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GAChF;IAAC,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACrE;IAAC,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAA;CAAC,GAC9C;IAAC,IAAI,EAAE,wBAAwB,CAAC,oBAAoB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,GACpE;IAAC,IAAI,EAAE,wBAAwB,CAAC,sBAAsB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,GACtE;IAAC,IAAI,EAAE,wBAAwB,CAAC,oBAAoB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,GACrE;IAAC,IAAI,EAAE,wBAAwB,CAAC,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAC,GACnF;IAAC,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,GACpF;IAAC,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,GACpF;IAAC,IAAI,EAAE,wBAAwB,CAAC,cAAc,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,GAC/D;IAAC,IAAI,EAAE,wBAAwB,CAAC,gBAAgB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAC,GACpE;IAAC,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,GACtE;IAAC,IAAI,EAAE,wBAAwB,CAAC,cAAc,CAAA;CAAC,CAAC;AAEpD,qBAAa,oBAAqB,SAAQ,aAAa,CAAC,wBAAwB,CAAC;IAC/E,YAAY,IAAI,EAAE,wBAAwB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAG5E;CACF;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEzD"}

package/lib/db/flatFileStore/errors.js

@@ -0,0 +1,32 @@
+import { LodestarError } from "@lodestar/utils";
+export var DataColumnStoreErrorCode;
+(function (DataColumnStoreErrorCode) {
+    DataColumnStoreErrorCode["INVALID_ROOT"] = "DATA_COLUMN_STORE_INVALID_ROOT";
+    DataColumnStoreErrorCode["INVALID_SLOT"] = "DATA_COLUMN_STORE_INVALID_SLOT";
+    DataColumnStoreErrorCode["SLOT_OUT_OF_RANGE"] = "DATA_COLUMN_STORE_SLOT_OUT_OF_RANGE";
+    DataColumnStoreErrorCode["INVALID_BITMAP_LENGTH"] = "DATA_COLUMN_STORE_INVALID_BITMAP_LENGTH";
+    DataColumnStoreErrorCode["INVALID_BLOCK_ROOT_LENGTH"] = "DATA_COLUMN_STORE_INVALID_BLOCK_ROOT_LENGTH";
+    DataColumnStoreErrorCode["FILE_TOO_SMALL"] = "DATA_COLUMN_STORE_FILE_TOO_SMALL";
+    DataColumnStoreErrorCode["UNSUPPORTED_VERSION"] = "DATA_COLUMN_STORE_UNSUPPORTED_VERSION";
+    DataColumnStoreErrorCode["EMPTY_COLUMNS"] = "DATA_COLUMN_STORE_EMPTY_COLUMNS";
+    DataColumnStoreErrorCode["INVALID_COLUMN_INDEX"] = "DATA_COLUMN_STORE_INVALID_COLUMN_INDEX";
+    DataColumnStoreErrorCode["DUPLICATE_COLUMN_INDEX"] = "DATA_COLUMN_STORE_DUPLICATE_COLUMN_INDEX";
+    DataColumnStoreErrorCode["INVALID_OFFSET_TABLE"] = "DATA_COLUMN_STORE_INVALID_OFFSET_TABLE";
+    DataColumnStoreErrorCode["SLOT_PRUNED"] = "DATA_COLUMN_STORE_SLOT_PRUNED";
+    DataColumnStoreErrorCode["SLOT_MISMATCH"] = "DATA_COLUMN_STORE_SLOT_MISMATCH";
+    DataColumnStoreErrorCode["ROOT_MISMATCH"] = "DATA_COLUMN_STORE_ROOT_MISMATCH";
+    DataColumnStoreErrorCode["UNEXPECTED_EOF"] = "DATA_COLUMN_STORE_UNEXPECTED_EOF";
+    DataColumnStoreErrorCode["OPERATION_FAILED"] = "DATA_COLUMN_STORE_OPERATION_FAILED";
+    DataColumnStoreErrorCode["BATCH_DELETE_FAILED"] = "DATA_COLUMN_STORE_BATCH_DELETE_FAILED";
+    DataColumnStoreErrorCode["STARTUP_FAILED"] = "DATA_COLUMN_STORE_STARTUP_FAILED";
+})(DataColumnStoreErrorCode || (DataColumnStoreErrorCode = {}));
+export class DataColumnStoreError extends LodestarError {
+    constructor(type, message, cause) {
+        super(type, message);
+        this.cause = cause;
+    }
+}
+export function isFsNotFoundError(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT";
+}
+//# sourceMappingURL=errors.js.map

package/lib/db/flatFileStore/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,MAAM,CAAN,IAAY,wBAmBX;AAnBD,WAAY,wBAAwB;IAClC,2EAA+C,CAAA;IAC/C,2EAA+C,CAAA;IAC/C,qFAAyD,CAAA;IACzD,6FAAiE,CAAA;IACjE,qGAAyE,CAAA;IACzE,+EAAmD,CAAA;IACnD,yFAA6D,CAAA;IAC7D,6EAAiD,CAAA;IACjD,2FAA+D,CAAA;IAC/D,+FAAmE,CAAA;IACnE,2FAA+D,CAAA;IAC/D,yEAA6C,CAAA;IAC7C,6EAAiD,CAAA;IACjD,6EAAiD,CAAA;IACjD,+EAAmD,CAAA;IACnD,mFAAuD,CAAA;IACvD,yFAA6D,CAAA;IAC7D,+EAAmD,CAAA;AACrD,CAAC,EAnBW,wBAAwB,KAAxB,wBAAwB,QAmBnC;AAsBD,MAAM,OAAO,oBAAqB,SAAQ,aAAuC;IAC/E,YAAY,IAA8B,EAAE,OAAgB,EAAE,KAAe;QAC3E,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC9E,CAAC"}

package/lib/db/flatFileStore/flatFileStore.d.ts

@@ -0,0 +1,41 @@
+import { ChainForkConfig } from "@lodestar/config";
+import { DataColumnSidecar, RootHex, Slot } from "@lodestar/types";
+import { Logger } from "@lodestar/utils";
+import type { IFlatFileStore } from "./interface.js";
+import { type FlatFileStoreMetrics } from "./metrics.js";
+/**
+ * Filesystem storage for data columns, keyed by slot and block root.
+ * Hot and finalized data share the same layout. The top-level slot index is rebuilt on startup
+ * without inspecting the contents of slot directories.
+ * Per-slot mutation locking coordinates incremental writes, deletion, and pruning.
+ */
+export declare class FlatFileStore implements IFlatFileStore {
+    private readonly dataColumnDir;
+    private readonly config;
+    private readonly logger;
+    private readonly slotIndex;
+    private readonly mutationLocks;
+    private minRetainedSlot;
+    private metrics;
+    constructor(dataColumnDir: string, config: ChainForkConfig, logger: Logger, metrics?: FlatFileStoreMetrics | null);
+    setMetrics(metrics: FlatFileStoreMetrics | null): void;
+    init(): Promise<void>;
+    close(): Promise<void>;
+    getDataColumns(slot: Slot, blockRoot: RootHex): Promise<DataColumnSidecar[]>;
+    getDataColumnsBinary(slot: Slot, blockRoot: RootHex, indices: number[]): Promise<(Uint8Array | undefined)[] | null>;
+    putDataColumnsBinary(slot: Slot, blockRoot: RootHex, columns: {
+        index: number;
+        data: Uint8Array;
+    }[]): Promise<void>;
+    deleteMany(items: {
+        slot: Slot;
+        blockRoot: RootHex;
+    }[]): Promise<void>;
+    pruneBefore(minSlot: Slot): Promise<Slot[]>;
+    private filePath;
+    private acquireMutationLock;
+    private readFile;
+    private getColumnsBinaryUninstrumented;
+    private delete;
+}
+//# sourceMappingURL=flatFileStore.d.ts.map

package/lib/db/flatFileStore/flatFileStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatFileStore.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/flatFileStore.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAC,MAAM,EAAqB,MAAM,iBAAiB,CAAC;AAe3D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAC,KAAK,oBAAoB,EAAwD,MAAM,cAAc,CAAC;AAK9G;;;;;GAKG;AACH,qBAAa,aAAc,YAAW,cAAc;IAOhD,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IARzB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,OAAO,CAAqC;IAEpD,YACmB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,MAAM,EAC/B,OAAO,GAAE,oBAAoB,GAAG,IAAW,EAG5C;IAED,UAAU,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,GAAG,IAAI,CAIrD;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CA0B1B;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAEK,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAajF;IAEK,oBAAoB,CACxB,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,OAAO,EAClB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,GAAG,IAAI,CAAC,CAI5C;IAEK,oBAAoB,CACxB,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,OAAO,EAClB,OAAO,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAC,EAAE,GAC3C,OAAO,CAAC,IAAI,CAAC,CAiCf;IAEK,UAAU,CAAC,KAAK,EAAE;QAAC,IAAI,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAYzE;IAEK,WAAW,CAAC,OAAO,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAoBhD;IAED,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,mBAAmB;YAsBb,QAAQ;YASR,8BAA8B;YA+C9B,MAAM;CAUrB"}

package/lib/db/flatFileStore/flatFileStore.js

@@ -0,0 +1,251 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fromHex, toRootHex } from "@lodestar/utils";
+import { atomicWrite, mkdirDurable } from "./atomicWrite.js";
+import { DCOL_HEADER_SIZE, encodeDcolFile, getColumnByteRange, mergeDcolColumns, offsetTableSize, parseDcolHeader, parseDcolOffsets, readAllColumns, totalBits, } from "./dcolFormat.js";
+import { DataColumnStoreError, DataColumnStoreErrorCode, isFsNotFoundError } from "./errors.js";
+import { FlatFileStoreOperation, observeFlatFileStoreOperation } from "./metrics.js";
+import { assertValidRootHex, padSlot } from "./path.js";
+import { SlotIndex } from "./slotIndex.js";
+import { uncompress } from "./snappy.js";
+/**
+ * Filesystem storage for data columns, keyed by slot and block root.
+ * Hot and finalized data share the same layout. The top-level slot index is rebuilt on startup
+ * without inspecting the contents of slot directories.
+ * Per-slot mutation locking coordinates incremental writes, deletion, and pruning.
+ */
+export class FlatFileStore {
+    dataColumnDir;
+    config;
+    logger;
+    slotIndex = new SlotIndex();
+    mutationLocks = new Map();
+    minRetainedSlot = 0;
+    metrics = null;
+    constructor(dataColumnDir, config, logger, metrics = null) {
+        this.dataColumnDir = dataColumnDir;
+        this.config = config;
+        this.logger = logger;
+        this.setMetrics(metrics);
+    }
+    setMetrics(metrics) {
+        if (metrics === this.metrics)
+            return;
+        this.metrics = metrics;
+        metrics?.slotIndexSize.addCollect((metric) => metric.set(this.slotIndex.size));
+    }
+    async init() {
+        const endTimer = this.metrics?.startupDuration.startTimer();
+        try {
+            await mkdirDurable(this.dataColumnDir);
+            const stats = await this.slotIndex.rebuildFromDisk(this.dataColumnDir);
+            if (stats.ignoredEntries > 0) {
+                this.logger.warn("Ignored non-canonical flat file store entries", {
+                    entries: stats.ignoredEntries,
+                });
+            }
+            this.logger.info("Flat file store initialized", {
+                slots: stats.slots,
+            });
+        }
+        catch (e) {
+            this.metrics?.startupErrors.inc();
+            if (e instanceof DataColumnStoreError) {
+                throw e;
+            }
+            throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.STARTUP_FAILED }, "Flat file store initialization failed", e);
+        }
+        finally {
+            endTimer?.();
+        }
+    }
+    async close() {
+        // All operations are stateless file I/O.
+    }
+    async getDataColumns(slot, blockRoot) {
+        return observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.read, async () => {
+            const data = await this.readFile(slot, blockRoot);
+            if (!data)
+                return [];
+            this.metrics?.readBytes.inc(data.length);
+            const header = parseDcolHeader(data);
+            validateDcolHeader(header, slot, blockRoot);
+            const columns = readAllColumns(data, header);
+            const dataColumnSidecarType = this.config.getForkTypes(slot).DataColumnSidecar;
+            return columns.map((col) => dataColumnSidecarType.deserialize(col.data));
+        });
+    }
+    async getDataColumnsBinary(slot, blockRoot, indices) {
+        return observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.read, () => this.getColumnsBinaryUninstrumented(slot, blockRoot, indices));
+    }
+    async putDataColumnsBinary(slot, blockRoot, columns) {
+        const rootBytes = fromHex(blockRoot);
+        if (columns.length === 0)
+            return;
+        await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.write, async () => {
+            const release = await this.acquireMutationLock(slot);
+            try {
+                if (slot < this.minRetainedSlot) {
+                    throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.SLOT_PRUNED, slot, minRetainedSlot: this.minRetainedSlot }, `Cannot write pruned data column slot=${slot} minRetainedSlot=${this.minRetainedSlot}`);
+                }
+                const existing = await this.readFile(slot, blockRoot);
+                let fileData;
+                if (existing) {
+                    this.metrics?.readBytes.inc(existing.length);
+                    const header = parseDcolHeader(existing);
+                    validateDcolHeader(header, slot, blockRoot);
+                    fileData = mergeDcolColumns(existing, columns);
+                }
+                else {
+                    fileData = encodeDcolFile(rootBytes, slot, columns);
+                }
+                // The rename can succeed even if syncing its directory fails.
+                this.slotIndex.add(slot);
+                await atomicWrite(this.filePath(slot, blockRoot), fileData);
+                this.metrics?.writeBytes.inc(fileData.length);
+            }
+            finally {
+                release();
+            }
+        });
+    }
+    async deleteMany(items) {
+        const results = await Promise.allSettled(items.map(({ slot, blockRoot }) => this.delete(slot, blockRoot)));
+        const errors = results
+            .filter((result) => result.status === "rejected")
+            .map((result) => result.reason);
+        if (errors.length > 0) {
+            throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.BATCH_DELETE_FAILED, failures: errors.length }, "Failed to delete flat file data", new AggregateError(errors));
+        }
+    }
+    async pruneBefore(minSlot) {
+        return observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.prune, async () => {
+            this.minRetainedSlot = Math.max(this.minRetainedSlot, minSlot);
+            const slotsToPrune = new Set(this.slotIndex.getBefore(this.minRetainedSlot));
+            for (const slot of this.mutationLocks.keys()) {
+                if (slot < this.minRetainedSlot)
+                    slotsToPrune.add(slot);
+            }
+            for (const slot of slotsToPrune) {
+                const release = await this.acquireMutationLock(slot);
+                try {
+                    await fs.promises.rm(path.join(this.dataColumnDir, padSlot(slot)), { recursive: true, force: true });
+                    this.slotIndex.remove(slot);
+                    this.metrics?.prunedDirectories.inc();
+                }
+                finally {
+                    release();
+                }
+            }
+            return [...slotsToPrune].sort((a, b) => a - b);
+        });
+    }
+    filePath(slot, rootHex) {
+        assertValidRootHex(rootHex);
+        return path.join(this.dataColumnDir, padSlot(slot), `${rootHex}.dcol`);
+    }
+    acquireMutationLock(slot) {
+        let release;
+        const prev = this.mutationLocks.get(slot) ?? Promise.resolve();
+        const next = new Promise((resolve) => {
+            release = resolve;
+        });
+        const chained = prev.then(() => next);
+        this.mutationLocks.set(slot, chained);
+        return prev.then(() => {
+            let released = false;
+            return () => {
+                if (released)
+                    return;
+                released = true;
+                release();
+                if (this.mutationLocks.get(slot) === chained) {
+                    this.mutationLocks.delete(slot);
+                }
+            };
+        });
+    }
+    async readFile(slot, rootHex) {
+        try {
+            return await fs.promises.readFile(this.filePath(slot, rootHex));
+        }
+        catch (e) {
+            if (!isFsNotFoundError(e))
+                throw e;
+            return null;
+        }
+    }
+    async getColumnsBinaryUninstrumented(slot, rootHex, indices) {
+        let fd;
+        try {
+            fd = await fs.promises.open(this.filePath(slot, rootHex), "r");
+        }
+        catch (e) {
+            if (!isFsNotFoundError(e))
+                throw e;
+            return null;
+        }
+        try {
+            const headerBuf = new Uint8Array(DCOL_HEADER_SIZE);
+            await readExactly(fd, headerBuf, 0);
+            this.metrics?.readBytes.inc(headerBuf.length);
+            const header = parseDcolHeader(headerBuf);
+            validateDcolHeader(header, slot, rootHex);
+            const N = totalBits(header.bitmap);
+            const tableSize = offsetTableSize(N);
+            const offsetTable = new Uint8Array(tableSize);
+            await readExactly(fd, offsetTable, DCOL_HEADER_SIZE);
+            this.metrics?.readBytes.inc(offsetTable.length);
+            const fileSize = (await fd.stat()).size;
+            const offsets = parseDcolOffsets(offsetTable, N, fileSize - DCOL_HEADER_SIZE - tableSize);
+            const results = [];
+            for (const idx of indices) {
+                const range = getColumnByteRange(header, offsets, idx);
+                if (!range) {
+                    results.push(undefined);
+                    continue;
+                }
+                const buf = new Uint8Array(range.length);
+                await readExactly(fd, buf, range.offset);
+                this.metrics?.readBytes.inc(buf.length);
+                results.push(uncompress(buf));
+            }
+            return results;
+        }
+        finally {
+            await fd.close();
+        }
+    }
+    async delete(slot, rootHex) {
+        await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.delete, async () => {
+            const release = await this.acquireMutationLock(slot);
+            try {
+                await fs.promises.rm(this.filePath(slot, rootHex), { force: true });
+            }
+            finally {
+                release();
+            }
+        });
+    }
+}
+function validateDcolHeader(header, slot, rootHex) {
+    if (header.slot !== slot) {
+        throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.SLOT_MISMATCH, headerSlot: header.slot, pathSlot: slot }, `Dcol slot mismatch: header=${header.slot} path=${slot}`);
+    }
+    const headerRoot = toRootHex(header.blockRoot);
+    if (headerRoot !== rootHex) {
+        throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.ROOT_MISMATCH, headerRoot, pathRoot: rootHex }, `Dcol block root mismatch: header=${headerRoot} path=${rootHex}`);
+    }
+}
+async function readExactly(fd, buffer, position) {
+    let totalRead = 0;
+    while (totalRead < buffer.length) {
+        const { bytesRead } = await fd.read(buffer, totalRead, buffer.length - totalRead, position + totalRead);
+        if (bytesRead === 0) {
+            const offset = position + totalRead;
+            throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.UNEXPECTED_EOF, offset }, `Unexpected end of dcol file at offset ${offset}`);
+        }
+        totalRead += bytesRead;
+    }
+}
+//# sourceMappingURL=flatFileStore.js.map

package/lib/db/flatFileStore/flatFileStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatFileStore.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/flatFileStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAS,OAAO,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAC,WAAW,EAAE,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EACL,gBAAgB,EAEhB,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,SAAS,GACV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAC,oBAAoB,EAAE,wBAAwB,EAAE,iBAAiB,EAAC,MAAM,aAAa,CAAC;AAE9F,OAAO,EAA4B,sBAAsB,EAAE,6BAA6B,EAAC,MAAM,cAAc,CAAC;AAC9G,OAAO,EAAC,kBAAkB,EAAE,OAAO,EAAC,MAAM,WAAW,CAAC;AACtD,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,aAAa,CAAC;AAEvC;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IAOL,aAAa;IACb,MAAM;IACN,MAAM;IARR,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;IAC5B,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IACxD,eAAe,GAAS,CAAC,CAAC;IAC1B,OAAO,GAAgC,IAAI,CAAC;IAEpD,YACmB,aAAqB,EACrB,MAAuB,EACvB,MAAc,EAC/B,OAAO,GAAgC,IAAI;6BAH1B,aAAa;sBACb,MAAM;sBACN,MAAM;QAGvB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CAAC,OAAoC;QAC7C,IAAI,OAAO,KAAK,IAAI,CAAC,OAAO;YAAE,OAAO;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,UAAU,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACvE,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE;oBAChE,OAAO,EAAE,KAAK,CAAC,cAAc;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;gBAC9C,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,GAAG,EAAE,CAAC;YAClC,IAAI,CAAC,YAAY,oBAAoB,EAAE,CAAC;gBACtC,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,cAAc,EAAC,EAC/C,uCAAuC,EACvC,CAAC,CACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,QAAQ,EAAE,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,yCAAyC;IAC3C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAU,EAAE,SAAkB;QACjD,OAAO,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAClD,IAAI,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YAErB,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACrC,kBAAkB,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM,qBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAe,IAAI,CAAC,CAAC,iBAAiB,CAAC;YAE7F,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,qBAAqB,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,IAAU,EACV,SAAkB,EAClB,OAAiB;QAEjB,OAAO,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,IAAI,EAAE,GAAG,EAAE,CACnF,IAAI,CAAC,8BAA8B,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAC9D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,IAAU,EACV,SAAkB,EAClB,OAA4C;QAE5C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,MAAM,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC;gBACH,IAAI,IAAI,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;oBAChC,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAC,EACzF,wCAAwC,IAAI,oBAAoB,IAAI,CAAC,eAAe,EAAE,CACvF,CAAC;gBACJ,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBACtD,IAAI,QAAoB,CAAC;gBAEzB,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC7C,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;oBACzC,kBAAkB,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC5C,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBACtD,CAAC;gBAED,8DAA8D;gBAC9D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzB,MAAM,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC;gBAC5D,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChD,CAAC;oBAAS,CAAC;gBACT,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAyC;QACxD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,OAAO;aACnB,MAAM,CAAC,CAAC,MAAM,EAAmC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC;aACjF,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAC,EAC7E,iCAAiC,EACjC,IAAI,cAAc,CAAC,MAAM,CAAC,CAC3B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,OAAa;QAC7B,OAAO,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;YAC1F,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YAC/D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;YAC7E,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC7C,IAAI,IAAI,GAAG,IAAI,CAAC,eAAe;oBAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1D,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;gBACrD,IAAI,CAAC;oBACH,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAC;oBACnG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5B,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC;gBACxC,CAAC;wBAAS,CAAC;oBACT,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,OAAO,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,IAAU,EAAE,OAAgB;QAC3C,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC;IACzE,CAAC;IAEO,mBAAmB,CAAC,IAAU;QACpC,IAAI,OAAoB,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/D,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACzC,OAAO,GAAG,OAAO,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEtC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,OAAO,GAAG,EAAE;gBACV,IAAI,QAAQ;oBAAE,OAAO;gBACrB,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;oBAC7C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,IAAU,EAAE,OAAgB;QACjD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAC1C,IAAU,EACV,OAAgB,EAChB,OAAiB;QAEjB,IAAI,EAA0B,CAAC;QAC/B,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,gBAAgB,CAAC,CAAC;YACnD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;YAC1C,kBAAkB,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAE1C,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,WAAW,CAAC,EAAE,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;YACrD,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;YACxC,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAC,EAAE,QAAQ,GAAG,gBAAgB,GAAG,SAAS,CAAC,CAAC;YAE1F,MAAM,OAAO,GAA+B,EAAE,CAAC;YAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBACvD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACxC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,OAAgB;QAC/C,MAAM,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAC,KAAK,EAAE,IAAI,EAAC,CAAC,CAAC;YACpE,CAAC;oBAAS,CAAC;gBACT,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,kBAAkB,CAAC,MAAkB,EAAE,IAAU,EAAE,OAAgB;IAC1E,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAC,EACvF,8BAA8B,MAAM,CAAC,IAAI,SAAS,IAAI,EAAE,CACzD,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/C,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAC,EAC7E,oCAAoC,UAAU,SAAS,OAAO,EAAE,CACjE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,EAA0B,EAAE,MAAkB,EAAE,QAAgB;IACzF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,EAAC,SAAS,EAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,QAAQ,GAAG,SAAS,CAAC,CAAC;QACtG,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;YACpC,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,cAAc,EAAE,MAAM,EAAC,EACvD,yCAAyC,MAAM,EAAE,CAClD,CAAC;QACJ,CAAC;QACD,SAAS,IAAI,SAAS,CAAC;IACzB,CAAC;AACH,CAAC"}

package/lib/db/flatFileStore/index.d.ts

@@ -0,0 +1,4 @@
+export { FlatFileStore } from "./flatFileStore.js";
+export type { IFlatFileStore } from "./interface.js";
+export type { FlatFileStoreMetrics } from "./metrics.js";
+//# sourceMappingURL=index.d.ts.map

package/lib/db/flatFileStore/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACjD,YAAY,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAC,oBAAoB,EAAC,MAAM,cAAc,CAAC"}

package/lib/db/flatFileStore/index.js

@@ -0,0 +1,2 @@
+export { FlatFileStore } from "./flatFileStore.js";
+//# sourceMappingURL=index.js.map

package/lib/db/flatFileStore/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC"}

package/lib/db/flatFileStore/interface.d.ts

@@ -0,0 +1,25 @@
+import { DataColumnSidecar, RootHex, Slot } from "@lodestar/types";
+/**
+ * Flat file storage interface for data columns.
+ *
+ * No hot/cold distinction — data is always keyed by (slot, blockRoot).
+ * All filesystem operations are crash-safe via atomic writes.
+ *
+ */
+export interface IFlatFileStore {
+    init(): Promise<void>;
+    close(): Promise<void>;
+    getDataColumns(slot: Slot, blockRoot: RootHex): Promise<DataColumnSidecar[]>;
+    /** Returns null if the file is absent, and undefined entries for columns missing from an existing file. */
+    getDataColumnsBinary(slot: Slot, blockRoot: RootHex, indices: number[]): Promise<(Uint8Array | undefined)[] | null>;
+    putDataColumnsBinary(slot: Slot, blockRoot: RootHex, columns: {
+        index: number;
+        data: Uint8Array;
+    }[]): Promise<void>;
+    deleteMany(items: {
+        slot: Slot;
+        blockRoot: RootHex;
+    }[]): Promise<void>;
+    pruneBefore(slot: Slot): Promise<Slot[]>;
+}
+//# sourceMappingURL=interface.d.ts.map

package/lib/db/flatFileStore/interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC7E,2GAA2G;IAC3G,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACpH,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElH,UAAU,CAAC,KAAK,EAAE;QAAC,IAAI,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAC1C"}

package/lib/db/flatFileStore/interface.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=interface.js.map

package/lib/db/flatFileStore/interface.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interface.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/interface.ts"],"names":[],"mappings":""}

package/lib/db/flatFileStore/metrics.d.ts

@@ -0,0 +1,24 @@
+import type { Counter, GaugeExtra, Histogram } from "@lodestar/utils";
+export declare const FlatFileStoreOperation: {
+    readonly read: "read";
+    readonly write: "write";
+    readonly delete: "delete";
+    readonly prune: "prune";
+};
+export type FlatFileStoreOperation = (typeof FlatFileStoreOperation)[keyof typeof FlatFileStoreOperation];
+type OperationLabels = {
+    operation: FlatFileStoreOperation;
+};
+export type FlatFileStoreMetrics = {
+    operationDuration: Record<FlatFileStoreOperation, Histogram>;
+    operationErrors: Counter<OperationLabels>;
+    readBytes: Counter;
+    writeBytes: Counter;
+    prunedDirectories: Counter;
+    slotIndexSize: GaugeExtra;
+    startupDuration: Histogram;
+    startupErrors: Counter;
+};
+export declare function observeFlatFileStoreOperation<T>(metrics: FlatFileStoreMetrics | null, operation: FlatFileStoreOperation, fn: () => Promise<T>): Promise<T>;
+export {};
+//# sourceMappingURL=metrics.d.ts.map

package/lib/db/flatFileStore/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAGpE,eAAO,MAAM,sBAAsB;aACjC,IAAI,EAAE,MAAM;aACZ,KAAK,EAAE,OAAO;aACd,MAAM,EAAE,QAAQ;aAChB,KAAK,EAAE,OAAO;CACN,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,OAAO,sBAAsB,CAAC,CAAC;AAE1G,KAAK,eAAe,GAAG;IAAC,SAAS,EAAE,sBAAsB,CAAA;CAAC,CAAC;AAE3D,MAAM,MAAM,oBAAoB,GAAG;IACjC,iBAAiB,EAAE,MAAM,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAC;IAC7D,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAC1C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,UAAU,CAAC;IAC1B,eAAe,EAAE,SAAS,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,CAAC,EACnD,OAAO,EAAE,oBAAoB,GAAG,IAAI,EACpC,SAAS,EAAE,sBAAsB,EACjC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAkBZ"}

package/lib/db/flatFileStore/metrics.js

@@ -0,0 +1,25 @@
+import { DataColumnStoreError, DataColumnStoreErrorCode } from "./errors.js";
+export const FlatFileStoreOperation = {
+    read: "read",
+    write: "write",
+    delete: "delete",
+    prune: "prune",
+};
+export async function observeFlatFileStoreOperation(metrics, operation, fn) {
+    const labels = { operation };
+    const endTimer = metrics?.operationDuration[operation].startTimer();
+    try {
+        return await fn();
+    }
+    catch (e) {
+        metrics?.operationErrors.inc(labels);
+        if (e instanceof DataColumnStoreError) {
+            throw e;
+        }
+        throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.OPERATION_FAILED, operation }, `Flat file store ${operation} operation failed`, e);
+    }
+    finally {
+        endTimer?.();
+    }
+}
+//# sourceMappingURL=metrics.js.map

package/lib/db/flatFileStore/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,oBAAoB,EAAE,wBAAwB,EAAC,MAAM,aAAa,CAAC;AAE3E,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;IACd,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,OAAO;CACN,CAAC;AAiBX,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAoC,EACpC,SAAiC,EACjC,EAAoB;IAEpB,MAAM,MAAM,GAAG,EAAC,SAAS,EAAC,CAAC;IAC3B,MAAM,QAAQ,GAAG,OAAO,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;IACpE,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,oBAAoB,EAAE,CAAC;YACtC,MAAM,CAAC,CAAC;QACV,CAAC;QACD,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,gBAAgB,EAAE,SAAS,EAAC,EAC5D,mBAAmB,SAAS,mBAAmB,EAC/C,CAAC,CACF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,QAAQ,EAAE,EAAE,CAAC;IACf,CAAC;AACH,CAAC"}

package/lib/db/flatFileStore/path.d.ts

@@ -0,0 +1,9 @@
+import type { RootHex } from "@lodestar/types";
+export declare function isValidRootHex(rootHex: string): rootHex is RootHex;
+export declare function assertValidRootHex(rootHex: string): asserts rootHex is RootHex;
+/**
+ * Zero-pad a slot number to 12 digits for lexicographic ordering in directory names.
+ * Twelve digits covers many centuries of Ethereum; larger safe integers remain untruncated.
+ */
+export declare function padSlot(slot: number): string;
+//# sourceMappingURL=path.d.ts.map

package/lib/db/flatFileStore/path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"path.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/path.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAK7C,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,IAAI,OAAO,CAElE;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAO9E;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAS5C"}

package/lib/db/flatFileStore/path.js

@@ -0,0 +1,21 @@
+import { DataColumnStoreError, DataColumnStoreErrorCode } from "./errors.js";
+const ROOT_HEX_PATTERN = /^0x[0-9a-f]{64}$/;
+export function isValidRootHex(rootHex) {
+    return ROOT_HEX_PATTERN.test(rootHex);
+}
+export function assertValidRootHex(rootHex) {
+    if (!isValidRootHex(rootHex)) {
+        throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.INVALID_ROOT, root: rootHex }, `Invalid flat file root: ${rootHex}`);
+    }
+}
+/**
+ * Zero-pad a slot number to 12 digits for lexicographic ordering in directory names.
+ * Twelve digits covers many centuries of Ethereum; larger safe integers remain untruncated.
+ */
+export function padSlot(slot) {
+    if (!Number.isSafeInteger(slot) || slot < 0) {
+        throw new DataColumnStoreError({ code: DataColumnStoreErrorCode.INVALID_SLOT, slot }, `Invalid flat file slot: ${slot}`);
+    }
+    return String(slot).padStart(12, "0");
+}
+//# sourceMappingURL=path.js.map

package/lib/db/flatFileStore/path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"path.js","sourceRoot":"","sources":["../../../src/db/flatFileStore/path.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,oBAAoB,EAAE,wBAAwB,EAAC,MAAM,aAAa,CAAC;AAE3E,MAAM,gBAAgB,GAAG,kBAAkB,CAAC;AAE5C,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,EAAC,EAC5D,2BAA2B,OAAO,EAAE,CACrC,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,oBAAoB,CAC5B,EAAC,IAAI,EAAE,wBAAwB,CAAC,YAAY,EAAE,IAAI,EAAC,EACnD,2BAA2B,IAAI,EAAE,CAClC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;AACxC,CAAC"}

package/lib/db/flatFileStore/slotIndex.d.ts

@@ -0,0 +1,14 @@
+import { Slot } from "@lodestar/types";
+export type SlotIndexRebuildStats = {
+    slots: number;
+    ignoredEntries: number;
+};
+export declare class SlotIndex {
+    private readonly slots;
+    get size(): number;
+    add(slot: Slot): void;
+    remove(slot: Slot): void;
+    getBefore(minSlot: Slot): Slot[];
+    rebuildFromDisk(columnsDir: string): Promise<SlotIndexRebuildStats>;
+}
+//# sourceMappingURL=slotIndex.d.ts.map

package/lib/db/flatFileStore/slotIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slotIndex.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/slotIndex.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAIrC,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IAEzC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAEpB;IAED,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAEvB;IAED,SAAS,CAAC,OAAO,EAAE,IAAI,GAAG,IAAI,EAAE,CAM/B;IAEK,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAqBxE;CACF"}