@wxn0brp/db-storage-bin 0.0.2

package/src/bin/head.ts

@@ -0,0 +1,183 @@
+import { BinManager, CollectionMeta } from ".";
+import { getFileCrc } from "../crc32";
+import { _log } from "../log";
+import { findFreeSlot } from "./data";
+import { HEADER_SIZE, VERSION } from "./static";
+import { detectCollisions, pushToFreeList, roundUpCapacity, writeData } from "./utils";
+
+export interface Block {
+    offset: number;
+    capacity: number;
+};
+
+export interface FileMeta {
+    collections: CollectionMeta[];
+    freeList: Block[];
+    fileSize: number;
+    payloadLength: number;
+    payloadOffset: number;
+    blockSize: number;
+}
+
+export async function openFile(cmp: BinManager) {
+    const { fd, options } = cmp;
+    const stats = await fd.stat();
+    const fileSize = stats.size;
+    await _log(2, "File size:", fileSize);
+
+    const meta: FileMeta = {
+        collections: [],
+        freeList: [],
+        fileSize,
+        payloadLength: 0,
+        payloadOffset: 0,
+        blockSize: options.preferredSize ?? 256,
+    }
+    cmp.meta = meta;
+
+    if (fileSize < HEADER_SIZE) {
+        await _log(2, "Initializing new file header");
+        await saveHeaderAndPayload(cmp);
+        await _log(6, "Header initialized with size:", HEADER_SIZE);
+        return meta;
+    }
+
+    const headerBuf = Buffer.alloc(HEADER_SIZE);
+    await fd.read(headerBuf, 0, HEADER_SIZE, 0);
+    await _log(6, "Header read from file");
+
+    const version = headerBuf.readUInt32LE(0);
+    if (version !== VERSION) {
+        await _log(6, "err", `Unsupported file version: ${version}`);
+        throw new Error(`Unsupported file version ${version}`);
+    }
+    await _log(2, "File version:", version);
+
+    const payloadLength = headerBuf.readUInt32LE(4);
+    meta.payloadLength = payloadLength;
+    await _log(6, "Payload length:", payloadLength);
+
+    const payloadOffset = headerBuf.readUInt32LE(8);
+    meta.payloadOffset = payloadOffset;
+    await _log(6, "Payload offset:", payloadOffset);
+
+    const blockSize = headerBuf.readUInt32LE(12);
+    meta.blockSize = blockSize;
+    await _log(2, "Block size:", blockSize);
+
+    if (options.crc) {
+        const { computedCrc, storedCrc } = await getFileCrc(fd);
+        const validCrc = computedCrc === storedCrc || storedCrc === 0;
+        await _log(2, "CRC:", computedCrc, "Needed CRC:", storedCrc, "Valid:", validCrc);
+        if (storedCrc === 0) {
+            await _log(1, "Warning: CRC is zero, CRC will not be checked");
+        }
+        if (!validCrc) {
+            await _log(0, "err", "Invalid CRC");
+            if (options.crc === 2)
+                throw new Error("Invalid CRC");
+        }
+    }
+
+    if (payloadOffset + payloadLength > fileSize - HEADER_SIZE) {
+        await _log(6, "err", "Invalid payload length");
+        throw new Error("Invalid payload length");
+    }
+
+    if (payloadLength === 0) {
+        await _log(6, "Empty payload, initializing collections and freeList");
+        return meta;
+    }
+
+    await readHeaderPayload(cmp);
+    return meta;
+}
+
+export async function readHeaderPayload(cmp: BinManager) {
+    const { fd, meta } = cmp;
+    const { payloadLength, payloadOffset } = meta;
+
+    const payloadBuf = Buffer.alloc(payloadLength);
+    const { bytesRead } = await fd.read(payloadBuf, 0, payloadLength, HEADER_SIZE + payloadOffset);
+    await _log(6, `Payload header read, bytesRead: ${bytesRead}`);
+
+    if (bytesRead < payloadLength) {
+        await _log(6, "err", `Incomplete payload header read: expected ${payloadLength} bytes, got ${bytesRead}`);
+        throw new Error(`Incomplete payload header read: expected ${payloadLength} bytes, got ${bytesRead}`);
+    }
+
+    const obj = await cmp.options.format.decode(payloadBuf) as {
+        c: [string, number, number][];
+        f: [number, number][];
+    };
+
+    meta.collections = (obj.c || []).map(([name, offset, capacity]) => ({ name, offset, capacity }));
+    meta.freeList = (obj.f || []).map(([offset, capacity]) => ({ offset, capacity }));
+
+    await _log(6, "Collections and freeList loaded", meta);
+}
+
+export function getHeaderPayload(meta: FileMeta) {
+    return {
+        c: meta.collections.map(({ name, offset, capacity }) => ([name, offset, capacity])),
+        f: meta.freeList.map(({ offset, capacity }) => [offset, capacity]),
+    };
+}
+
+export async function saveHeaderAndPayload(cmp: BinManager, recursion = false) {
+    const { fd, meta, options } = cmp;
+    if (!fd) throw new Error("File not open");
+
+    const { collections, freeList, fileSize } = meta;
+    await _log(6, "Saving header payload:", collections, freeList);
+
+    const payloadObj = getHeaderPayload(meta);
+
+    const payloadBuf = Buffer.from(await cmp.options.format.encode(payloadObj));
+    if (payloadBuf.length > 64 * 1024) {
+        console.error("Header payload too large");
+        throw new Error("Header payload too large");
+    }
+
+    await _log(6, "Header payload length:", payloadBuf.length);
+
+    const headerBuf = Buffer.alloc(HEADER_SIZE);
+    headerBuf.writeUInt32LE(VERSION, 0);
+    headerBuf.writeUInt32LE(payloadBuf.length, 4);
+    headerBuf.writeUInt32LE(meta.payloadOffset, 8);
+    headerBuf.writeUInt32LE(meta.blockSize, 12);
+    meta.payloadLength = payloadBuf.length;
+
+    if (options.crc) {
+        const { computedCrc: crc } = await getFileCrc(fd);
+        headerBuf.writeUInt32LE(crc, 16);
+    }
+
+    await _log(6, "Writing header:", headerBuf.toString("hex"));
+
+    // Write header
+    await fd.write(headerBuf, 0, HEADER_SIZE, 0);
+    // Write payload
+    const roundPayload = roundUpCapacity(meta, payloadBuf.length);
+
+    if (detectCollisions(meta, HEADER_SIZE + meta.payloadOffset, roundPayload)) {
+        await _log(2, "Collision detected");
+        const slot = !recursion && await findFreeSlot(cmp, roundPayload);
+        if (slot) {
+            meta.payloadOffset = slot.offset - HEADER_SIZE;
+        } else {
+            meta.payloadOffset = meta.fileSize - HEADER_SIZE;
+            meta.fileSize += roundPayload;
+        }
+        pushToFreeList(meta, meta.payloadOffset, roundPayload);
+
+        return await saveHeaderAndPayload(cmp, true);
+    }
+
+    await writeData(fd, HEADER_SIZE + meta.payloadOffset, payloadBuf, roundPayload);
+
+    await _log(6, "Payload written");
+
+    // Update file size if header + payload bigger
+    meta.fileSize = Math.max(fileSize, HEADER_SIZE + roundPayload);
+}

package/src/bin/index.ts

@@ -0,0 +1,109 @@
+import * as msgpack from "@msgpack/msgpack";
+import { access, constants, FileHandle, open } from "fs/promises";
+import { getFileCrc } from "../crc32";
+import { _log } from "../log";
+import { readLogic, writeLogic } from "./data";
+import { FileMeta, openFile } from "./head";
+import { optimize } from "./optimize";
+import { removeCollection } from "./rm";
+
+async function safeOpen(path: string) {
+    try {
+        await access(path, constants.F_OK);
+        return await open(path, "r+");
+    } catch {
+        _log(1, "Creating new file");
+        return await open(path, "w+");
+    }
+}
+
+export interface CollectionMeta {
+    name: string;
+    offset: number;
+    capacity: number;
+}
+
+export interface Options {
+    preferredSize: number;
+    /**
+     * 0 - crc off
+     * 1 - warn if error
+     * 2 - throw if error
+     */
+    crc: number;
+    overwriteRemovedCollection: boolean;
+    format: {
+        encode(data: any): Promise<Parameters<typeof Buffer.from>[0]>;
+        decode(data: Buffer): Promise<any>;
+    }
+}
+
+export class BinManager {
+    public fd: null | FileHandle = null;
+    public meta: FileMeta;
+    public options: Options;
+
+    /**
+     * Constructs a new BinManager instance.
+     * @param path - File path.
+     * @param [preferredSize=512] - The preferred block size for the database. Must be a positive number (preferredSize > 0)
+     * @throws If the path is not provided, or the preferred size is
+     * not a positive number.
+     */
+    constructor(public path: string, options?: Partial<Options>) {
+        if (!path) throw new Error("Path not provided");
+
+        this.options = {
+            preferredSize: 512,
+            crc: 2,
+            overwriteRemovedCollection: false,
+            format: {
+                encode: async (data: any) => msgpack.encode(data),
+                decode: async (data: Buffer) => msgpack.decode(data)
+            },
+            ...options
+        }
+
+        if (!this.options.preferredSize || this.options.preferredSize <= 0) throw new Error("Preferred size not provided");
+    }
+
+    async open() {
+        this.fd = await safeOpen(this.path);
+        await openFile(this);
+    }
+
+    async close() {
+        if (this.fd) {
+            const buff = Buffer.alloc(8);
+            if (this.options.crc) {
+                const { computedCrc: crc } = await getFileCrc(this.fd);
+                buff.writeUInt32LE(crc, 0);
+            } else {
+                buff.fill(0, 0, 8);
+            }
+            await this.fd.write(buff, 0, 8, 16);
+            await this.fd.close();
+            this.fd = null;
+        }
+    }
+
+    async write(collection: string, data: object[]) {
+        if (!this.fd) throw new Error("File not open");
+        await writeLogic(this, collection, data);
+    }
+
+    async read(collection: string) {
+        if (!this.fd) throw new Error("File not open");
+        return await readLogic(this, collection);
+    }
+
+    async optimize() {
+        if (!this.fd) throw new Error("File not open");
+        await optimize(this);
+    }
+
+    async removeCollection(collection: string) {
+        if (!this.fd) throw new Error("File not open");
+        await removeCollection(this, collection);
+    }
+}

package/src/bin/optimize.ts

@@ -0,0 +1,46 @@
+import { unlink } from "fs/promises";
+import { BinManager } from ".";
+import { saveHeaderAndPayload } from "./head";
+import { HEADER_SIZE } from "./static";
+import { readData, roundUpCapacity, writeData } from "./utils";
+import { _log } from "../log";
+
+export async function optimize(cmp: BinManager) {
+    await _log(3, "Starting database optimization");
+    const collections = cmp.meta.collections;
+
+    const allData = new Map<string, Buffer>();
+    for (const { name, offset } of collections) {
+        await _log(6, "Reading collection for optimization:", name);
+        const len = await readData(cmp.fd, offset, 4);
+        const data = await readData(cmp.fd, offset + 4, len.readInt32LE(0));
+        allData.set(name, data);
+    }
+
+    await _log(5, "Closing file for optimization");
+    await cmp.close();
+    await _log(6, "Deleting old database file for optimization");
+    await unlink(cmp.path);
+    await new Promise(resolve => setTimeout(resolve, 100));
+    await _log(5, "Re-opening database file for optimization");
+    await cmp.open();
+
+    let offset = roundUpCapacity(cmp.meta, cmp.meta.payloadLength + HEADER_SIZE) + cmp.meta.blockSize;
+    for (const [collection, data] of allData) {
+        await _log(6, "Writing optimized collection:", collection);
+        const len = roundUpCapacity(cmp.meta, data.length + 4);
+        const buf = Buffer.alloc(4);
+        buf.writeInt32LE(data.length, 0);
+        await writeData(cmp.fd, offset, buf, 4);
+        await writeData(cmp.fd, offset + 4, data, len);
+        cmp.meta.collections.push({
+            name: collection,
+            offset,
+            capacity: len
+        });
+        offset += len;
+    }
+
+    await saveHeaderAndPayload(cmp);
+    await _log(3, "Database optimization complete");
+}

package/src/bin/rm.ts

@@ -0,0 +1,30 @@
+import { BinManager } from ".";
+import { findCollection } from "./data";
+import { saveHeaderAndPayload } from "./head";
+import { writeData } from "./utils";
+
+export async function removeCollection(cmp: BinManager, collection: string) {
+    const { meta, fd, options } = cmp;
+    const collectionMeta = findCollection(cmp, collection);
+    if (!collectionMeta) throw new Error("Collection not found");
+
+    if (meta.collections.length === 1) {
+        meta.collections = [];
+        meta.freeList = [];
+        await fd.truncate(0);
+        await saveHeaderAndPayload(cmp);
+        return;
+    }
+
+    meta.collections.splice(meta.collections.findIndex(c => c.name === collection), 1);
+    meta.freeList.push({
+        offset: collectionMeta.offset,
+        capacity: collectionMeta.capacity
+    });
+
+    if (options.overwriteRemovedCollection) {
+        await writeData(fd, collectionMeta.offset, Buffer.alloc(collectionMeta.capacity), collectionMeta.capacity);
+    }
+
+    await saveHeaderAndPayload(cmp);
+}

package/src/bin/static.ts

@@ -0,0 +1,2 @@
+export const HEADER_SIZE = 64;
+export const VERSION = 1;

package/src/bin/utils.ts

@@ -0,0 +1,90 @@
+import { FileHandle } from "fs/promises";
+import { _log } from "../log";
+import { Block, FileMeta } from "./head";
+
+export function roundUpCapacity(result: FileMeta, size: number) {
+    return Math.ceil(size / result.blockSize) * result.blockSize;
+}
+
+export async function writeData(fd: FileHandle, offset: number, data: Buffer, capacity: number) {
+    if (!fd) throw new Error("File not open");
+    if (data.length > capacity) throw new Error("Data size exceeds capacity");
+
+    await _log(6, "Writing data at offset:", offset, "length:", data.length, "capacity:", capacity);
+
+    const { bytesWritten } = await fd.write(data, 0, data.length, offset);
+    await _log(5, "Bytes written:", bytesWritten);
+
+    if (data.length < capacity) {
+        const pad = Buffer.alloc(capacity - data.length, 0);
+        const padStart = offset + data.length;
+        await _log(6, "Padding with zeros:", pad.length, "at offset:", padStart);
+        const { bytesWritten: padBytesWritten } = await fd.write(pad, 0, pad.length, padStart);
+        await _log(6, "Bytes written:", padBytesWritten);
+    }
+
+    await _log(6, "Data written");
+}
+
+export async function readData(fd: FileHandle, offset: number, length: number): Promise<Buffer> {
+    if (!fd) throw new Error("File not open");
+
+    await _log(6, "Reading data from offset:", offset, "length:", length);
+
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await fd.read(buf, 0, length, offset);
+
+    await _log(5, "Bytes read:", bytesRead);
+
+    return buf;
+}
+
+export function optimizeFreeList(blocks: Block[]): Block[] {
+    if (blocks.length <= 1) return blocks;
+
+    const sorted = [...blocks].sort((a, b) => a.offset - b.offset);
+
+    const merged: Block[] = [];
+    let current = sorted[0];
+
+    for (let i = 1; i < sorted.length; i++) {
+        const next = sorted[i];
+
+        if (current.offset + current.capacity === next.offset) {
+            current = {
+                offset: current.offset,
+                capacity: current.capacity + next.capacity
+            };
+        } else {
+            merged.push(current);
+            current = next;
+        }
+    }
+
+    merged.push(current);
+
+    return merged;
+}
+
+function checkCollection(start1: number, end1: number, start2: number, end2: number) {
+    _log(6, "Checking collection:", start1, end1, start2, end2);
+    return start1 < end2 && start2 < end1;
+}
+
+export function detectCollisions(result: FileMeta, start: number, size: number, skip: string[] = []) {
+    for (const { name, offset, capacity } of result.collections) {
+        if (skip.includes(name)) continue;
+        if (checkCollection(offset, offset + capacity, start, start + size))
+            return true;
+    }
+
+    return false;
+}
+
+export function pushToFreeList(result: FileMeta, offset: number, len: number) {
+    result.freeList.push({
+        offset,
+        capacity: roundUpCapacity(result, len),
+    });
+    result.freeList = optimizeFreeList(result.freeList);
+}

package/src/crc32.ts

@@ -0,0 +1,44 @@
+import { FileHandle } from "fs/promises";
+import { HEADER_SIZE } from "./bin/static";
+
+const CRC32_TABLE = new Uint32Array(256);
+
+for (let i = 0; i < 256; i++) {
+    let crc = i;
+    for (let j = 0; j < 8; j++) {
+        crc = (crc & 1) ? (0xEDB88320 ^ (crc >>> 1)) : (crc >>> 1);
+    }
+    CRC32_TABLE[i] = crc >>> 0;
+}
+
+export function crc32(buf: Uint8Array | string, seed = 0xFFFFFFFF): number {
+    if (typeof buf === "string") {
+        buf = new TextEncoder().encode(buf);
+    }
+
+    let crc = seed ^ 0xFFFFFFFF;
+    for (let i = 0; i < buf.length; i++) {
+        const byte = buf[i];
+        crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ byte) & 0xFF];
+    }
+    return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+export async function getFileCrc(fd: FileHandle, short = false) {
+    const { size } = await fd.stat();
+    if (size < HEADER_SIZE) return { storedCrc: 0, computedCrc: 0 };
+
+    const buffer = Buffer.alloc(size);
+    await fd.read(buffer, 0, size, 0);
+
+    const storedCrc = buffer.readUInt32LE(16);
+    if (short && storedCrc === 0) return { storedCrc: 0, computedCrc: 0 };
+
+    buffer.fill(0, 16, 20);
+    const computedCrc = crc32(buffer);
+
+    return {
+        storedCrc,
+        computedCrc
+    }
+}

package/src/index.ts

@@ -0,0 +1,20 @@
+import { ValtheraClass } from "@wxn0brp/db-core";
+import { BinFileAction } from "./actions";
+import { BinManager, Options } from "./bin";
+
+export * from "./actions";
+export * from "./bin";
+
+export async function createBinValthera(path: string, opts: Partial<Options> = {}, init = true) {
+    const mgr = new BinManager(path, opts);
+    const actions = new BinFileAction(mgr);
+    const db = new ValtheraClass({ dbAction: actions });
+
+    if (init) await actions.init();
+
+    return {
+        db,
+        actions,
+        mgr,
+    }
+}

package/src/log.ts

@@ -0,0 +1,13 @@
+const dir = process.cwd() + "/";
+export async function _log(level: number, ...data: any[]) {
+    const logLevel = parseInt(process.env.VDB_BIN_LOG_LEVEL || '0', 10);
+    if (logLevel < level) return;
+
+    let line = new Error().stack.split('\n')[3].trim();
+    let path = line.slice(line.indexOf("(")).replace(dir, "").replace("(", "").replace(")", "");
+    const at = line.slice(3, line.indexOf("(") - 1);
+
+    if (path.length < 2) path = line.replace(dir, "").replace("at ", ""); // if path is 2 (callback):
+
+    console.log(`[${level}] ` + "\x1b[36m" + path + ":", "\x1b[33m" + at + "\x1b[0m", ...data);
+}

package/suglite.json

@@ -0,0 +1,10 @@
+{
+    "cmd": "NODE_ENV=development tsc --noEmit && yarn bun",
+    "watch": [
+        "src"
+    ],
+    "restart_cmd": "clear",
+    "events": {
+        "rs": "clear"
+    }
+}

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+    "compilerOptions": {
+        "module": "ES2022",
+        "target": "ES2022",
+        "moduleResolution": "bundler",
+        "paths": {},
+        "esModuleInterop": true,
+        "skipLibCheck": true,
+        "outDir": "./dist",
+        "declaration": true
+    },
+    "include": [
+       "./src"
+    ],
+    "exclude": [
+        "node_modules"
+    ],
+    "tsc-alias": {
+        "resolveFullPaths": true,
+        "verbose": false
+    }
+}