@wxn0brp/db-storage-bin 0.0.2

package/dist/bin/optimize.js

@@ -0,0 +1,40 @@
+import { unlink } from "fs/promises";
+import { saveHeaderAndPayload } from "./head.js";
+import { HEADER_SIZE } from "./static.js";
+import { readData, roundUpCapacity, writeData } from "./utils.js";
+import { _log } from "../log.js";
+export async function optimize(cmp) {
+    await _log(3, "Starting database optimization");
+    const collections = cmp.meta.collections;
+    const allData = new Map();
+    for (const { name, offset } of collections) {
+        await _log(6, "Reading collection for optimization:", name);
+        const len = await readData(cmp.fd, offset, 4);
+        const data = await readData(cmp.fd, offset + 4, len.readInt32LE(0));
+        allData.set(name, data);
+    }
+    await _log(5, "Closing file for optimization");
+    await cmp.close();
+    await _log(6, "Deleting old database file for optimization");
+    await unlink(cmp.path);
+    await new Promise(resolve => setTimeout(resolve, 100));
+    await _log(5, "Re-opening database file for optimization");
+    await cmp.open();
+    let offset = roundUpCapacity(cmp.meta, cmp.meta.payloadLength + HEADER_SIZE) + cmp.meta.blockSize;
+    for (const [collection, data] of allData) {
+        await _log(6, "Writing optimized collection:", collection);
+        const len = roundUpCapacity(cmp.meta, data.length + 4);
+        const buf = Buffer.alloc(4);
+        buf.writeInt32LE(data.length, 0);
+        await writeData(cmp.fd, offset, buf, 4);
+        await writeData(cmp.fd, offset + 4, data, len);
+        cmp.meta.collections.push({
+            name: collection,
+            offset,
+            capacity: len
+        });
+        offset += len;
+    }
+    await saveHeaderAndPayload(cmp);
+    await _log(3, "Database optimization complete");
+}

package/dist/bin/rm.d.ts

@@ -0,0 +1,2 @@
+import { BinManager } from "./index.js";
+export declare function removeCollection(cmp: BinManager, collection: string): Promise<void>;

package/dist/bin/rm.js

@@ -0,0 +1,25 @@
+import { findCollection } from "./data.js";
+import { saveHeaderAndPayload } from "./head.js";
+import { writeData } from "./utils.js";
+export async function removeCollection(cmp, collection) {
+    const { meta, fd, options } = cmp;
+    const collectionMeta = findCollection(cmp, collection);
+    if (!collectionMeta)
+        throw new Error("Collection not found");
+    if (meta.collections.length === 1) {
+        meta.collections = [];
+        meta.freeList = [];
+        await fd.truncate(0);
+        await saveHeaderAndPayload(cmp);
+        return;
+    }
+    meta.collections.splice(meta.collections.findIndex(c => c.name === collection), 1);
+    meta.freeList.push({
+        offset: collectionMeta.offset,
+        capacity: collectionMeta.capacity
+    });
+    if (options.overwriteRemovedCollection) {
+        await writeData(fd, collectionMeta.offset, Buffer.alloc(collectionMeta.capacity), collectionMeta.capacity);
+    }
+    await saveHeaderAndPayload(cmp);
+}

package/dist/bin/static.d.ts

@@ -0,0 +1,2 @@
+export declare const HEADER_SIZE = 64;
+export declare const VERSION = 1;

package/dist/bin/static.js

@@ -0,0 +1,2 @@
+export const HEADER_SIZE = 64;
+export const VERSION = 1;

package/dist/bin/utils.d.ts

@@ -0,0 +1,8 @@
+import { FileHandle } from "fs/promises";
+import { Block, FileMeta } from "./head.js";
+export declare function roundUpCapacity(result: FileMeta, size: number): number;
+export declare function writeData(fd: FileHandle, offset: number, data: Buffer, capacity: number): Promise<void>;
+export declare function readData(fd: FileHandle, offset: number, length: number): Promise<Buffer>;
+export declare function optimizeFreeList(blocks: Block[]): Block[];
+export declare function detectCollisions(result: FileMeta, start: number, size: number, skip?: string[]): boolean;
+export declare function pushToFreeList(result: FileMeta, offset: number, len: number): void;

package/dist/bin/utils.js

@@ -0,0 +1,72 @@
+import { _log } from "../log.js";
+export function roundUpCapacity(result, size) {
+    return Math.ceil(size / result.blockSize) * result.blockSize;
+}
+export async function writeData(fd, offset, data, capacity) {
+    if (!fd)
+        throw new Error("File not open");
+    if (data.length > capacity)
+        throw new Error("Data size exceeds capacity");
+    await _log(6, "Writing data at offset:", offset, "length:", data.length, "capacity:", capacity);
+    const { bytesWritten } = await fd.write(data, 0, data.length, offset);
+    await _log(5, "Bytes written:", bytesWritten);
+    if (data.length < capacity) {
+        const pad = Buffer.alloc(capacity - data.length, 0);
+        const padStart = offset + data.length;
+        await _log(6, "Padding with zeros:", pad.length, "at offset:", padStart);
+        const { bytesWritten: padBytesWritten } = await fd.write(pad, 0, pad.length, padStart);
+        await _log(6, "Bytes written:", padBytesWritten);
+    }
+    await _log(6, "Data written");
+}
+export async function readData(fd, offset, length) {
+    if (!fd)
+        throw new Error("File not open");
+    await _log(6, "Reading data from offset:", offset, "length:", length);
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await fd.read(buf, 0, length, offset);
+    await _log(5, "Bytes read:", bytesRead);
+    return buf;
+}
+export function optimizeFreeList(blocks) {
+    if (blocks.length <= 1)
+        return blocks;
+    const sorted = [...blocks].sort((a, b) => a.offset - b.offset);
+    const merged = [];
+    let current = sorted[0];
+    for (let i = 1; i < sorted.length; i++) {
+        const next = sorted[i];
+        if (current.offset + current.capacity === next.offset) {
+            current = {
+                offset: current.offset,
+                capacity: current.capacity + next.capacity
+            };
+        }
+        else {
+            merged.push(current);
+            current = next;
+        }
+    }
+    merged.push(current);
+    return merged;
+}
+function checkCollection(start1, end1, start2, end2) {
+    _log(6, "Checking collection:", start1, end1, start2, end2);
+    return start1 < end2 && start2 < end1;
+}
+export function detectCollisions(result, start, size, skip = []) {
+    for (const { name, offset, capacity } of result.collections) {
+        if (skip.includes(name))
+            continue;
+        if (checkCollection(offset, offset + capacity, start, start + size))
+            return true;
+    }
+    return false;
+}
+export function pushToFreeList(result, offset, len) {
+    result.freeList.push({
+        offset,
+        capacity: roundUpCapacity(result, len),
+    });
+    result.freeList = optimizeFreeList(result.freeList);
+}

package/dist/crc32.d.ts

@@ -0,0 +1,6 @@
+import { FileHandle } from "fs/promises";
+export declare function crc32(buf: Uint8Array | string, seed?: number): number;
+export declare function getFileCrc(fd: FileHandle, short?: boolean): Promise<{
+    storedCrc: number;
+    computedCrc: number;
+}>;

package/dist/crc32.js

@@ -0,0 +1,36 @@
+import { HEADER_SIZE } from "./bin/static.js";
+const CRC32_TABLE = new Uint32Array(256);
+for (let i = 0; i < 256; i++) {
+    let crc = i;
+    for (let j = 0; j < 8; j++) {
+        crc = (crc & 1) ? (0xEDB88320 ^ (crc >>> 1)) : (crc >>> 1);
+    }
+    CRC32_TABLE[i] = crc >>> 0;
+}
+export function crc32(buf, seed = 0xFFFFFFFF) {
+    if (typeof buf === "string") {
+        buf = new TextEncoder().encode(buf);
+    }
+    let crc = seed ^ 0xFFFFFFFF;
+    for (let i = 0; i < buf.length; i++) {
+        const byte = buf[i];
+        crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ byte) & 0xFF];
+    }
+    return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+export async function getFileCrc(fd, short = false) {
+    const { size } = await fd.stat();
+    if (size < HEADER_SIZE)
+        return { storedCrc: 0, computedCrc: 0 };
+    const buffer = Buffer.alloc(size);
+    await fd.read(buffer, 0, size, 0);
+    const storedCrc = buffer.readUInt32LE(16);
+    if (short && storedCrc === 0)
+        return { storedCrc: 0, computedCrc: 0 };
+    buffer.fill(0, 16, 20);
+    const computedCrc = crc32(buffer);
+    return {
+        storedCrc,
+        computedCrc
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+import { ValtheraClass } from "@wxn0brp/db-core";
+import { BinFileAction } from "./actions.js";
+import { BinManager, Options } from "./bin/index.js";
+export * from "./actions.js";
+export * from "./bin/index.js";
+export declare function createBinValthera(path: string, opts?: Partial<Options>, init?: boolean): Promise<{
+    db: ValtheraClass;
+    actions: BinFileAction;
+    mgr: BinManager;
+}>;

package/dist/index.js

@@ -0,0 +1,17 @@
+import { ValtheraClass } from "@wxn0brp/db-core";
+import { BinFileAction } from "./actions.js";
+import { BinManager } from "./bin/index.js";
+export * from "./actions.js";
+export * from "./bin/index.js";
+export async function createBinValthera(path, opts = {}, init = true) {
+    const mgr = new BinManager(path, opts);
+    const actions = new BinFileAction(mgr);
+    const db = new ValtheraClass({ dbAction: actions });
+    if (init)
+        await actions.init();
+    return {
+        db,
+        actions,
+        mgr,
+    };
+}

package/dist/log.d.ts

@@ -0,0 +1 @@
+export declare function _log(level: number, ...data: any[]): Promise<void>;

package/dist/log.js

@@ -0,0 +1,12 @@
+const dir = process.cwd() + "/";
+export async function _log(level, ...data) {
+    const logLevel = parseInt(process.env.VDB_BIN_LOG_LEVEL || '0', 10);
+    if (logLevel < level)
+        return;
+    let line = new Error().stack.split('\n')[3].trim();
+    let path = line.slice(line.indexOf("(")).replace(dir, "").replace("(", "").replace(")", "");
+    const at = line.slice(3, line.indexOf("(") - 1);
+    if (path.length < 2)
+        path = line.replace(dir, "").replace("at ", ""); // if path is 2 (callback):
+    console.log(`[${level}] ` + "\x1b[36m" + path + ":", "\x1b[33m" + at + "\x1b[0m", ...data);
+}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@wxn0brp/db-storage-bin",
+  "version": "0.0.2",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "wxn0brP",
+  "license": "MIT",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wxn0brp/ValtheraDB-storage-bin.git"
+  },
+  "homepage": "https://github.com/wxn0brp/ValtheraDB-storage-bin",
+  "devDependencies": {
+    "@types/node": "^24.3.0",
+    "@wxn0brp/db-core": "^0.1.5",
+    "tsc-alias": "^1.8.10",
+    "typescript": "^5.9.2"
+  },
+  "dependencies": {
+    "@msgpack/msgpack": "^3.1.2"
+  },
+  "peerDependencies": {
+    "@wxn0brp/db-core": ">=0.1.5"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js",
+      "import": "./dist/index.js"
+    },
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "default": "./dist/*.js",
+      "import": "./dist/*.js"
+    }
+  }
+}

package/src/actions.ts

@@ -0,0 +1,122 @@
+import { CustomFileCpu, genId } from "@wxn0brp/db-core";
+import dbActionBase from "@wxn0brp/db-core/base/actions";
+import Data from "@wxn0brp/db-core/types/data";
+import FileCpu from "@wxn0brp/db-core/types/fileCpu";
+import { DbOpts } from "@wxn0brp/db-core/types/options";
+import { VQuery } from "@wxn0brp/db-core/types/query";
+import { findUtil } from "@wxn0brp/db-core/utils/action";
+import { BinManager } from "./bin";
+
+export class BinFileAction extends dbActionBase {
+    folder: string;
+    options: DbOpts;
+    fileCpu: FileCpu;
+
+    /**
+     * Creates a new instance of dbActionC.
+     * @constructor
+     * @param folder - The folder where database files are stored.
+     * @param options - The options object.
+     */
+    constructor(private mgr: BinManager) {
+        super();
+        this.fileCpu = new CustomFileCpu(this.mgr.read.bind(this.mgr), this.mgr.write.bind(this.mgr));
+    }
+
+    async init() {
+        await this.mgr.open();
+    }
+
+    /**
+     * Get a list of available databases in the specified folder.
+     */
+    async getCollections() {
+        const collections = this.mgr.meta.collections.map(c => c.name);
+        return collections;
+    }
+
+    /**
+     * Check and create the specified collection if it doesn't exist.
+     */
+    async ensureCollection({ collection }: VQuery) {
+        if (await this.issetCollection(arguments[0])) return;
+        await this.mgr.write(collection, []);
+        return true;
+    }
+
+    /**
+     * Check if a collection exists.
+     */
+    async issetCollection({ collection }: VQuery) {
+        const collections = await this.getCollections();
+        return collections.includes(collection);
+    }
+
+    /**
+     * Add a new entry to the specified database.
+     */
+    async add({ collection, data, id_gen = true }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+
+        if (id_gen) data._id = data._id || genId();
+        await this.fileCpu.add(collection, data);
+        return data;
+    }
+
+    /**
+     * Find entries in the specified database based on search criteria.
+     */
+    async find(query: VQuery) {
+        await this.ensureCollection(query);
+        return await findUtil(query, this.fileCpu, [query.collection]);
+    }
+
+    /**
+     * Find the first matching entry in the specified database based on search criteria.
+     */
+    async findOne({ collection, search, context = {}, findOpts = {} }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+        let data = await this.fileCpu.findOne(collection, search, context, findOpts) as Data;
+        return data || null;
+    }
+
+    /**
+     * Update entries in the specified database based on search criteria and an updater function or object.
+     */
+    async update({ collection, search, updater, context = {} }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+        return await this.fileCpu.update(collection, false, search, updater, context);
+    }
+
+    /**
+     * Update the first matching entry in the specified database based on search criteria and an updater function or object.
+     */
+    async updateOne({ collection, search, updater, context = {} }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+        return await this.fileCpu.update(collection, true, search, updater, context);
+    }
+
+    /**
+     * Remove entries from the specified database based on search criteria.
+     */
+    async remove({ collection, search, context = {} }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+        return await this.fileCpu.remove(collection, false, search, context);
+    }
+
+    /**
+     * Remove the first matching entry from the specified database based on search criteria.
+     */
+    async removeOne({ collection, search, context = {} }: VQuery) {
+        await this.ensureCollection(arguments[0]);
+        return await this.fileCpu.remove(collection, true, search, context);
+    }
+
+    /**
+     * Removes a database collection from the file system.
+     */
+    async removeCollection({ collection }: VQuery) {
+        await this.mgr.removeCollection(collection);
+        return true;
+    }
+}

package/src/bin/data.ts

@@ -0,0 +1,91 @@
+import { BinManager, CollectionMeta } from ".";
+import { _log } from "../log";
+import { FileMeta, saveHeaderAndPayload } from "./head";
+import { detectCollisions, pushToFreeList, readData, roundUpCapacity, writeData } from "./utils";
+
+export function findCollection(cmp: BinManager, name: string): CollectionMeta | undefined {
+    return cmp.meta.collections.find(c => c.name === name);
+}
+
+export async function findFreeSlot(cmp: BinManager, size: number): Promise<FileMeta["freeList"][number] | undefined> {
+    const { meta } = cmp;
+    await _log(6, "Finding free slot for size:", size);
+    const idx = meta.freeList.findIndex(f => f.capacity >= size);
+
+    if (idx === -1) {
+        await _log(6, "No suitable free slot found.");
+        return undefined;
+    }
+
+    const slot = meta.freeList[idx];
+    await _log(6, "Free slot found at index:", idx, "with capacity:", slot.capacity);
+
+    meta.freeList.splice(idx, 1);
+    await _log(6, "Slot removed from freeList:", slot);
+
+    return slot;
+}
+
+export async function writeLogic(cmp: BinManager, collection: string, data: object[]) {
+    const { fd, meta } = cmp;
+    await _log(3, "Writing data to collection:", collection);
+
+    const existingCollection = findCollection(cmp, collection);
+    const encoded = Buffer.from(await cmp.options.format.encode(data));
+    const length = encoded.length;
+    const capacity = roundUpCapacity(meta, length + 4);
+
+    let offset = existingCollection?.offset;
+    let existingOffset = existingCollection?.offset;
+    let existingCapacity = existingCollection?.capacity;
+
+    const collision = detectCollisions(meta, offset, capacity, [collection]);
+    if (collision || !existingCollection) {
+        if (collision) await _log(2, "Collision detected");
+        const slot = await findFreeSlot(cmp, capacity);
+        if (slot) {
+            offset = slot.offset;
+            await _log(4, "Found free slot at offset:", offset);
+        } else {
+            offset = meta.fileSize;
+            meta.fileSize += capacity;
+            await _log(4, "No free slot found, appending at offset:", offset);
+        }
+
+        if (!existingCollection) {
+            meta.collections.push({ name: collection, offset, capacity });
+        } else if (collision) {
+            pushToFreeList(meta, existingOffset, existingCapacity);
+            meta.collections = meta.collections.map(c => {
+                if (c.offset === existingOffset) return { name: c.name, offset, capacity };
+                return c;
+            })
+        }
+
+        await _log(3, "Collection written");
+        await saveHeaderAndPayload(cmp);
+    }
+
+    const buf = Buffer.alloc(4);
+    buf.writeUInt32LE(length, 0);
+    await writeData(fd, offset, buf, 4);
+    await writeData(fd, offset + 4, encoded, capacity);
+
+    if (existingCollection && length >= existingCollection.capacity) {
+        meta.collections = meta.collections.map(c => {
+            if (c.offset === offset) return { name: c.name, offset, capacity };
+            return c;
+        });
+        await saveHeaderAndPayload(cmp);
+        await _log(2, "Capacity exceeded");
+    }
+}
+
+export async function readLogic(cmp: BinManager, collection: string) {
+    const collectionMeta = findCollection(cmp, collection);
+    if (!collectionMeta) throw new Error("Collection not found");
+
+    const len = await readData(cmp.fd, collectionMeta.offset, 4);
+    const data = await readData(cmp.fd, collectionMeta.offset + 4, len.readUInt32LE(0));
+    return await cmp.options.format.decode(data);
+}