@wxn0brp/db-storage-bin 0.0.2 → 0.0.4

package/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.4](https://github.com/wxn0brp/ValtheraDB-storage-bin/compare/v0.0.3...v0.0.4) (2025-09-04)
+
+
+### Features
+
+* add generic, BinManager.write<T> ([a2bb851](https://github.com/wxn0brp/ValtheraDB-storage-bin/commit/a2bb851c2b3977d133b85bde917dd268b3256a41))
+* format.encode(data, collection) ([6a20cb5](https://github.com/wxn0brp/ValtheraDB-storage-bin/commit/6a20cb57444ac6e9e1d5a61a63316f6e3184118e))
+
+### [0.0.3](https://github.com/wxn0brp/ValtheraDB-storage-bin/compare/v0.0.2...v0.0.3) (2025-08-24)
+
+
+### Bug Fixes
+
+* crc ([2b872a1](https://github.com/wxn0brp/ValtheraDB-storage-bin/commit/2b872a11fc42c32bbbeed80c76d62524426392d5))
+
 ### 0.0.2 (2025-08-24)
 
 

package/README.md

@@ -1,13 +1,11 @@
 # ValtheraDB Bin Plugin
 
-This is a proof-of-concept for an addon/plugin for the `@wxn0brp/db` (ValtheraDB) library.
-
 The purpose of this experiment is to create a storage layer that allows ValtheraDB, which normally operates on a directory/file structure, to instead use a single binary file for data storage.
 
 ## Installation
 
 ```bash
-yarn add github:wxn0brP/ValtheraDB-storage-bin#dist
+yarn add @wxn0brp/db-storage-bin
 ```
 
 ## Usage
@@ -55,6 +53,11 @@ Returns an object containing:
 -   `options`:
     -   `preferredSize`: The preferred block size for the database (default: `256`).
 
+## Documentation
+
+- [Data Structure](https://github.com/wxn0brP/ValtheraDB-storage-bin/blob/master/docs/data-structure.md)
+- [ValtheraDB-Core](https://github.com/wxn0brP/ValtheraDB-core)
+
 ## License
 
 This project is licensed under the MIT License.

package/dist/bin/data.d.ts

@@ -2,5 +2,5 @@ import { BinManager, CollectionMeta } from "./index.js";
 import { FileMeta } from "./head.js";
 export declare function findCollection(cmp: BinManager, name: string): CollectionMeta | undefined;
 export declare function findFreeSlot(cmp: BinManager, size: number): Promise<FileMeta["freeList"][number] | undefined>;
-export declare function writeLogic(cmp: BinManager, collection: string, data: object[]): Promise<void>;
+export declare function writeLogic(cmp: BinManager, collection: string, data: any): Promise<void>;
 export declare function readLogic(cmp: BinManager, collection: string): Promise<any>;

package/dist/bin/data.js

@@ -1,3 +1,4 @@
+import { getFileCrc } from "../crc32.js";
 import { _log } from "../log.js";
 import { saveHeaderAndPayload } from "./head.js";
 import { detectCollisions, pushToFreeList, readData, roundUpCapacity, writeData } from "./utils.js";
@@ -22,7 +23,7 @@ export async function writeLogic(cmp, collection, data) {
     const { fd, meta } = cmp;
     await _log(3, "Writing data to collection:", collection);
     const existingCollection = findCollection(cmp, collection);
-    const encoded = Buffer.from(await cmp.options.format.encode(data));
+    const encoded = Buffer.from(await cmp.options.format.encode(data, collection));
     const length = encoded.length;
     const capacity = roundUpCapacity(meta, length + 4);
     let offset = existingCollection?.offset;
@@ -69,6 +70,14 @@ export async function writeLogic(cmp, collection, data) {
         await saveHeaderAndPayload(cmp);
         await _log(2, "Capacity exceeded");
     }
+    else {
+        if (cmp.options.crc) {
+            const { computedCrc } = await getFileCrc(fd);
+            const crcBuf = Buffer.alloc(16);
+            crcBuf.writeUInt32LE(computedCrc);
+            await writeData(fd, 16, crcBuf, 16);
+        }
+    }
 }
 export async function readLogic(cmp, collection) {
     const collectionMeta = findCollection(cmp, collection);
@@ -76,5 +85,5 @@ export async function readLogic(cmp, collection) {
         throw new Error("Collection not found");
     const len = await readData(cmp.fd, collectionMeta.offset, 4);
     const data = await readData(cmp.fd, collectionMeta.offset + 4, len.readUInt32LE(0));
-    return await cmp.options.format.decode(data);
+    return await cmp.options.format.decode(data, collection);
 }

package/dist/bin/head.js

@@ -76,7 +76,7 @@ export async function readHeaderPayload(cmp) {
         await _log(6, "err", `Incomplete payload header read: expected ${payloadLength} bytes, got ${bytesRead}`);
         throw new Error(`Incomplete payload header read: expected ${payloadLength} bytes, got ${bytesRead}`);
     }
-    const obj = await cmp.options.format.decode(payloadBuf);
+    const obj = await cmp.options.format.decode(payloadBuf, "");
     meta.collections = (obj.c || []).map(([name, offset, capacity]) => ({ name, offset, capacity }));
     meta.freeList = (obj.f || []).map(([offset, capacity]) => ({ offset, capacity }));
     await _log(6, "Collections and freeList loaded", meta);
@@ -94,7 +94,7 @@ export async function saveHeaderAndPayload(cmp, recursion = false) {
     const { collections, freeList, fileSize } = meta;
     await _log(6, "Saving header payload:", collections, freeList);
     const payloadObj = getHeaderPayload(meta);
-    const payloadBuf = Buffer.from(await cmp.options.format.encode(payloadObj));
+    const payloadBuf = Buffer.from(await cmp.options.format.encode(payloadObj, ""));
     if (payloadBuf.length > 64 * 1024) {
         console.error("Header payload too large");
         throw new Error("Header payload too large");

package/dist/bin/index.d.ts

@@ -15,8 +15,8 @@ export interface Options {
     crc: number;
     overwriteRemovedCollection: boolean;
     format: {
-        encode(data: any): Promise<Parameters<typeof Buffer.from>[0]>;
-        decode(data: Buffer): Promise<any>;
+        encode(data: any, collection: string): Promise<Parameters<typeof Buffer.from>[0]>;
+        decode(data: Buffer, collection: string): Promise<any>;
     };
 }
 export declare class BinManager {
@@ -34,7 +34,7 @@ export declare class BinManager {
     constructor(path: string, options?: Partial<Options>);
     open(): Promise<void>;
     close(): Promise<void>;
-    write(collection: string, data: object[]): Promise<void>;
+    write<T = object[]>(collection: string, data: T): Promise<void>;
     read(collection: string): Promise<any>;
     optimize(): Promise<void>;
     removeCollection(collection: string): Promise<void>;

package/docs/data-structure.md

@@ -0,0 +1,44 @@
+# Data Structure
+
+This document describes the data structure used by the binary database format.
+
+## File Layout
+
+The file is structured as follows:
+
+1. **Header (64 bytes)**: Contains metadata about the file.
+2. **Payload**: Contains the serialized collections and free list data.
+3. **Data Blocks**: Contains the actual data for each collection.
+
+### Header
+
+The header is a fixed-size block of 64 bytes, structured as follows:
+
+| Offset | Size | Name         | Description                                           |
+|--------|------|--------------|-------------------------------------------------------|
+| 0      | 4    | Version      | File format version (currently 1)                     |
+| 4      | 4    | Payload Len  | Length of the payload data                            |
+| 8      | 4    | Payload Off  | Offset of the payload data from the header start      |
+| 12     | 4    | Block Size   | Preferred block size for allocations                  |
+| 16     | 4    | CRC32        | CRC32 checksum of the file (excluding this field)     |
+| 20     | 44   | Reserved     | Reserved for future use                               |
+
+### Payload
+
+The payload contains the serialized list of collections and free blocks. It is a msgpack-encoded object with the following structure:
+
+```ts
+{
+  c: [string, number, number][]; // Collections: [name, offset, capacity]
+  f: [number, number][];         // Free blocks: [offset, capacity]
+}
+```
+
+### Data Blocks
+
+Each collection's data is stored in a data block. A data block consists of:
+
+1. **Length (4 bytes)**: A 32-bit unsigned integer representing the length of the data (Uint32).
+2. **Data (variable)**: The actual data, padded to the nearest block size.
+
+The data is serialized using msgpack.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wxn0brp/db-storage-bin",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "author": "wxn0brP",

package/src/bin/data.ts

@@ -1,4 +1,5 @@
 import { BinManager, CollectionMeta } from ".";
+import { getFileCrc } from "../crc32";
 import { _log } from "../log";
 import { FileMeta, saveHeaderAndPayload } from "./head";
 import { detectCollisions, pushToFreeList, readData, roundUpCapacity, writeData } from "./utils";
@@ -26,12 +27,12 @@ export async function findFreeSlot(cmp: BinManager, size: number): Promise<FileM
     return slot;
 }
 
-export async function writeLogic(cmp: BinManager, collection: string, data: object[]) {
+export async function writeLogic(cmp: BinManager, collection: string, data: any) {
     const { fd, meta } = cmp;
     await _log(3, "Writing data to collection:", collection);
 
     const existingCollection = findCollection(cmp, collection);
-    const encoded = Buffer.from(await cmp.options.format.encode(data));
+    const encoded = Buffer.from(await cmp.options.format.encode(data, collection));
     const length = encoded.length;
     const capacity = roundUpCapacity(meta, length + 4);
 
@@ -78,6 +79,13 @@ export async function writeLogic(cmp: BinManager, collection: string, data: obje
         });
         await saveHeaderAndPayload(cmp);
         await _log(2, "Capacity exceeded");
+    } else {
+        if (cmp.options.crc) {
+            const { computedCrc } = await getFileCrc(fd);
+            const crcBuf = Buffer.alloc(16);
+            crcBuf.writeUInt32LE(computedCrc);
+            await writeData(fd, 16, crcBuf, 16);
+        }
     }
 }
 
@@ -87,5 +95,5 @@ export async function readLogic(cmp: BinManager, collection: string) {
 
     const len = await readData(cmp.fd, collectionMeta.offset, 4);
     const data = await readData(cmp.fd, collectionMeta.offset + 4, len.readUInt32LE(0));
-    return await cmp.options.format.decode(data);
+    return await cmp.options.format.decode(data, collection);
 }

package/src/bin/head.ts

@@ -106,7 +106,7 @@ export async function readHeaderPayload(cmp: BinManager) {
         throw new Error(`Incomplete payload header read: expected ${payloadLength} bytes, got ${bytesRead}`);
     }
 
-    const obj = await cmp.options.format.decode(payloadBuf) as {
+    const obj = await cmp.options.format.decode(payloadBuf, "") as {
         c: [string, number, number][];
         f: [number, number][];
     };
@@ -133,7 +133,7 @@ export async function saveHeaderAndPayload(cmp: BinManager, recursion = false) {
 
     const payloadObj = getHeaderPayload(meta);
 
-    const payloadBuf = Buffer.from(await cmp.options.format.encode(payloadObj));
+    const payloadBuf = Buffer.from(await cmp.options.format.encode(payloadObj, ""));
     if (payloadBuf.length > 64 * 1024) {
         console.error("Header payload too large");
         throw new Error("Header payload too large");

package/src/bin/index.ts

@@ -33,8 +33,8 @@ export interface Options {
     crc: number;
     overwriteRemovedCollection: boolean;
     format: {
-        encode(data: any): Promise<Parameters<typeof Buffer.from>[0]>;
-        decode(data: Buffer): Promise<any>;
+        encode(data: any, collection: string): Promise<Parameters<typeof Buffer.from>[0]>;
+        decode(data: Buffer, collection: string): Promise<any>;
     }
 }
 
@@ -87,7 +87,7 @@ export class BinManager {
         }
     }
 
-    async write(collection: string, data: object[]) {
+    async write<T = object[]>(collection: string, data: T) {
         if (!this.fd) throw new Error("File not open");
         await writeLogic(this, collection, data);
     }