@wxn0brp/db-storage-bin 0.110.0-alpha.0 → 0.110.0-alpha.1

package/README.md

@@ -34,6 +34,12 @@ This will reclaim unused space in the binary file.
 await mgr.optimize();
 ```
 
+### Closing
+
+```typescript
+await db.close();
+```
+
 ## API
 
 ### `createBinValthera(path, opts, init)`

package/dist/bin/optimize.js

@@ -1,40 +1,77 @@
-import { unlink } from "fs/promises";
+import { rename, rm } from "fs/promises";
+import { BinManager } from "./index.js";
+import { ensureCollection } from "./collection.js";
 import { saveHeaderAndPayload } from "./head.js";
-import { HEADER_SIZE, INT_SIZE } from "./static.js";
-import { readData, roundUpCapacity, writeData } from "./utils.js";
+import { INT_SIZE } from "./static.js";
+import { readCollectionEof, readData, writeData } from "./utils.js";
 import { _log } from "../log.js";
+function isRemovedRecord(data) {
+    return new Uint8Array(data).every(byte => byte === 0);
+}
+async function appendRawRecord(target, collectionName, recordLengthBuffer, data) {
+    const collection = await ensureCollection(target, collectionName, data.length, true);
+    const collectionEOF = await readCollectionEof(target.fd, collection.offset);
+    const offset = collection.offset + collectionEOF + INT_SIZE;
+    await writeData(target.fd, offset, recordLengthBuffer, INT_SIZE);
+    await writeData(target.fd, offset + INT_SIZE, data, data.length);
+    const newEOF = collectionEOF + INT_SIZE + data.length;
+    const collectionLengthBuffer = Buffer.alloc(INT_SIZE);
+    collectionLengthBuffer.writeUInt32LE(newEOF, 0);
+    await writeData(target.fd, collection.offset, collectionLengthBuffer, INT_SIZE);
+}
+async function copyLiveRecords(source, target, collectionName, sourceOffset, collectionEOF) {
+    let readCursor = sourceOffset + INT_SIZE;
+    const endOffset = readCursor + collectionEOF;
+    while (readCursor < endOffset) {
+        const recordLengthBuffer = await readData(source.fd, readCursor, INT_SIZE);
+        const recordLength = recordLengthBuffer.readUInt32LE(0);
+        readCursor += INT_SIZE;
+        const data = await readData(source.fd, readCursor, recordLength);
+        readCursor += recordLength;
+        if (isRemovedRecord(data))
+            continue;
+        await appendRawRecord(target, collectionName, recordLengthBuffer, data);
+    }
+}
 export async function optimize(cmp) {
     await _log(3, "Starting database optimization");
-    const collections = cmp.meta.collections;
-    const allData = new Map();
-    for (const { name, offset } of collections) {
-        await _log(6, "Reading collection for optimization:", name);
-        const len = await readData(cmp.fd, offset, 4);
-        const data = await readData(cmp.fd, offset + 4, len.readInt32LE(0));
-        allData.set(name, data);
+    const collections = [...cmp.meta.collections];
+    const tmpPath = `${cmp.path}.tmp`;
+    await _log(6, "Removing stale optimization temp file:", tmpPath);
+    await rm(tmpPath, { force: true });
+    const tmpMgr = new BinManager(tmpPath, cmp.options);
+    let tmpOpened = false;
+    let originalClosed = false;
+    try {
+        await tmpMgr.init();
+        tmpOpened = true;
+        const lengthBuffer = Buffer.alloc(INT_SIZE);
+        for (const { name, offset } of collections) {
+            await _log(6, "Optimizing collection:", name);
+            const collectionEOF = await readCollectionEof(cmp.fd, offset);
+            const collectionMeta = await ensureCollection(tmpMgr, name, 0, false);
+            lengthBuffer.writeUInt32LE(0, 0);
+            await writeData(tmpMgr.fd, collectionMeta.offset, lengthBuffer, INT_SIZE);
+            await copyLiveRecords(cmp, tmpMgr, name, offset, collectionEOF);
+        }
+        await saveHeaderAndPayload(tmpMgr);
+        await tmpMgr.close();
+        tmpOpened = false;
+        await _log(5, "Closing original file for optimization");
+        await cmp.close();
+        originalClosed = true;
+        await _log(5, "Replacing original database with optimized temp file");
+        await rename(tmpPath, cmp.path);
+        await cmp.init();
+        originalClosed = false;
     }
-    await _log(5, "Closing file for optimization");
-    await cmp.close();
-    await _log(6, "Deleting old database file for optimization");
-    await unlink(cmp.path);
-    await new Promise(resolve => setTimeout(resolve, 100));
-    await _log(5, "Re-opening database file for optimization");
-    await cmp.init();
-    let offset = roundUpCapacity(cmp.meta, cmp.meta.payloadLength + HEADER_SIZE) + cmp.meta.blockSize;
-    const lengthBuffer = Buffer.alloc(INT_SIZE);
-    for (const [collection, data] of allData) {
-        await _log(6, "Writing optimized collection:", collection);
-        const len = roundUpCapacity(cmp.meta, data.length + INT_SIZE);
-        lengthBuffer.writeInt32LE(data.length, 0);
-        await writeData(cmp.fd, offset, lengthBuffer, INT_SIZE);
-        await writeData(cmp.fd, offset + INT_SIZE, data, len);
-        cmp.meta.collections.push({
-            name: collection,
-            offset,
-            capacity: len
-        });
-        offset += len;
+    catch (err) {
+        if (tmpOpened)
+            await tmpMgr.close();
+        await rm(tmpPath, { force: true });
+        if (originalClosed)
+            await cmp.init();
+        throw err;
     }
-    await saveHeaderAndPayload(cmp);
     await _log(3, "Database optimization complete");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wxn0brp/db-storage-bin",
-  "version": "0.110.0-alpha.0",
+  "version": "0.110.0-alpha.1",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "author": "wxn0brP",
@@ -19,7 +19,7 @@
   ],
   "devDependencies": {
     "@types/bun": "*",
-    "@wxn0brp/db-core": "^0.11.0-alpha.4",
+    "@wxn0brp/db-core": "^0.11.1-alpha.2",
     "tsc-alias": "^1",
     "typescript": "^6"
   },
@@ -27,7 +27,7 @@
     "@msgpack/msgpack": "^3.1.3"
   },
   "peerDependencies": {
-    "@wxn0brp/db-core": "^0.11.0-alpha.4"
+    "@wxn0brp/db-core": "^0.11.1-alpha.2"
   },
   "files": [
     "dist"