document-dataply 0.0.3-alpha.1 → 0.0.3-alpha.2

package/README.md

@@ -1,3 +1,6 @@
+![node.js workflow](https://github.com/izure1/document-dataply/actions/workflows/node.js.yml/badge.svg)
+![License](https://img.shields.io/badge/license-MIT-blue.svg)
+
 # Document-Dataply
 
 > [!WARNING]
@@ -175,6 +178,27 @@ Returns a new `Transaction` object.
 ### `db.close()`
 Flushes changes and closes the database files.
 
+## Benchmark
+
+The following benchmarks were conducted using the `npm run benchmark` command. The results show the performance of various operations on a dataset of **10,000** documents.
+
+- **Package Version**: `0.0.3-alpha.1`
+- **Batch Size**: 1,000 items per batch (for bulk operations)
+- **Iterations**: 5 full lifecycle cycles
+
+### Benchmark Results
+
+| Operation | Count | Avg Time (ms) | Ops/s | Min (ms) | Max (ms) | Remarks |
+| :--- | :---: | :---: | :---: | :---: | :---: | :--- |
+| **InsertBatch** | 10,000 | 6499.22 | **1,538** | 5891.53 | 7403.67 | 1000 items per batch |
+| **Select** | 100 | 30.09 | **3,323** | 26.09 | 37.16 | Indexed Equality |
+| **Partial Update** | 100 | 86.44 | **1,157** | 74.15 | 115.22 | Bulk Update |
+| **Full Update** | 1 | 20.77 | **48** | 16.10 | 25.61 | Single Update |
+| **Delete** | 100 | 5893.65 | **17** | 5523.06 | 6173.39 | Page Reclamation |
+
+> [!NOTE]
+> **Performance Analysis**: The `Delete (Bulk)` operation takes more time compared to other operations because it involves intensive index tree restructuring and physical **Page Reclamation (Garbage Collection)** logic to maintain storage efficiency after massive data removal.
+
 ## License
 
 MIT

package/dist/cjs/index.js

@@ -7944,20 +7944,32 @@ var require_cjs = __commonJS({
           } else {
             let nextPageId = manager.getNextPageId(page);
             if (nextPageId !== -1) {
-              let pendingFreePageId = nextPageId;
-              while (pendingFreePageId !== -1) {
-                const pendingPage = await this.get(pendingFreePageId, tx);
-                const pendingManager = this.pageFactory.getManager(pendingPage);
-                const next = pendingManager.getNextPageId(pendingPage);
-                await this.setFreePage(pendingFreePageId, tx);
-                pendingFreePageId = next;
-              }
+              await this.freeChain(nextPageId, tx);
               manager.setNextPageId(page, -1);
               await this.setPage(currentPageId, page, tx);
             }
           }
         }
       }
+      /**
+       * Free chain of pages.
+       * @param startPageId Start page ID
+       * @param tx Transaction
+       */
+      async freeChain(startPageId, tx) {
+        let currentPageId = startPageId;
+        const visited = /* @__PURE__ */ new Set();
+        while (currentPageId !== -1 && currentPageId !== 0) {
+          if (visited.has(currentPageId)) {
+            break;
+          }
+          visited.add(currentPageId);
+          const page = await this.get(currentPageId, tx);
+          const nextPageId = this.pageFactory.getManager(page).getNextPageId(page);
+          await this.setFreePage(currentPageId, tx);
+          currentPageId = nextPageId;
+        }
+      }
       /**
        * Frees the page and marks it as available in the bitmap.
        * It also adds the page to the linked list of free pages in metadata.
@@ -7965,6 +7977,7 @@ var require_cjs = __commonJS({
        * @param tx Transaction
        */
       async setFreePage(pageId, tx) {
+        if (pageId <= 0) return;
         await tx.__acquireWriteLock(0);
         await tx.__acquireWriteLock(pageId);
         const metadata = await this.getMetadata(tx);
@@ -8137,17 +8150,8 @@ var require_cjs = __commonJS({
       }
       async delete(id) {
         const tx = this.txContext.get();
-        const manager = this.factory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_INDEX);
-        let pageId = +id;
-        while (true) {
-          const page = await this.pfs.get(pageId, tx);
-          const nextPageId = manager.getNextPageId(page);
-          await this.pfs.setFreePage(pageId, tx);
-          if (nextPageId === -1) {
-            break;
-          }
-          pageId = nextPageId;
-        }
+        const pageId = +id;
+        await this.pfs.freeChain(pageId, tx);
       }
       async readHead() {
         const tx = this.txContext.get();
@@ -8568,14 +8572,8 @@ var require_cjs = __commonJS({
           return;
         }
         if (this.rowManager.getOverflowFlag(row)) {
-          let overflowPageId = bytesToNumber(this.rowManager.getBody(row));
-          while (overflowPageId !== -1) {
-            const overflowPage = await this.pfs.get(overflowPageId, tx);
-            const manager = this.factory.getManager(overflowPage);
-            const nextPageId = manager.getNextPageId(overflowPage);
-            await this.pfs.setFreePage(overflowPageId, tx);
-            overflowPageId = nextPageId;
-          }
+          const overflowPageId = bytesToNumber(this.rowManager.getBody(row));
+          await this.pfs.freeChain(overflowPageId, tx);
         }
         this.rowManager.setDeletedFlag(row, true);
         await this.pfs.setPage(pageId, page, tx);
@@ -8606,7 +8604,7 @@ var require_cjs = __commonJS({
           }
         }
         if (allDeleted) {
-          await this.pfs.setFreePage(pageId, tx);
+          await this.pfs.freeChain(pageId, tx);
         }
       }
       /**
@@ -9532,6 +9530,15 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       return tx;
     });
   }
+  async getDocument(pk, tx) {
+    return this.runWithDefault(async (tx2) => {
+      const row = await this.select(pk, false, tx2);
+      if (!row) {
+        throw new Error(`Document not found with PK: ${pk}`);
+      }
+      return JSON.parse(row);
+    }, tx);
+  }
   async readLock(fn) {
     let lockId;
     return this.lock.readLock(async (_lockId) => {
@@ -9550,15 +9557,6 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       this.lock.writeUnlock(lockId);
     });
   }
-  async getDocument(pk, tx) {
-    return this.runWithDefault(async (tx2) => {
-      const row = await this.select(pk, false, tx2);
-      if (!row) {
-        throw new Error(`Document not found with PK: ${pk}`);
-      }
-      return JSON.parse(row);
-    }, tx);
-  }
   /**
    * Backfill indices for fields that were added with `true` option after data was inserted.
    * This method should be called after `init()` if you want to index existing documents
@@ -9796,9 +9794,16 @@ var DocumentDataply = class _DocumentDataply {
       const tree = this.api.trees.get(field);
       if (!tree) continue;
       const condition = query[field];
-      candidates.push({ tree, condition, field });
+      const treeTx = await tree.createTransaction();
+      candidates.push({ tree: treeTx, condition, field });
     }
+    const rollback = () => {
+      for (const { tree } of candidates) {
+        tree.rollback();
+      }
+    };
     if (candidates.length === 0) {
+      rollback();
       return null;
     }
     if (orderByField) {
@@ -9806,7 +9811,8 @@ var DocumentDataply = class _DocumentDataply {
       if (orderByCandidate) {
         return {
           driver: orderByCandidate,
-          others: candidates.filter((c) => c.field !== orderByField)
+          others: candidates.filter((c) => c.field !== orderByField),
+          rollback
         };
       }
     }
@@ -9817,9 +9823,41 @@ var DocumentDataply = class _DocumentDataply {
     if (!res) return null;
     return {
       driver: res,
-      others: candidates.filter((c) => c.tree !== res.tree)
+      others: candidates.filter((c) => c.tree !== res.tree),
+      rollback
     };
   }
+  /**
+   * Get Primary Keys based on query and index selection.
+   * Internal common method to unify query optimization.
+   */
+  async getKeys(query, orderBy, sortOrder = "asc") {
+    const isQueryEmpty = Object.keys(query).length === 0;
+    const normalizedQuery = isQueryEmpty ? { _id: { gte: 0 } } : query;
+    const verbose = this.verboseQuery(normalizedQuery);
+    const selectivity = await this.getSelectivityCandidate(
+      verbose,
+      orderBy
+    );
+    if (!selectivity) return /* @__PURE__ */ new Set();
+    const { driver, others, rollback } = selectivity;
+    const isDriverOrderByField = orderBy === void 0 || driver.field === orderBy;
+    if (isDriverOrderByField) {
+      let keys = await driver.tree.keys(driver.condition, void 0, sortOrder);
+      for (const { tree, condition } of others) {
+        keys = await tree.keys(condition, keys, sortOrder);
+      }
+      rollback();
+      return keys;
+    } else {
+      let keys = await driver.tree.keys(driver.condition, void 0);
+      for (const { tree, condition } of others) {
+        keys = await tree.keys(condition, keys);
+      }
+      rollback();
+      return keys;
+    }
+  }
   async insertDocument(document, tx) {
     const metadata = await this.api.getDocumentInnerMetadata(tx);
     const id = ++metadata.lastId;
@@ -9897,7 +9935,10 @@ var DocumentDataply = class _DocumentDataply {
             console.error(`BPTree indexing failed for field: ${field}`, error);
           }
         }
-        await treeTx.commit();
+        const res = await treeTx.commit();
+        if (!res.success) {
+          throw res.error;
+        }
       }
       return ids;
     }, tx));
@@ -9910,37 +9951,42 @@ var DocumentDataply = class _DocumentDataply {
    * @returns The number of updated documents
    */
   async updateInternal(query, computeUpdatedDoc, tx) {
-    const idTree = this.api.trees.get("_id");
-    if (!idTree) {
-      throw new Error("ID tree not found");
-    }
-    const { stream } = this.select(query, {}, tx);
+    const pks = await this.getKeys(query);
     let updatedCount = 0;
-    for await (const doc of stream) {
-      const id = doc._id;
-      let pk = null;
-      for await (const [entryPk] of idTree.whereStream({ primaryEqual: { v: id } })) {
-        pk = entryPk;
-        break;
-      }
-      if (pk === null) continue;
+    const treeTxs = /* @__PURE__ */ new Map();
+    for (const [field, tree] of this.api.trees) {
+      treeTxs.set(field, await tree.createTransaction());
+    }
+    treeTxs.delete("_id");
+    for (const pk of pks) {
+      const doc = await this.api.getDocument(pk, tx);
+      if (!doc) continue;
       const updatedDoc = computeUpdatedDoc(doc);
       const oldFlatDoc = this.api.flattenDocument(doc);
       const newFlatDoc = this.api.flattenDocument(updatedDoc);
-      for (const [field, tree] of this.api.trees) {
+      for (const [field, treeTx] of treeTxs) {
         const oldV = oldFlatDoc[field];
         const newV = newFlatDoc[field];
         if (oldV === newV) continue;
-        if (oldV !== void 0) {
-          await tree.delete(pk, { k: pk, v: oldV });
+        if (field in oldFlatDoc) {
+          await treeTx.delete(pk, { k: pk, v: oldV });
         }
-        if (newV !== void 0) {
-          await tree.insert(pk, { k: pk, v: newV });
+        if (field in newFlatDoc) {
+          await treeTx.insert(pk, { k: pk, v: newV });
         }
       }
       await this.api.update(pk, JSON.stringify(updatedDoc), tx);
       updatedCount++;
     }
+    for (const [field, treeTx] of treeTxs) {
+      const result = await treeTx.commit();
+      if (!result.success) {
+        for (const rollbackTx of treeTxs.values()) {
+          rollbackTx.rollback();
+        }
+        throw result.error;
+      }
+    }
     return updatedCount;
   }
   /**
@@ -9951,7 +9997,7 @@ var DocumentDataply = class _DocumentDataply {
    * @returns The number of updated documents
    */
   async fullUpdate(query, newRecord, tx) {
-    return this.api.writeLock(() => this.api.runWithDefault(async (tx2) => {
+    return await this.api.writeLock(() => this.api.runWithDefault(async (tx2) => {
       return this.updateInternal(query, (doc) => {
         const newDoc = typeof newRecord === "function" ? newRecord(doc) : newRecord;
         return { _id: doc._id, ...newDoc };
@@ -9982,20 +10028,11 @@ var DocumentDataply = class _DocumentDataply {
    */
   async delete(query, tx) {
     return this.api.writeLock(() => this.api.runWithDefault(async (tx2) => {
-      const idTree = this.api.trees.get("_id");
-      if (!idTree) {
-        throw new Error("ID tree not found");
-      }
-      const { stream } = this.select(query, {}, tx2);
+      const pks = await this.getKeys(query);
       let deletedCount = 0;
-      for await (const doc of stream) {
-        const id = doc._id;
-        let pk = null;
-        for await (const [entryPk] of idTree.whereStream({ primaryEqual: { v: id } })) {
-          pk = entryPk;
-          break;
-        }
-        if (pk === null) continue;
+      for (const pk of pks) {
+        const doc = await this.api.getDocument(pk, tx2);
+        if (!doc) continue;
         const flatDoc = this.api.flattenDocument(doc);
         for (const [field, tree] of this.api.trees) {
           const v = flatDoc[field];
@@ -10028,47 +10065,30 @@ var DocumentDataply = class _DocumentDataply {
     }
     const {
       limit = Infinity,
-      sortOrder = "asc"
+      sortOrder = "asc",
+      orderBy: orderByField
     } = options;
     const self = this;
     const stream = this.api.streamWithDefault(async function* (tx2) {
-      const isQueryEmpty = Object.keys(query).length === 0;
-      const normalizedQuery = isQueryEmpty ? { _id: { gte: 0 } } : query;
-      const verbose = self.verboseQuery(normalizedQuery);
+      const keys = await self.getKeys(query, orderByField, sortOrder);
       const selectivity = await self.getSelectivityCandidate(
-        verbose,
-        orderBy
+        self.verboseQuery(query),
+        orderByField
       );
-      if (!selectivity) return;
-      const { driver, others } = selectivity;
-      const isDriverOrderByField = orderBy === void 0 || driver.field === orderBy;
-      if (isDriverOrderByField) {
-        let keys = await driver.tree.keys(driver.condition, void 0, sortOrder);
-        for (const { tree, condition } of others) {
-          keys = await tree.keys(condition, keys, sortOrder);
-        }
-        let i = 0;
-        for (const key of keys) {
-          if (i >= limit) break;
-          const stringified = await self.api.select(key, false, tx2);
-          if (!stringified) continue;
-          yield JSON.parse(stringified);
-          i++;
-        }
-      } else {
+      const isDriverOrderByField = orderByField === void 0 || selectivity && selectivity.driver.field === orderByField;
+      if (selectivity) {
+        selectivity.rollback();
+      }
+      if (!isDriverOrderByField && orderByField) {
         const results = [];
-        let keys = await driver.tree.keys(driver.condition, void 0);
-        for (const { tree, condition } of others) {
-          keys = await tree.keys(condition, keys);
-        }
         for (const key of keys) {
           const stringified = await self.api.select(key, false, tx2);
           if (!stringified) continue;
           results.push(JSON.parse(stringified));
         }
         results.sort((a, b) => {
-          const aVal = a[orderBy] ?? a._id;
-          const bVal = b[orderBy] ?? b._id;
+          const aVal = a[orderByField] ?? a._id;
+          const bVal = b[orderByField] ?? b._id;
           const cmp = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
           return sortOrder === "asc" ? cmp : -cmp;
         });
@@ -10076,6 +10096,15 @@ var DocumentDataply = class _DocumentDataply {
         for (const doc of limitedResults) {
           yield doc;
         }
+      } else {
+        let i = 0;
+        for (const key of keys) {
+          if (i >= limit) break;
+          const stringified = await self.api.select(key, false, tx2);
+          if (!stringified) continue;
+          yield JSON.parse(stringified);
+          i++;
+        }
       }
     }, tx);
     const drain = async () => {

package/dist/types/core/document.d.ts

@@ -10,9 +10,9 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON, IC extends Index
     private pendingBackfillFields;
     private readonly lock;
     constructor(file: string, options: DocumentDataplyOptions<T, IC>);
+    getDocument(pk: number, tx?: Transaction): Promise<DataplyDocument<T>>;
     readLock<T>(fn: () => T): Promise<T>;
     writeLock<T>(fn: () => T): Promise<T>;
-    getDocument(pk: number, tx?: Transaction): Promise<DataplyDocument<T>>;
     /**
      * Backfill indices for fields that were added with `true` option after data was inserted.
      * This method should be called after `init()` if you want to index existing documents
@@ -96,7 +96,13 @@ export declare class DocumentDataply<T extends DocumentJSON, IC extends IndexCon
             condition: Partial<DocumentDataplyCondition<U>>;
             field: string;
         }[];
+        rollback: () => void;
     } | null>;
+    /**
+     * Get Primary Keys based on query and index selection.
+     * Internal common method to unify query optimization.
+     */
+    private getKeys;
     private insertDocument;
     /**
      * Insert a document into the database

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.3-alpha.1",
+  "version": "0.0.3-alpha.2",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -23,6 +23,9 @@
   },
   "scripts": {
     "test": "jest -i",
+    "pretest": "esbuild test/wal_crash_sim.ts --bundle --platform=node --outfile=test/wal_crash_sim_bundled.js",
+    "benchmark": "npx tsx benchmark/MassiveInsert.bench.ts",
+    "prebenchmark": "npm run pretest",
     "build": "node build/index.js && tsc"
   },
   "keywords": [
@@ -39,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.18"
+    "dataply": "^0.0.19-alpha.0"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",