dataply 0.0.24-alpha.0 → 0.0.24-alpha.2

package/dist/cjs/index.js

@@ -1549,83 +1549,217 @@ var BPTreeTransaction = class _BPTreeTransaction {
       return regexp.test(nodeValue);
     }
   };
-  verifierStartNode = {
-    gt: (v) => this.insertableNodeByPrimary(v),
-    gte: (v) => this.insertableNodeByPrimary(v),
-    lt: (v) => this.insertableNodeByPrimary(v),
-    lte: (v) => this.insertableRightestNodeByPrimary(v),
-    equal: (v) => this.insertableNodeByPrimary(v),
-    notEqual: (v) => this.leftestNode(),
-    or: (v) => this.insertableNodeByPrimary(this.lowestPrimaryValue(this.ensureValues(v))),
-    primaryGt: (v) => this.insertableNodeByPrimary(v),
-    primaryGte: (v) => this.insertableNodeByPrimary(v),
-    primaryLt: (v) => this.insertableNodeByPrimary(v),
-    primaryLte: (v) => this.insertableRightestNodeByPrimary(v),
-    primaryEqual: (v) => this.insertableNodeByPrimary(v),
-    primaryNotEqual: (v) => this.leftestNode(),
-    primaryOr: (v) => this.insertableNodeByPrimary(this.lowestPrimaryValue(this.ensureValues(v))),
-    like: (v) => this.leftestNode()
-  };
-  verifierEndNode = {
-    gt: (v) => null,
-    gte: (v) => null,
-    lt: (v) => null,
-    lte: (v) => null,
-    equal: (v) => this.insertableEndNode(v, this.verifierDirection.equal),
-    notEqual: (v) => null,
-    or: (v) => this.insertableEndNode(
-      this.highestValue(this.ensureValues(v)),
-      this.verifierDirection.or
-    ),
-    primaryGt: (v) => null,
-    primaryGte: (v) => null,
-    primaryLt: (v) => null,
-    primaryLte: (v) => null,
-    primaryEqual: (v) => this.insertableRightestEndNodeByPrimary(v),
-    primaryNotEqual: (v) => null,
-    primaryOr: (v) => this.insertableRightestEndNodeByPrimary(
-      this.highestPrimaryValue(this.ensureValues(v))
-    ),
-    like: (v) => null
-  };
-  verifierDirection = {
-    gt: 1,
-    gte: 1,
-    lt: -1,
-    lte: -1,
-    equal: 1,
-    notEqual: 1,
-    or: 1,
-    primaryGt: 1,
-    primaryGte: 1,
-    primaryLt: -1,
-    primaryLte: -1,
-    primaryEqual: 1,
-    primaryNotEqual: 1,
-    primaryOr: 1,
-    like: 1
-  };
-  /**
-   * Determines whether early termination is allowed for each condition.
-   * When true, the search will stop once a match is found and then a non-match is encountered.
-   * Only applicable for conditions that guarantee contiguous matches in a sorted B+Tree.
-   */
-  verifierEarlyTerminate = {
-    gt: false,
-    gte: false,
-    lt: false,
-    lte: false,
-    equal: true,
-    notEqual: false,
-    or: false,
-    primaryGt: false,
-    primaryGte: false,
-    primaryLt: false,
-    primaryLte: false,
-    primaryEqual: true,
-    primaryNotEqual: false,
-    primaryOr: false,
-    like: false
+  searchConfigs = {
+    gt: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    gte: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    lt: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], 1),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    lte: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], 1),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableRightestNodeByPrimary(v[0]),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    equal: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: (tx, v) => tx.insertableEndNode(v[0], 1),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableEndNode(v[0], 1),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    notEqual: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    or: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(tx.lowestValue(v)),
+        end: (tx, v) => tx.insertableEndNode(tx.highestValue(v), 1),
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx, v) => tx.insertableEndNode(tx.highestValue(v), 1),
+        end: (tx, v) => tx.insertableEndNode(tx.lowestValue(v), -1),
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    primaryGt: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    primaryGte: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    primaryLt: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], 1),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    primaryLte: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: (tx, v) => tx.insertableEndNode(v[0], 1),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableRightestNodeByPrimary(v[0]),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    primaryEqual: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+        end: (tx, v) => tx.insertableRightestEndNodeByPrimary(v[0]),
+        direction: 1,
+        earlyTerminate: true
+      },
+      desc: {
+        start: (tx, v) => tx.insertableRightestEndNodeByPrimary(v[0]),
+        end: (tx, v) => tx.insertableEndNode(v[0], -1),
+        direction: -1,
+        earlyTerminate: true
+      }
+    },
+    primaryNotEqual: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    primaryOr: {
+      asc: {
+        start: (tx, v) => tx.insertableNodeByPrimary(tx.lowestPrimaryValue(v)),
+        end: (tx, v) => tx.insertableRightestEndNodeByPrimary(tx.highestPrimaryValue(v)),
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx, v) => tx.insertableRightestEndNodeByPrimary(tx.highestPrimaryValue(v)),
+        end: (tx, v) => tx.insertableEndNode(tx.lowestPrimaryValue(v), -1),
+        direction: -1,
+        earlyTerminate: false
+      }
+    },
+    like: {
+      asc: {
+        start: (tx) => tx.leftestNode(),
+        end: () => null,
+        direction: 1,
+        earlyTerminate: false
+      },
+      desc: {
+        start: (tx) => tx.rightestNode(),
+        end: () => null,
+        direction: -1,
+        earlyTerminate: false
+      }
+    }
   };
   /**
    * Priority map for condition types.
@@ -2236,16 +2370,20 @@ var BPTreeSyncTransaction = class extends BPTreeTransaction {
     const driverKey = this.getDriverKey(condition);
     if (!driverKey) return;
     const value = condition[driverKey];
-    let startNode = this.verifierStartNode[driverKey](value);
-    let endNode = this.verifierEndNode[driverKey](value);
-    let direction = this.verifierDirection[driverKey];
+    const v = this.ensureValues(value);
+    const config = this.searchConfigs[driverKey][order];
+    let startNode = config.start(this, v);
+    let endNode = config.end(this, v);
+    const direction = config.direction;
+    const earlyTerminate = config.earlyTerminate;
+    if (order === "desc" && !startNode) {
+      startNode = this.rightestNode();
+    }
+    if (order === "asc" && !startNode) {
+      startNode = this.leftestNode();
+    }
+    if (!startNode) return;
     const comparator = this.verifierMap[driverKey];
-    const earlyTerminate = this.verifierEarlyTerminate[driverKey];
-    if (order === "desc") {
-      startNode = endNode ?? this.rightestNode();
-      endNode = null;
-      direction *= -1;
-    }
     const generator = this.getPairsGenerator(
       value,
       startNode,
@@ -2257,7 +2395,7 @@ var BPTreeSyncTransaction = class extends BPTreeTransaction {
     let count = 0;
     const intersection = filterValues && filterValues.size > 0 ? filterValues : null;
     for (const pair of generator) {
-      const [k, v] = pair;
+      const [k, v2] = pair;
       if (intersection && !intersection.has(k)) {
         continue;
       }
@@ -2266,7 +2404,7 @@ var BPTreeSyncTransaction = class extends BPTreeTransaction {
         if (key === driverKey) continue;
         const verify = this.verifierMap[key];
         const condValue = condition[key];
-        if (!verify(v, condValue)) {
+        if (!verify(v2, condValue)) {
           isMatch = false;
           break;
         }
@@ -3366,16 +3504,20 @@ var BPTreeAsyncTransaction = class extends BPTreeTransaction {
     const driverKey = this.getDriverKey(condition);
     if (!driverKey) return;
     const value = condition[driverKey];
-    let startNode = await this.verifierStartNode[driverKey](value);
-    let endNode = await this.verifierEndNode[driverKey](value);
-    let direction = this.verifierDirection[driverKey];
+    const v = this.ensureValues(value);
+    const config = this.searchConfigs[driverKey][order];
+    let startNode = await config.start(this, v);
+    let endNode = await config.end(this, v);
+    const direction = config.direction;
+    const earlyTerminate = config.earlyTerminate;
+    if (order === "desc" && !startNode) {
+      startNode = await this.rightestNode();
+    }
+    if (order === "asc" && !startNode) {
+      startNode = await this.leftestNode();
+    }
+    if (!startNode) return;
     const comparator = this.verifierMap[driverKey];
-    const earlyTerminate = this.verifierEarlyTerminate[driverKey];
-    if (order === "desc") {
-      startNode = endNode ?? await this.rightestNode();
-      endNode = null;
-      direction *= -1;
-    }
     const generator = this.getPairsGenerator(
       value,
       startNode,
@@ -3387,7 +3529,7 @@ var BPTreeAsyncTransaction = class extends BPTreeTransaction {
     let count = 0;
     const intersection = filterValues && filterValues.size > 0 ? filterValues : null;
     for await (const pair of generator) {
-      const [k, v] = pair;
+      const [k, v2] = pair;
       if (intersection && !intersection.has(k)) {
         continue;
       }
@@ -3396,7 +3538,7 @@ var BPTreeAsyncTransaction = class extends BPTreeTransaction {
         if (key === driverKey) continue;
         const verify = this.verifierMap[key];
         const condValue = condition[key];
-        if (!verify(v, condValue)) {
+        if (!verify(v2, condValue)) {
           isMatch = false;
           break;
         }
@@ -8341,7 +8483,7 @@ var PageFileSystem = class {
   /**
    * Appends and inserts a new page.
    * If a free page is available in the free list, it reuses it.
-   * Otherwise, it appends a new page to the end of the file.
+   * Otherwise, it preallocates `pagePreallocationCount` pages to support sequential reads.
    * @returns Created or reused page ID
    */
   async appendNewPage(pageType = PageManager.CONSTANT.PAGE_TYPE_EMPTY, tx) {
@@ -8362,12 +8504,26 @@ var PageFileSystem = class {
       await this.setPage(reusedPageId, newPage2, tx);
       return reusedPageId;
     }
+    const preallocationCount = this.options.pagePreallocationCount;
     const pageCount = metadataManager.getPageCount(metadata);
     const newPageIndex = pageCount;
-    const newTotalCount = pageCount + 1;
+    const newTotalCount = pageCount + preallocationCount;
     const manager = this.pageFactory.getManagerFromType(pageType);
     const newPage = manager.create(this.pageSize, newPageIndex);
     await this.setPage(newPageIndex, newPage, tx);
+    const emptyManager = this.pageFactory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_EMPTY);
+    const firstFreeIndex = newPageIndex + 1;
+    const lastFreeIndex = newPageIndex + preallocationCount - 1;
+    for (let i = firstFreeIndex; i <= lastFreeIndex; i++) {
+      const emptyPage = emptyManager.create(this.pageSize, i);
+      const nextId = i < lastFreeIndex ? i + 1 : -1;
+      emptyManager.setNextPageId(emptyPage, nextId);
+      await this.setPage(i, emptyPage, tx);
+      await this.updateBitmap(i, true, tx);
+    }
+    if (preallocationCount > 1) {
+      metadataManager.setFreePageId(metadata, firstFreeIndex);
+    }
     metadataManager.setPageCount(metadata, newTotalCount);
     await this.setPage(0, metadata, tx);
     return newPageIndex;
@@ -8451,7 +8607,8 @@ var PageFileSystem = class {
   }
   /**
    * Frees the page and marks it as available in the bitmap.
-   * It also adds the page to the linked list of free pages in metadata.
+   * Inserts the page into the free list in ascending order by page ID
+   * to support HDD sequential reads.
    * @param pageId Page ID
    * @param tx Transaction
    */
@@ -8461,13 +8618,30 @@ var PageFileSystem = class {
     await tx.__acquireWriteLock(pageId);
     const metadata = await this.getMetadata(tx);
     const metadataManager = this.pageFactory.getManager(metadata);
-    const currentHeadFreePageId = metadataManager.getFreePageId(metadata);
     const emptyPageManager = this.pageFactory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_EMPTY);
     const emptyPage = emptyPageManager.create(this.pageSize, pageId);
-    emptyPageManager.setNextPageId(emptyPage, currentHeadFreePageId);
-    await this.setPage(pageId, emptyPage, tx);
+    const headFreePageId = metadataManager.getFreePageId(metadata);
+    if (headFreePageId === -1 || pageId < headFreePageId) {
+      emptyPageManager.setNextPageId(emptyPage, headFreePageId);
+      await this.setPage(pageId, emptyPage, tx);
+      metadataManager.setFreePageId(metadata, pageId);
+    } else {
+      let prevPageId = headFreePageId;
+      let prevPage = await this.get(prevPageId, tx);
+      let prevManager = this.pageFactory.getManager(prevPage);
+      let nextPageId = prevManager.getNextPageId(prevPage);
+      while (nextPageId !== -1 && nextPageId < pageId) {
+        prevPageId = nextPageId;
+        prevPage = await this.get(prevPageId, tx);
+        prevManager = this.pageFactory.getManager(prevPage);
+        nextPageId = prevManager.getNextPageId(prevPage);
+      }
+      emptyPageManager.setNextPageId(emptyPage, nextPageId);
+      await this.setPage(pageId, emptyPage, tx);
+      prevManager.setNextPageId(prevPage, pageId);
+      await this.setPage(prevPageId, prevPage, tx);
+    }
     await this.updateBitmap(pageId, true, tx);
-    metadataManager.setFreePageId(metadata, pageId);
     await this.setPage(0, metadata, tx);
   }
   /**
@@ -9177,7 +9351,9 @@ var RowTableEngine = class {
       }
       pageGroupMap.get(pageId).push({ pk: pair.pk, slotIndex, index: pair.index });
     }
-    await Promise.all(Array.from(pageGroupMap).map(async ([pageId, items]) => {
+    const sortedPageIds = Array.from(pageGroupMap.keys()).sort((a, b) => a - b);
+    await Promise.all(sortedPageIds.map(async (pageId) => {
+      const items = pageGroupMap.get(pageId);
       const page = await this.pfs.get(pageId, tx);
       if (!this.factory.isDataPage(page)) {
         throw new Error(`Page ${pageId} is not a data page`);
@@ -9538,6 +9714,7 @@ var DataplyAPI = class {
     return Object.assign({
       pageSize: 8192,
       pageCacheCapacity: 1e4,
+      pagePreallocationCount: 1e3,
       wal: null,
       walCheckpointThreshold: 1e3
     }, options);

package/dist/types/core/PageFileSystem.d.ts

@@ -107,7 +107,7 @@ export declare class PageFileSystem {
     /**
      * Appends and inserts a new page.
      * If a free page is available in the free list, it reuses it.
-     * Otherwise, it appends a new page to the end of the file.
+     * Otherwise, it preallocates `pagePreallocationCount` pages to support sequential reads.
      * @returns Created or reused page ID
      */
     appendNewPage(pageType: number | undefined, tx: Transaction): Promise<number>;
@@ -127,7 +127,8 @@ export declare class PageFileSystem {
     freeChain(startPageId: number, tx: Transaction): Promise<void>;
     /**
      * Frees the page and marks it as available in the bitmap.
-     * It also adds the page to the linked list of free pages in metadata.
+     * Inserts the page into the free list in ascending order by page ID
+     * to support HDD sequential reads.
      * @param pageId Page ID
      * @param tx Transaction
      */

package/dist/types/types/index.d.ts

@@ -13,6 +13,11 @@ export interface DataplyOptions {
      * Default is 10000.
      */
     pageCacheCapacity?: number;
+    /**
+     * The number of pages to preallocate when creating a new page.
+     * Default is 1000.
+     */
+    pagePreallocationCount?: number;
     /**
      * The total number of pages written to the WAL before automatically clearing it.
      * Default is 1000.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dataply",
-  "version": "0.0.24-alpha.0",
+  "version": "0.0.24-alpha.2",
   "description": "A lightweight storage engine for Node.js with support for MVCC, WAL.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -49,6 +49,6 @@
     "hookall": "^2.2.0",
     "mvcc-api": "^1.3.4",
     "ryoiki": "^1.2.0",
-    "serializable-bptree": "^8.3.1"
+    "serializable-bptree": "^8.3.2"
   }
 }