dataply 0.0.24 → 0.0.25-alpha.1

package/dist/cjs/index.js

@@ -6463,65 +6463,22 @@ function crc32(buf) {
 }
 
 // src/utils/array.ts
-function calcThreshold(sortedGaps, n) {
-  const gLen = sortedGaps.length;
-  if (gLen === 0) return 0;
-  const median = sortedGaps[Math.floor(gLen * 0.5)];
-  const q1 = sortedGaps[Math.floor(gLen * 0.25)];
-  const q3 = sortedGaps[Math.floor(gLen * 0.75)];
-  const iqr = q3 - q1;
-  const logN = Math.max(1, Math.log10(n));
-  if (iqr > 0) {
-    const threshold2 = q3 + iqr * 1.5 * logN;
-    const minJump = Math.max(median * 5, 20);
-    return Math.max(threshold2, minJump);
-  }
-  const baseGap = median > 0 ? median : 1;
-  const p90 = sortedGaps[Math.floor(gLen * 0.9)];
-  if (p90 > baseGap) {
-    const threshold2 = baseGap + (p90 - baseGap) * 0.5 * logN;
-    return Math.max(threshold2, baseGap * 5, 20);
-  }
-  let mean = 0;
-  for (let i = 0; i < gLen; i++) mean += sortedGaps[i];
-  mean /= gLen;
-  let variance = 0;
-  for (let i = 0; i < gLen; i++) {
-    const d = sortedGaps[i] - mean;
-    variance += d * d;
-  }
-  const stddev = Math.sqrt(variance / gLen);
-  if (stddev === 0) {
-    return baseGap * 2;
-  }
-  const threshold = mean + stddev * logN;
-  return Math.max(threshold, baseGap * 5, 20);
-}
-function clusterNumbers(numbers, maxGap) {
+function clusterNumbersByPagination(numbers, pagingSize, startPageId = 0) {
   const n = numbers.length;
   if (n === 0) return [];
   if (n === 1) return [new Float64Array([numbers[0]])];
   const sorted = (numbers instanceof Float64Array ? numbers.slice() : Float64Array.from(numbers)).sort();
-  const gaps = new Float64Array(n - 1);
-  for (let i = 0, len = n - 1; i < len; i++) {
-    gaps[i] = sorted[i + 1] - sorted[i];
-  }
-  const sortedGaps = gaps.slice().sort();
-  let threshold;
-  if (maxGap !== void 0) {
-    threshold = maxGap;
-  } else {
-    threshold = calcThreshold(sortedGaps, n);
-  }
   const clusters = [];
-  let clusterStart = 0;
+  let start = 0;
   for (let i = 0, len = n - 1; i < len; i++) {
-    if (gaps[i] > threshold) {
-      clusters.push(sorted.subarray(clusterStart, i + 1));
-      clusterStart = i + 1;
+    const paginationIndex = Math.floor((sorted[i] - startPageId) / pagingSize);
+    const nextPaginationIndex = Math.floor((sorted[i + 1] - startPageId) / pagingSize);
+    if (paginationIndex !== nextPaginationIndex) {
+      clusters.push(sorted.subarray(start, i + 1));
+      start = i + 1;
     }
   }
-  clusters.push(sorted.subarray(clusterStart));
+  clusters.push(sorted.subarray(start));
   return clusters;
 }
 
@@ -9205,10 +9162,15 @@ var RowTableEngine = class {
     }
     const metadataPage = await this.pfs.getMetadata(tx);
     const manager = this.factory.getManagerFromType(MetadataPageManager.CONSTANT.PAGE_TYPE_METADATA);
+    const pageSize = manager.getPageSize(metadataPage);
+    const pageCount = manager.getPageCount(metadataPage);
+    const rowCount = manager.getRowCount(metadataPage);
+    const usage = pageCount / Math.pow(2, 32);
     return {
-      pageSize: manager.getPageSize(metadataPage),
-      pageCount: manager.getPageCount(metadataPage),
-      rowCount: manager.getRowCount(metadataPage)
+      pageSize,
+      pageCount,
+      rowCount,
+      usage
     };
   }
   /**
@@ -9479,19 +9441,34 @@ var RowTableEngine = class {
    * @returns Array of raw data of the rows in the same order as input PKs
    */
   async selectMany(pks, tx) {
+    const collections = await this.collectItemsByPage(pks, tx);
+    return this.fetchRowsByRids(collections, pks.length, tx);
+  }
+  /**
+   * Collects items by page ID to minimize I/O.
+   * @param pks Array of PKs to look up
+   * @param tx Transaction
+   * @returns Map of page ID to array of {pk, slotIndex, index} pairs
+   */
+  async collectItemsByPage(pks, tx) {
     if (pks.length === 0) {
-      return [];
+      return /* @__PURE__ */ new Map();
     }
     const pkIndexMap = /* @__PURE__ */ new Map();
     for (let i = 0, len = pks.length; i < len; i++) {
       pkIndexMap.set(pks[i], i);
     }
-    const validCount = pks.length;
-    const pkArray = new Float64Array(validCount).fill(0);
-    const ridArray = new Float64Array(validCount).fill(0);
-    const indexArray = new Float64Array(validCount).fill(0);
     const btx = await this.getBPTreeTransaction(tx);
-    const clusters = clusterNumbers(pks, this.order / 2);
+    const clusters = clusterNumbersByPagination(pks, this.order, 1);
+    const collections = /* @__PURE__ */ new Map();
+    const insertToCollections = (pk, rid, index) => {
+      const slotIndex = rid % 65536;
+      const pageId = Math.floor(rid / 65536);
+      if (!collections.has(pageId)) {
+        collections.set(pageId, []);
+      }
+      collections.get(pageId).push({ pk, slotIndex, index });
+    };
     for (let i = 0, len = clusters.length; i < len; i++) {
       const cluster = clusters[i];
       const minPk = cluster[0];
@@ -9502,9 +9479,7 @@ var RowTableEngine = class {
           const rid = keys.values().next().value;
           const index = pkIndexMap.get(minPk);
           if (index !== void 0) {
-            pkArray[index] = minPk;
-            ridArray[index] = rid;
-            indexArray[index] = index;
+            insertToCollections(minPk, rid, index);
           }
         }
         continue;
@@ -9513,13 +9488,11 @@ var RowTableEngine = class {
       for await (const [rid, pk] of stream) {
         const index = pkIndexMap.get(pk);
         if (index !== void 0) {
-          pkArray[index] = pk;
-          ridArray[index] = rid;
-          indexArray[index] = index;
+          insertToCollections(pk, rid, index);
         }
       }
     }
-    return this.fetchRowsByRids(validCount, pkArray, ridArray, indexArray, tx);
+    return collections;
   }
   /**
    * Fetches multiple rows by their RID and PK combinations, grouping by page ID to minimize I/O.
@@ -9527,25 +9500,12 @@ var RowTableEngine = class {
    * @param tx Transaction
    * @returns Array of row data in the same order as input PKs
    */
-  async fetchRowsByRids(validCount, pkArray, ridArray, indexArray, tx) {
-    const result = new Array(validCount).fill(null);
-    if (validCount === 0) return result;
-    const pageGroupMap = /* @__PURE__ */ new Map();
-    for (let i = 0; i < validCount; i++) {
-      const pk = pkArray[i];
-      const rid = ridArray[i];
-      const index = indexArray[i];
-      if (pk === 0 && rid === 0 && index === 0) continue;
-      const slotIndex = rid % 65536;
-      const pageId = Math.floor(rid / 65536);
-      if (!pageGroupMap.has(pageId)) {
-        pageGroupMap.set(pageId, []);
-      }
-      pageGroupMap.get(pageId).push({ pk, slotIndex, index });
-    }
-    const sortedPageIds = Array.from(pageGroupMap.keys()).sort((a, b) => a - b);
+  async fetchRowsByRids(collections, itemsCount, tx) {
+    const result = new Array(itemsCount).fill(null);
+    if (itemsCount === 0) return result;
+    const sortedPageIds = Array.from(collections.keys()).sort((a, b) => a - b);
     await Promise.all(sortedPageIds.map(async (pageId) => {
-      const items = pageGroupMap.get(pageId);
+      const items = collections.get(pageId);
       const page = await this.pfs.get(pageId, tx);
       if (!this.factory.isDataPage(page)) {
         throw new Error(`Page ${pageId} is not a data page`);
@@ -10302,9 +10262,11 @@ var DataplyAPI = class {
     if (!this.initialized) {
       throw new Error("Dataply instance is not initialized");
     }
-    return this.hook.trigger("close", void 0, async () => {
-      await this.pfs.close();
-      import_node_fs3.default.closeSync(this.fileHandle);
+    return this.runWithDefaultWrite(() => {
+      return this.hook.trigger("close", void 0, async () => {
+        await this.pfs.close();
+        import_node_fs3.default.closeSync(this.fileHandle);
+      });
     });
   }
 };

package/dist/types/core/RowTableEngine.d.ts

@@ -131,6 +131,17 @@ export declare class RowTableEngine {
      * @returns Array of raw data of the rows in the same order as input PKs
      */
     selectMany(pks: number[] | Float64Array, tx: Transaction): Promise<(Uint8Array | null)[]>;
+    /**
+     * Collects items by page ID to minimize I/O.
+     * @param pks Array of PKs to look up
+     * @param tx Transaction
+     * @returns Map of page ID to array of {pk, slotIndex, index} pairs
+     */
+    collectItemsByPage(pks: number[] | Float64Array, tx: Transaction): Promise<Map<number, {
+        pk: number;
+        slotIndex: number;
+        index: number;
+    }[]>>;
     /**
      * Fetches multiple rows by their RID and PK combinations, grouping by page ID to minimize I/O.
      * @param pkRidPairs Array of {pk, rid} pairs

package/dist/types/types/index.d.ts

@@ -37,6 +37,11 @@ export interface DataplyMetadata {
      * The total number of data rows in the dataply.
      */
     rowCount: number;
+    /**
+     * The usage of the dataply. It is calculated based on the remaining page capacity.
+     * The value is between 0 and 1.
+     */
+    usage: number;
 }
 export type DataPage = Uint8Array & {
     __pageType: 'data';

package/dist/types/utils/array.d.ts

@@ -8,5 +8,6 @@ export declare function getMinMaxValue(array: SupportedNumberArray): [number, nu
  * @param maxGap Optional fixed gap threshold. If not provided, it is calculated automatically.
  * @returns Array of clusters
  */
-export declare function clusterNumbers(numbers: number[] | Float64Array, maxGap?: number): Float64Array[];
+export declare function clusterNumbersByGap(numbers: number[] | Float64Array, maxGap?: number): Float64Array[];
+export declare function clusterNumbersByPagination(numbers: number[] | Float64Array, pagingSize: number, startPageId?: number): Float64Array[];
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dataply",
-  "version": "0.0.24",
+  "version": "0.0.25-alpha.1",
   "description": "A lightweight storage engine for Node.js with support for MVCC, WAL.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",