document-dataply 0.0.4-alpha.4 → 0.0.4

package/dist/cjs/index.js

@@ -7611,9 +7611,7 @@ var require_cjs = __commonJS({
         }
         const cached = this.cache.get(pageId);
         if (cached) {
-          const copy = new Uint8Array(this.pageSize);
-          copy.set(cached);
-          return copy;
+          return cached;
         }
         const buffer = new Uint8Array(this.pageSize);
         const pageStartPos = pageId * this.pageSize;
@@ -8101,6 +8099,7 @@ var require_cjs = __commonJS({
         }
       }
     };
+    var import_node_os = __toESM2(require("node:os"));
     var TextCodec = class _TextCodec {
       static TextEncoder = new TextEncoder();
       static TextDecoder = new TextDecoder();
@@ -8340,11 +8339,14 @@ var require_cjs = __commonJS({
         this.maxBodySize = this.pfs.pageSize - DataPageManager.CONSTANT.SIZE_PAGE_HEADER;
         this.order = this.getOptimalOrder(pfs.pageSize, IndexPageManager.CONSTANT.SIZE_KEY, IndexPageManager.CONSTANT.SIZE_VALUE);
         this.strategy = new RowIdentifierStrategy(this.order, pfs, txContext);
+        const budget = import_node_os.default.freemem() * 0.1;
+        const nodeMemory = this.order * 24 + 256;
+        const capacity = Math.max(1e3, Math.min(1e6, Math.floor(budget / nodeMemory)));
         this.bptree = new BPTreeAsync2(
           this.strategy,
           new NumericComparator(),
           {
-            capacity: this.options.pageCacheCapacity
+            capacity
           }
         );
       }
@@ -8711,6 +8713,30 @@ var require_cjs = __commonJS({
         }
         return this.fetchRowByRid(pk, rid, tx);
       }
+      /**
+       * Selects multiple rows by their PKs in a single B+ Tree traversal.
+       * @param pks Array of PKs to look up
+       * @param tx Transaction
+       * @returns Array of raw data of the rows in the same order as input PKs
+       */
+      async selectMany(pks, tx) {
+        if (pks.length === 0) {
+          return [];
+        }
+        const minPk = Math.min(...pks);
+        const maxPk = Math.max(...pks);
+        const pkSet = new Set(pks);
+        const resultMap = /* @__PURE__ */ new Map();
+        const btx = await this.getBPTreeTransaction(tx);
+        const stream = btx.whereStream({ gte: minPk, lte: maxPk });
+        for await (const [rid, pk] of stream) {
+          if (pkSet.has(pk)) {
+            const rowData = await this.fetchRowByRid(pk, rid, tx);
+            resultMap.set(pk, rowData);
+          }
+        }
+        return pks.map((pk) => resultMap.get(pk) ?? null);
+      }
       async fetchRowByRid(pk, rid, tx) {
         this.keyManager.setBufferFromKey(rid, this.ridBuffer);
         const pageId = this.keyManager.getPageId(this.ridBuffer);
@@ -9336,6 +9362,19 @@ var require_cjs = __commonJS({
           return this.textCodec.decode(data);
         }, tx);
       }
+      async selectMany(pks, asRaw = false, tx) {
+        if (!this.initialized) {
+          throw new Error("Dataply instance is not initialized");
+        }
+        return this.runWithDefault(async (tx2) => {
+          const results = await this.rowTableEngine.selectMany(pks, tx2);
+          return results.map((data) => {
+            if (data === null) return null;
+            if (asRaw) return data;
+            return this.textCodec.decode(data);
+          });
+        }, tx);
+      }
       /**
        * Closes the dataply file.
        */
@@ -9424,6 +9463,9 @@ var require_cjs = __commonJS({
       async select(pk, asRaw = false, tx) {
         return this.api.select(pk, asRaw, tx);
       }
+      async selectMany(pks, asRaw = false, tx) {
+        return this.api.selectMany(pks, asRaw, tx);
+      }
       /**
        * Closes the dataply file.
        */
@@ -9484,6 +9526,7 @@ __export(src_exports, {
 module.exports = __toCommonJS(src_exports);
 
 // src/core/document.ts
+var os = __toESM(require("node:os"));
 var import_dataply3 = __toESM(require_cjs());
 
 // src/core/bptree/documentStrategy.ts
@@ -9573,6 +9616,71 @@ async function catchPromise(promise) {
   return promise.then((res) => [void 0, res]).catch((reason) => [reason]);
 }
 
+// src/utils/heap.ts
+var BinaryHeap = class {
+  constructor(comparator) {
+    this.comparator = comparator;
+  }
+  heap = [];
+  get size() {
+    return this.heap.length;
+  }
+  peek() {
+    return this.heap[0];
+  }
+  push(value) {
+    this.heap.push(value);
+    this.bubbleUp(this.heap.length - 1);
+  }
+  pop() {
+    if (this.size === 0) return void 0;
+    const top = this.heap[0];
+    const bottom = this.heap.pop();
+    if (this.size > 0) {
+      this.heap[0] = bottom;
+      this.sinkDown(0);
+    }
+    return top;
+  }
+  /**
+   * Replace the root element with a new value and re-heapify.
+   * Faster than pop() followed by push().
+   */
+  replace(value) {
+    const top = this.heap[0];
+    this.heap[0] = value;
+    this.sinkDown(0);
+    return top;
+  }
+  toArray() {
+    return [...this.heap];
+  }
+  bubbleUp(index) {
+    while (index > 0) {
+      const parentIndex = Math.floor((index - 1) / 2);
+      if (this.comparator(this.heap[index], this.heap[parentIndex]) >= 0) break;
+      [this.heap[index], this.heap[parentIndex]] = [this.heap[parentIndex], this.heap[index]];
+      index = parentIndex;
+    }
+  }
+  sinkDown(index) {
+    while (true) {
+      let smallest = index;
+      const left = 2 * index + 1;
+      const right = 2 * index + 2;
+      if (left < this.size && this.comparator(this.heap[left], this.heap[smallest]) < 0) {
+        smallest = left;
+      }
+      if (right < this.size && this.comparator(this.heap[right], this.heap[smallest]) < 0) {
+        smallest = right;
+      }
+      if (smallest === index) break;
+      [this.heap[index], this.heap[smallest]] = [this.heap[smallest], this.heap[index]];
+      index = smallest;
+    }
+  }
+};
+
 // src/core/document.ts
 var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
   indices = {};
@@ -10196,7 +10304,10 @@ var DocumentDataply = class _DocumentDataply {
     } = options;
     const self = this;
     const stream = this.api.streamWithDefault(async function* (tx2) {
-      const keys = await self.getKeys(query, orderByField, sortOrder);
+      const keySet = await self.getKeys(query, orderByField, sortOrder);
+      const keys = new Uint32Array(keySet);
+      const totalKeys = keys.length;
+      if (totalKeys === 0) return;
       const selectivity = await self.getSelectivityCandidate(
         self.verboseQuery(query),
         orderByField
@@ -10205,14 +10316,72 @@ var DocumentDataply = class _DocumentDataply {
       if (selectivity) {
         selectivity.rollback();
       }
+      let CHUNK_SIZE = 100;
       if (!isDriverOrderByField && orderByField) {
+        const isTopK = limit !== Infinity;
+        const heapSizeLimit = isTopK ? limit + offset : Infinity;
+        const heapComparator = (a, b) => {
+          const aVal = a[orderByField] ?? a._id;
+          const bVal = b[orderByField] ?? b._id;
+          const cmp = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
+          return sortOrder === "asc" ? -cmp : cmp;
+        };
+        const heap = isTopK ? new BinaryHeap(heapComparator) : null;
         const results = [];
-        for (const key of keys) {
-          const stringified = await self.api.select(key, false, tx2);
-          if (!stringified) continue;
-          results.push(JSON.parse(stringified));
+        let i = 0;
+        const firstChunk = Array.from(keys.subarray(i, i + 100));
+        const firstResults = await self.api.selectMany(firstChunk, false, tx2);
+        let totalBytes = 0;
+        let count = 0;
+        for (const s of firstResults) {
+          if (!s) continue;
+          totalBytes += s.length;
+          count++;
+          const doc = JSON.parse(s);
+          if (heap) {
+            if (heap.size < heapSizeLimit) {
+              heap.push(doc);
+            } else if (heapComparator(doc, heap.peek()) > 0) {
+              heap.replace(doc);
+            }
+          } else {
+            results.push(doc);
+          }
+        }
+        const avgSize = count > 0 ? totalBytes / count : 1024;
+        CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+        i += firstChunk.length;
+        let nextSortChunkPromise = null;
+        if (i < totalKeys) {
+          const nextChunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
+          nextSortChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+          i += nextChunk.length;
+        }
+        while (nextSortChunkPromise) {
+          const stringifiedResults = await nextSortChunkPromise;
+          if (i < totalKeys) {
+            const nextChunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
+            nextSortChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+            i += nextChunk.length;
+          } else {
+            nextSortChunkPromise = null;
+          }
+          for (const stringified of stringifiedResults) {
+            if (!stringified) continue;
+            const doc = JSON.parse(stringified);
+            if (heap) {
+              if (heap.size < heapSizeLimit) {
+                heap.push(doc);
+              } else if (heapComparator(doc, heap.peek()) > 0) {
+                heap.replace(doc);
+              }
+            } else {
+              results.push(doc);
+            }
+          }
         }
-        results.sort((a, b) => {
+        const finalResults = heap ? heap.toArray() : results;
+        finalResults.sort((a, b) => {
           const aVal = a[orderByField] ?? a._id;
           const bVal = b[orderByField] ?? b._id;
           const cmp = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
@@ -10220,24 +10389,60 @@ var DocumentDataply = class _DocumentDataply {
         });
         const start = offset;
         const end = limit === Infinity ? void 0 : start + limit;
-        const limitedResults = results.slice(start, end);
+        const limitedResults = finalResults.slice(start, end);
         for (const doc of limitedResults) {
           yield doc;
         }
       } else {
-        let i = 0;
         let yieldedCount = 0;
-        for (const key of keys) {
+        let i = offset;
+        if (i >= totalKeys || yieldedCount >= limit) return;
+        const pksToFetchCount = Math.min(100, limit - yieldedCount);
+        const firstChunk = Array.from(keys.subarray(i, i + pksToFetchCount));
+        const firstResults = await self.api.selectMany(firstChunk, false, tx2);
+        let totalBytes = 0;
+        let count = 0;
+        for (const s of firstResults) {
+          if (!s) continue;
+          yield JSON.parse(s);
+          yieldedCount++;
+          totalBytes += s.length;
+          count++;
           if (yieldedCount >= limit) break;
-          if (i < offset) {
-            i++;
-            continue;
+        }
+        const avgSize = count > 0 ? totalBytes / count : 1024;
+        CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+        i += firstChunk.length;
+        let nextStreamChunkPromise = null;
+        if (i < totalKeys && yieldedCount < limit) {
+          const nextPksToFetchCount = Math.min(CHUNK_SIZE, limit - yieldedCount);
+          const nextChunk = Array.from(keys.subarray(i, i + nextPksToFetchCount));
+          nextStreamChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+          i += nextChunk.length;
+        }
+        while (nextStreamChunkPromise) {
+          const stringifiedResults = await nextStreamChunkPromise;
+          if (i < totalKeys && yieldedCount < limit) {
+            const nextPksToFetchCount = Math.min(CHUNK_SIZE, limit - (yieldedCount + stringifiedResults.filter(Boolean).length));
+            if (nextPksToFetchCount > 0) {
+              const nextChunk = Array.from(keys.subarray(i, i + nextPksToFetchCount));
+              nextStreamChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+              i += nextChunk.length;
+            } else {
+              nextStreamChunkPromise = null;
+            }
+          } else {
+            nextStreamChunkPromise = null;
+          }
+          for (const stringified of stringifiedResults) {
+            if (!stringified) continue;
+            yield JSON.parse(stringified);
+            yieldedCount++;
+            if (yieldedCount >= limit) {
+              nextStreamChunkPromise = null;
+              break;
+            }
           }
-          const stringified = await self.api.select(key, false, tx2);
-          if (!stringified) continue;
-          yield JSON.parse(stringified);
-          yieldedCount++;
-          i++;
         }
       }
     }, tx);

package/dist/types/utils/heap.d.ts

@@ -0,0 +1,21 @@
+export type Comparator<T> = (a: T, b: T) => number;
+/**
+ * A simple Binary Heap implementation.
+ */
+export declare class BinaryHeap<T> {
+    private comparator;
+    private heap;
+    constructor(comparator: Comparator<T>);
+    get size(): number;
+    peek(): T | undefined;
+    push(value: T): void;
+    pop(): T | undefined;
+    /**
+     * Replace the root element with a new value and re-heapify.
+     * Faster than pop() followed by push().
+     */
+    replace(value: T): T | undefined;
+    toArray(): T[];
+    private bubbleUp;
+    private sinkDown;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.4-alpha.4",
+  "version": "0.0.4",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.20-alpha.4"
+    "dataply": "^0.0.20"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",
@@ -51,4 +51,4 @@
     "ts-jest": "^29.4.6",
     "typescript": "^5.9.3"
   }
-}
+}