document-dataply 0.0.4-alpha.5 → 0.0.4

package/dist/cjs/index.js

@@ -8339,7 +8339,7 @@ var require_cjs = __commonJS({
         this.maxBodySize = this.pfs.pageSize - DataPageManager.CONSTANT.SIZE_PAGE_HEADER;
         this.order = this.getOptimalOrder(pfs.pageSize, IndexPageManager.CONSTANT.SIZE_KEY, IndexPageManager.CONSTANT.SIZE_VALUE);
         this.strategy = new RowIdentifierStrategy(this.order, pfs, txContext);
-        const budget = import_node_os.default.freemem() * 0.05;
+        const budget = import_node_os.default.freemem() * 0.1;
         const nodeMemory = this.order * 24 + 256;
         const capacity = Math.max(1e3, Math.min(1e6, Math.floor(budget / nodeMemory)));
         this.bptree = new BPTreeAsync2(
@@ -9616,6 +9616,71 @@ async function catchPromise(promise) {
   return promise.then((res) => [void 0, res]).catch((reason) => [reason]);
 }
 
+// src/utils/heap.ts
+var BinaryHeap = class {
+  constructor(comparator) {
+    this.comparator = comparator;
+  }
+  heap = [];
+  get size() {
+    return this.heap.length;
+  }
+  peek() {
+    return this.heap[0];
+  }
+  push(value) {
+    this.heap.push(value);
+    this.bubbleUp(this.heap.length - 1);
+  }
+  pop() {
+    if (this.size === 0) return void 0;
+    const top = this.heap[0];
+    const bottom = this.heap.pop();
+    if (this.size > 0) {
+      this.heap[0] = bottom;
+      this.sinkDown(0);
+    }
+    return top;
+  }
+  /**
+   * Replace the root element with a new value and re-heapify.
+   * Faster than pop() followed by push().
+   */
+  replace(value) {
+    const top = this.heap[0];
+    this.heap[0] = value;
+    this.sinkDown(0);
+    return top;
+  }
+  toArray() {
+    return [...this.heap];
+  }
+  bubbleUp(index) {
+    while (index > 0) {
+      const parentIndex = Math.floor((index - 1) / 2);
+      if (this.comparator(this.heap[index], this.heap[parentIndex]) >= 0) break;
+      [this.heap[index], this.heap[parentIndex]] = [this.heap[parentIndex], this.heap[index]];
+      index = parentIndex;
+    }
+  }
+  sinkDown(index) {
+    while (true) {
+      let smallest = index;
+      const left = 2 * index + 1;
+      const right = 2 * index + 2;
+      if (left < this.size && this.comparator(this.heap[left], this.heap[smallest]) < 0) {
+        smallest = left;
+      }
+      if (right < this.size && this.comparator(this.heap[right], this.heap[smallest]) < 0) {
+        smallest = right;
+      }
+      if (smallest === index) break;
+      [this.heap[index], this.heap[smallest]] = [this.heap[smallest], this.heap[index]];
+      index = smallest;
+    }
+  }
+};
+
 // src/core/document.ts
 var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
   indices = {};
@@ -10242,6 +10307,7 @@ var DocumentDataply = class _DocumentDataply {
       const keySet = await self.getKeys(query, orderByField, sortOrder);
       const keys = new Uint32Array(keySet);
       const totalKeys = keys.length;
+      if (totalKeys === 0) return;
       const selectivity = await self.getSelectivityCandidate(
         self.verboseQuery(query),
         orderByField
@@ -10252,29 +10318,70 @@ var DocumentDataply = class _DocumentDataply {
       }
       let CHUNK_SIZE = 100;
       if (!isDriverOrderByField && orderByField) {
+        const isTopK = limit !== Infinity;
+        const heapSizeLimit = isTopK ? limit + offset : Infinity;
+        const heapComparator = (a, b) => {
+          const aVal = a[orderByField] ?? a._id;
+          const bVal = b[orderByField] ?? b._id;
+          const cmp = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
+          return sortOrder === "asc" ? -cmp : cmp;
+        };
+        const heap = isTopK ? new BinaryHeap(heapComparator) : null;
         const results = [];
         let i = 0;
-        while (i < totalKeys) {
-          const chunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
-          const stringifiedResults = await self.api.selectMany(chunk, false, tx2);
-          if (i === 0) {
-            let totalBytes = 0;
-            let count = 0;
-            for (const s of stringifiedResults) {
-              if (s) {
-                totalBytes += s.length;
-                count++;
-              }
+        const firstChunk = Array.from(keys.subarray(i, i + 100));
+        const firstResults = await self.api.selectMany(firstChunk, false, tx2);
+        let totalBytes = 0;
+        let count = 0;
+        for (const s of firstResults) {
+          if (!s) continue;
+          totalBytes += s.length;
+          count++;
+          const doc = JSON.parse(s);
+          if (heap) {
+            if (heap.size < heapSizeLimit) {
+              heap.push(doc);
+            } else if (heapComparator(doc, heap.peek()) > 0) {
+              heap.replace(doc);
             }
-            const avgSize = count > 0 ? totalBytes / count : 1024;
-            CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+          } else {
+            results.push(doc);
+          }
+        }
+        const avgSize = count > 0 ? totalBytes / count : 1024;
+        CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+        i += firstChunk.length;
+        let nextSortChunkPromise = null;
+        if (i < totalKeys) {
+          const nextChunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
+          nextSortChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+          i += nextChunk.length;
+        }
+        while (nextSortChunkPromise) {
+          const stringifiedResults = await nextSortChunkPromise;
+          if (i < totalKeys) {
+            const nextChunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
+            nextSortChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+            i += nextChunk.length;
+          } else {
+            nextSortChunkPromise = null;
           }
           for (const stringified of stringifiedResults) {
-            if (stringified) results.push(JSON.parse(stringified));
+            if (!stringified) continue;
+            const doc = JSON.parse(stringified);
+            if (heap) {
+              if (heap.size < heapSizeLimit) {
+                heap.push(doc);
+              } else if (heapComparator(doc, heap.peek()) > 0) {
+                heap.replace(doc);
+              }
+            } else {
+              results.push(doc);
+            }
           }
-          i += chunk.length;
         }
-        results.sort((a, b) => {
+        const finalResults = heap ? heap.toArray() : results;
+        finalResults.sort((a, b) => {
           const aVal = a[orderByField] ?? a._id;
           const bVal = b[orderByField] ?? b._id;
           const cmp = aVal < bVal ? -1 : aVal > bVal ? 1 : 0;
@@ -10282,41 +10389,60 @@ var DocumentDataply = class _DocumentDataply {
         });
         const start = offset;
         const end = limit === Infinity ? void 0 : start + limit;
-        const limitedResults = results.slice(start, end);
+        const limitedResults = finalResults.slice(start, end);
         for (const doc of limitedResults) {
           yield doc;
         }
       } else {
         let yieldedCount = 0;
         let i = offset;
-        let isFirst = true;
-        while (i < totalKeys && yieldedCount < limit) {
-          const currentChunkLimit = isFirst ? 100 : CHUNK_SIZE;
-          const remainingLimit = limit - yieldedCount;
-          const pksToFetchCount = Math.min(currentChunkLimit, remainingLimit);
-          const chunk = Array.from(keys.subarray(i, i + pksToFetchCount));
-          const stringifiedResults = await self.api.selectMany(chunk, false, tx2);
-          if (isFirst) {
-            let totalBytes = 0;
-            let count = 0;
-            for (const s of stringifiedResults) {
-              if (s) {
-                totalBytes += s.length;
-                count++;
-              }
+        if (i >= totalKeys || yieldedCount >= limit) return;
+        const pksToFetchCount = Math.min(100, limit - yieldedCount);
+        const firstChunk = Array.from(keys.subarray(i, i + pksToFetchCount));
+        const firstResults = await self.api.selectMany(firstChunk, false, tx2);
+        let totalBytes = 0;
+        let count = 0;
+        for (const s of firstResults) {
+          if (!s) continue;
+          yield JSON.parse(s);
+          yieldedCount++;
+          totalBytes += s.length;
+          count++;
+          if (yieldedCount >= limit) break;
+        }
+        const avgSize = count > 0 ? totalBytes / count : 1024;
+        CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+        i += firstChunk.length;
+        let nextStreamChunkPromise = null;
+        if (i < totalKeys && yieldedCount < limit) {
+          const nextPksToFetchCount = Math.min(CHUNK_SIZE, limit - yieldedCount);
+          const nextChunk = Array.from(keys.subarray(i, i + nextPksToFetchCount));
+          nextStreamChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+          i += nextChunk.length;
+        }
+        while (nextStreamChunkPromise) {
+          const stringifiedResults = await nextStreamChunkPromise;
+          if (i < totalKeys && yieldedCount < limit) {
+            const nextPksToFetchCount = Math.min(CHUNK_SIZE, limit - (yieldedCount + stringifiedResults.filter(Boolean).length));
+            if (nextPksToFetchCount > 0) {
+              const nextChunk = Array.from(keys.subarray(i, i + nextPksToFetchCount));
+              nextStreamChunkPromise = self.api.selectMany(nextChunk, false, tx2);
+              i += nextChunk.length;
+            } else {
+              nextStreamChunkPromise = null;
             }
-            const avgSize = count > 0 ? totalBytes / count : 1024;
-            CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
-            isFirst = false;
+          } else {
+            nextStreamChunkPromise = null;
           }
           for (const stringified of stringifiedResults) {
-            if (stringified) {
-              yield JSON.parse(stringified);
-              yieldedCount++;
-              if (yieldedCount >= limit) break;
+            if (!stringified) continue;
+            yield JSON.parse(stringified);
+            yieldedCount++;
+            if (yieldedCount >= limit) {
+              nextStreamChunkPromise = null;
+              break;
             }
           }
-          i += chunk.length;
         }
       }
     }, tx);

package/dist/types/utils/heap.d.ts

@@ -0,0 +1,21 @@
+export type Comparator<T> = (a: T, b: T) => number;
+/**
+ * A simple Binary Heap implementation.
+ */
+export declare class BinaryHeap<T> {
+    private comparator;
+    private heap;
+    constructor(comparator: Comparator<T>);
+    get size(): number;
+    peek(): T | undefined;
+    push(value: T): void;
+    pop(): T | undefined;
+    /**
+     * Replace the root element with a new value and re-heapify.
+     * Faster than pop() followed by push().
+     */
+    replace(value: T): T | undefined;
+    toArray(): T[];
+    private bubbleUp;
+    private sinkDown;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.4-alpha.5",
+  "version": "0.0.4",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.20-alpha.6"
+    "dataply": "^0.0.20"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",