document-dataply 0.0.9-alpha.4 → 0.0.9-alpha.6

package/dist/cjs/index.js

@@ -1580,83 +1580,217 @@ var require_cjs = __commonJS({
           return regexp.test(nodeValue);
         }
       };
-      verifierStartNode = {
-        gt: (v) => this.insertableNodeByPrimary(v),
-        gte: (v) => this.insertableNodeByPrimary(v),
-        lt: (v) => this.insertableNodeByPrimary(v),
-        lte: (v) => this.insertableRightestNodeByPrimary(v),
-        equal: (v) => this.insertableNodeByPrimary(v),
-        notEqual: (v) => this.leftestNode(),
-        or: (v) => this.insertableNodeByPrimary(this.lowestPrimaryValue(this.ensureValues(v))),
-        primaryGt: (v) => this.insertableNodeByPrimary(v),
-        primaryGte: (v) => this.insertableNodeByPrimary(v),
-        primaryLt: (v) => this.insertableNodeByPrimary(v),
-        primaryLte: (v) => this.insertableRightestNodeByPrimary(v),
-        primaryEqual: (v) => this.insertableNodeByPrimary(v),
-        primaryNotEqual: (v) => this.leftestNode(),
-        primaryOr: (v) => this.insertableNodeByPrimary(this.lowestPrimaryValue(this.ensureValues(v))),
-        like: (v) => this.leftestNode()
-      };
-      verifierEndNode = {
-        gt: (v) => null,
-        gte: (v) => null,
-        lt: (v) => null,
-        lte: (v) => null,
-        equal: (v) => this.insertableEndNode(v, this.verifierDirection.equal),
-        notEqual: (v) => null,
-        or: (v) => this.insertableEndNode(
-          this.highestValue(this.ensureValues(v)),
-          this.verifierDirection.or
-        ),
-        primaryGt: (v) => null,
-        primaryGte: (v) => null,
-        primaryLt: (v) => null,
-        primaryLte: (v) => null,
-        primaryEqual: (v) => this.insertableRightestEndNodeByPrimary(v),
-        primaryNotEqual: (v) => null,
-        primaryOr: (v) => this.insertableRightestEndNodeByPrimary(
-          this.highestPrimaryValue(this.ensureValues(v))
-        ),
-        like: (v) => null
-      };
-      verifierDirection = {
-        gt: 1,
-        gte: 1,
-        lt: -1,
-        lte: -1,
-        equal: 1,
-        notEqual: 1,
-        or: 1,
-        primaryGt: 1,
-        primaryGte: 1,
-        primaryLt: -1,
-        primaryLte: -1,
-        primaryEqual: 1,
-        primaryNotEqual: 1,
-        primaryOr: 1,
-        like: 1
-      };
-      /**
-       * Determines whether early termination is allowed for each condition.
-       * When true, the search will stop once a match is found and then a non-match is encountered.
-       * Only applicable for conditions that guarantee contiguous matches in a sorted B+Tree.
-       */
-      verifierEarlyTerminate = {
-        gt: false,
-        gte: false,
-        lt: false,
-        lte: false,
-        equal: true,
-        notEqual: false,
-        or: false,
-        primaryGt: false,
-        primaryGte: false,
-        primaryLt: false,
-        primaryLte: false,
-        primaryEqual: true,
-        primaryNotEqual: false,
-        primaryOr: false,
-        like: false
+      searchConfigs = {
+        gt: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        gte: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        lt: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], 1),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        lte: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], 1),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableRightestNodeByPrimary(v[0]),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        equal: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: (tx, v) => tx.insertableEndNode(v[0], 1),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableEndNode(v[0], 1),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        notEqual: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        or: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(tx.lowestValue(v)),
+            end: (tx, v) => tx.insertableEndNode(tx.highestValue(v), 1),
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx, v) => tx.insertableEndNode(tx.highestValue(v), 1),
+            end: (tx, v) => tx.insertableEndNode(tx.lowestValue(v), -1),
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        primaryGt: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        primaryGte: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        primaryLt: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], 1),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        primaryLte: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: (tx, v) => tx.insertableEndNode(v[0], 1),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableRightestNodeByPrimary(v[0]),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        primaryEqual: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(v[0]),
+            end: (tx, v) => tx.insertableRightestEndNodeByPrimary(v[0]),
+            direction: 1,
+            earlyTerminate: true
+          },
+          desc: {
+            start: (tx, v) => tx.insertableRightestEndNodeByPrimary(v[0]),
+            end: (tx, v) => tx.insertableEndNode(v[0], -1),
+            direction: -1,
+            earlyTerminate: true
+          }
+        },
+        primaryNotEqual: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        primaryOr: {
+          asc: {
+            start: (tx, v) => tx.insertableNodeByPrimary(tx.lowestPrimaryValue(v)),
+            end: (tx, v) => tx.insertableRightestEndNodeByPrimary(tx.highestPrimaryValue(v)),
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx, v) => tx.insertableRightestEndNodeByPrimary(tx.highestPrimaryValue(v)),
+            end: (tx, v) => tx.insertableEndNode(tx.lowestPrimaryValue(v), -1),
+            direction: -1,
+            earlyTerminate: false
+          }
+        },
+        like: {
+          asc: {
+            start: (tx) => tx.leftestNode(),
+            end: () => null,
+            direction: 1,
+            earlyTerminate: false
+          },
+          desc: {
+            start: (tx) => tx.rightestNode(),
+            end: () => null,
+            direction: -1,
+            earlyTerminate: false
+          }
+        }
       };
       /**
        * Priority map for condition types.
@@ -2267,16 +2401,20 @@ var require_cjs = __commonJS({
         const driverKey = this.getDriverKey(condition);
         if (!driverKey) return;
         const value = condition[driverKey];
-        let startNode = this.verifierStartNode[driverKey](value);
-        let endNode = this.verifierEndNode[driverKey](value);
-        let direction = this.verifierDirection[driverKey];
+        const v = this.ensureValues(value);
+        const config = this.searchConfigs[driverKey][order];
+        let startNode = config.start(this, v);
+        let endNode = config.end(this, v);
+        const direction = config.direction;
+        const earlyTerminate = config.earlyTerminate;
+        if (order === "desc" && !startNode) {
+          startNode = this.rightestNode();
+        }
+        if (order === "asc" && !startNode) {
+          startNode = this.leftestNode();
+        }
+        if (!startNode) return;
         const comparator = this.verifierMap[driverKey];
-        const earlyTerminate = this.verifierEarlyTerminate[driverKey];
-        if (order === "desc") {
-          startNode = endNode ?? this.rightestNode();
-          endNode = null;
-          direction *= -1;
-        }
         const generator = this.getPairsGenerator(
           value,
           startNode,
@@ -2288,7 +2426,7 @@ var require_cjs = __commonJS({
         let count = 0;
         const intersection = filterValues && filterValues.size > 0 ? filterValues : null;
         for (const pair of generator) {
-          const [k, v] = pair;
+          const [k, v2] = pair;
           if (intersection && !intersection.has(k)) {
             continue;
           }
@@ -2297,7 +2435,7 @@ var require_cjs = __commonJS({
             if (key === driverKey) continue;
             const verify = this.verifierMap[key];
             const condValue = condition[key];
-            if (!verify(v, condValue)) {
+            if (!verify(v2, condValue)) {
               isMatch = false;
               break;
             }
@@ -3397,16 +3535,20 @@ var require_cjs = __commonJS({
         const driverKey = this.getDriverKey(condition);
         if (!driverKey) return;
         const value = condition[driverKey];
-        let startNode = await this.verifierStartNode[driverKey](value);
-        let endNode = await this.verifierEndNode[driverKey](value);
-        let direction = this.verifierDirection[driverKey];
+        const v = this.ensureValues(value);
+        const config = this.searchConfigs[driverKey][order];
+        let startNode = await config.start(this, v);
+        let endNode = await config.end(this, v);
+        const direction = config.direction;
+        const earlyTerminate = config.earlyTerminate;
+        if (order === "desc" && !startNode) {
+          startNode = await this.rightestNode();
+        }
+        if (order === "asc" && !startNode) {
+          startNode = await this.leftestNode();
+        }
+        if (!startNode) return;
         const comparator = this.verifierMap[driverKey];
-        const earlyTerminate = this.verifierEarlyTerminate[driverKey];
-        if (order === "desc") {
-          startNode = endNode ?? await this.rightestNode();
-          endNode = null;
-          direction *= -1;
-        }
         const generator = this.getPairsGenerator(
           value,
           startNode,
@@ -3418,7 +3560,7 @@ var require_cjs = __commonJS({
         let count = 0;
         const intersection = filterValues && filterValues.size > 0 ? filterValues : null;
         for await (const pair of generator) {
-          const [k, v] = pair;
+          const [k, v2] = pair;
           if (intersection && !intersection.has(k)) {
             continue;
           }
@@ -3427,7 +3569,7 @@ var require_cjs = __commonJS({
             if (key === driverKey) continue;
             const verify = this.verifierMap[key];
             const condValue = condition[key];
-            if (!verify(v, condValue)) {
+            if (!verify(v2, condValue)) {
               isMatch = false;
               break;
             }
@@ -8342,7 +8484,7 @@ var require_cjs = __commonJS({
       /**
        * Appends and inserts a new page.
        * If a free page is available in the free list, it reuses it.
-       * Otherwise, it appends a new page to the end of the file.
+       * Otherwise, it preallocates `pagePreallocationCount` pages to support sequential reads.
        * @returns Created or reused page ID
        */
       async appendNewPage(pageType = PageManager.CONSTANT.PAGE_TYPE_EMPTY, tx) {
@@ -8363,12 +8505,26 @@ var require_cjs = __commonJS({
           await this.setPage(reusedPageId, newPage2, tx);
           return reusedPageId;
         }
+        const preallocationCount = this.options.pagePreallocationCount;
         const pageCount = metadataManager.getPageCount(metadata);
         const newPageIndex = pageCount;
-        const newTotalCount = pageCount + 1;
+        const newTotalCount = pageCount + preallocationCount;
         const manager = this.pageFactory.getManagerFromType(pageType);
         const newPage = manager.create(this.pageSize, newPageIndex);
         await this.setPage(newPageIndex, newPage, tx);
+        const emptyManager = this.pageFactory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_EMPTY);
+        const firstFreeIndex = newPageIndex + 1;
+        const lastFreeIndex = newPageIndex + preallocationCount - 1;
+        for (let i = firstFreeIndex; i <= lastFreeIndex; i++) {
+          const emptyPage = emptyManager.create(this.pageSize, i);
+          const nextId = i < lastFreeIndex ? i + 1 : -1;
+          emptyManager.setNextPageId(emptyPage, nextId);
+          await this.setPage(i, emptyPage, tx);
+          await this.updateBitmap(i, true, tx);
+        }
+        if (preallocationCount > 1) {
+          metadataManager.setFreePageId(metadata, firstFreeIndex);
+        }
         metadataManager.setPageCount(metadata, newTotalCount);
         await this.setPage(0, metadata, tx);
         return newPageIndex;
@@ -8452,7 +8608,8 @@ var require_cjs = __commonJS({
       }
       /**
        * Frees the page and marks it as available in the bitmap.
-       * It also adds the page to the linked list of free pages in metadata.
+       * Inserts the page into the free list in ascending order by page ID
+       * to support HDD sequential reads.
        * @param pageId Page ID
        * @param tx Transaction
        */
@@ -8462,13 +8619,30 @@ var require_cjs = __commonJS({
         await tx.__acquireWriteLock(pageId);
         const metadata = await this.getMetadata(tx);
         const metadataManager = this.pageFactory.getManager(metadata);
-        const currentHeadFreePageId = metadataManager.getFreePageId(metadata);
         const emptyPageManager = this.pageFactory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_EMPTY);
         const emptyPage = emptyPageManager.create(this.pageSize, pageId);
-        emptyPageManager.setNextPageId(emptyPage, currentHeadFreePageId);
-        await this.setPage(pageId, emptyPage, tx);
+        const headFreePageId = metadataManager.getFreePageId(metadata);
+        if (headFreePageId === -1 || pageId < headFreePageId) {
+          emptyPageManager.setNextPageId(emptyPage, headFreePageId);
+          await this.setPage(pageId, emptyPage, tx);
+          metadataManager.setFreePageId(metadata, pageId);
+        } else {
+          let prevPageId = headFreePageId;
+          let prevPage = await this.get(prevPageId, tx);
+          let prevManager = this.pageFactory.getManager(prevPage);
+          let nextPageId = prevManager.getNextPageId(prevPage);
+          while (nextPageId !== -1 && nextPageId < pageId) {
+            prevPageId = nextPageId;
+            prevPage = await this.get(prevPageId, tx);
+            prevManager = this.pageFactory.getManager(prevPage);
+            nextPageId = prevManager.getNextPageId(prevPage);
+          }
+          emptyPageManager.setNextPageId(emptyPage, nextPageId);
+          await this.setPage(pageId, emptyPage, tx);
+          prevManager.setNextPageId(prevPage, pageId);
+          await this.setPage(prevPageId, prevPage, tx);
+        }
         await this.updateBitmap(pageId, true, tx);
-        metadataManager.setFreePageId(metadata, pageId);
         await this.setPage(0, metadata, tx);
       }
       /**
@@ -9168,7 +9342,9 @@ var require_cjs = __commonJS({
           }
           pageGroupMap.get(pageId).push({ pk: pair.pk, slotIndex, index: pair.index });
         }
-        await Promise.all(Array.from(pageGroupMap).map(async ([pageId, items]) => {
+        const sortedPageIds = Array.from(pageGroupMap.keys()).sort((a, b) => a - b);
+        await Promise.all(sortedPageIds.map(async (pageId) => {
+          const items = pageGroupMap.get(pageId);
           const page = await this.pfs.get(pageId, tx);
           if (!this.factory.isDataPage(page)) {
             throw new Error(`Page ${pageId} is not a data page`);
@@ -9519,6 +9695,7 @@ var require_cjs = __commonJS({
         return Object.assign({
           pageSize: 8192,
           pageCacheCapacity: 1e4,
+          pagePreallocationCount: 1e3,
           wal: null,
           walCheckpointThreshold: 1e3
         }, options);
@@ -10572,8 +10749,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       if (metadata.lastId === 0) {
         return 0;
       }
-      const indexTxMap = {};
-      const indexEntryMap = /* @__PURE__ */ new Map();
+      let indexTxMap = {};
       for (const indexName of backfillTargets) {
         const tree = this.trees.get(indexName);
         if (tree && indexName !== "_id") {
@@ -10581,6 +10757,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
         }
       }
       let backfilledCount = 0;
+      let chunkCount = 0;
+      const CHUNK_SIZE = 1e3;
       const idTree = this.trees.get("_id");
       if (!idTree) {
         throw new Error("ID tree not found");
@@ -10609,10 +10787,6 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
               const keyToInsert = this.getTokenKey(k, token);
               const entry = { k, v: token };
               batchInsertData.push([keyToInsert, entry]);
-              if (!indexEntryMap.has(btx)) {
-                indexEntryMap.set(btx, []);
-              }
-              indexEntryMap.get(btx).push({ k: keyToInsert, v: entry });
             }
             await btx.batchInsert(batchInsertData);
           } else {
@@ -10620,34 +10794,42 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
             if (indexVal === void 0) continue;
             const entry = { k, v: indexVal };
             const batchInsertData = [[k, entry]];
-            if (!indexEntryMap.has(btx)) {
-              indexEntryMap.set(btx, []);
-            }
-            indexEntryMap.get(btx).push(entry);
             await btx.batchInsert(batchInsertData);
           }
         }
         backfilledCount++;
-      }
-      const btxs = Object.values(indexTxMap);
-      const success = [];
-      try {
-        for (const btx of btxs) {
-          await btx.commit();
-          success.push(btx);
-        }
-      } catch (err) {
-        for (const btx of btxs) {
-          await btx.rollback();
+        chunkCount++;
+        if (chunkCount >= CHUNK_SIZE) {
+          try {
+            for (const btx of Object.values(indexTxMap)) {
+              await btx.commit();
+            }
+          } catch (err) {
+            for (const btx of Object.values(indexTxMap)) {
+              await btx.rollback();
+            }
+            throw err;
+          }
+          for (const indexName of backfillTargets) {
+            const tree = this.trees.get(indexName);
+            if (tree && indexName !== "_id") {
+              indexTxMap[indexName] = await tree.createTransaction();
+            }
+          }
+          chunkCount = 0;
         }
-        for (const btx of success) {
-          const entries = indexEntryMap.get(btx);
-          if (!entries) continue;
-          for (const entry of entries) {
-            await btx.delete(entry.k, entry);
+      }
+      if (chunkCount > 0) {
+        try {
+          for (const btx of Object.values(indexTxMap)) {
+            await btx.commit();
           }
+        } catch (err) {
+          for (const btx of Object.values(indexTxMap)) {
+            await btx.rollback();
+          }
+          throw err;
         }
-        throw err;
       }
       this.pendingBackfillFields = [];
       return backfilledCount;
@@ -10805,27 +10987,61 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
         const condition = query[primaryField];
         const treeTx = await tree.createTransaction();
         let score = 0;
-        const coveredFields = config.fields.filter((f) => queryFields.has(f));
-        score += coveredFields.length;
-        if (condition) {
-          if (typeof condition !== "object" || condition === null) {
-            score += 100;
-          } else if ("primaryEqual" in condition || "equal" in condition) {
-            score += 100;
-          } else if ("primaryGte" in condition || "primaryLte" in condition || "primaryGt" in condition || "primaryLt" in condition || "gte" in condition || "lte" in condition || "gt" in condition || "lt" in condition) {
-            score += 50;
-          } else if ("primaryOr" in condition || "or" in condition) {
-            score += 20;
-          } else if ("like" in condition) {
-            score += 15;
-          } else {
-            score += 10;
+        let isConsecutive = true;
+        const coveredFields = [];
+        const compositeVerifyFields = [];
+        for (const field of config.fields) {
+          if (!queryFields.has(field)) {
+            isConsecutive = false;
+            continue;
+          }
+          coveredFields.push(field);
+          if (field !== primaryField) {
+            compositeVerifyFields.push(field);
+          }
+          score += 1;
+          if (isConsecutive) {
+            const cond = query[field];
+            if (cond !== void 0) {
+              if (typeof cond !== "object" || cond === null) {
+                score += 100;
+              } else if ("primaryEqual" in cond || "equal" in cond) {
+                score += 100;
+              } else if ("primaryGte" in cond || "primaryLte" in cond || "primaryGt" in cond || "primaryLt" in cond || "gte" in cond || "lte" in cond || "gt" in cond || "lt" in cond) {
+                score += 50;
+                isConsecutive = false;
+              } else if ("primaryOr" in cond || "or" in cond) {
+                score += 20;
+                isConsecutive = false;
+              } else if ("like" in cond) {
+                score += 15;
+                isConsecutive = false;
+              } else {
+                score += 10;
+                isConsecutive = false;
+              }
+            }
           }
         }
-        if (orderByField && primaryField === orderByField) {
-          score += 200;
+        let isIndexOrderSupported = false;
+        if (orderByField) {
+          for (const field of config.fields) {
+            if (field === orderByField) {
+              isIndexOrderSupported = true;
+              break;
+            }
+            const cond = query[field];
+            let isExactMatch = false;
+            if (cond !== void 0) {
+              if (typeof cond !== "object" || cond === null) isExactMatch = true;
+              else if ("primaryEqual" in cond || "equal" in cond) isExactMatch = true;
+            }
+            if (!isExactMatch) break;
+          }
+          if (isIndexOrderSupported) {
+            score += 200;
+          }
         }
-        const compositeVerifyFields = coveredFields.filter((f) => f !== primaryField);
         candidates.push({
           tree: treeTx,
           condition,
@@ -10833,7 +11049,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           indexName,
           isFtsMatch: false,
           score,
-          compositeVerifyFields
+          compositeVerifyFields,
+          isIndexOrderSupported
         });
       } else if (config.type === "fts") {
         const field = config.fields;
@@ -10851,7 +11068,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           isFtsMatch: true,
           matchTokens,
           score: 90,
-          compositeVerifyFields: []
+          compositeVerifyFields: [],
+          isIndexOrderSupported: false
         });
       }
     }
@@ -10894,33 +11112,30 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
   getTokenKey(pk, token) {
     return pk + ":" + token;
   }
-  async applyCandidateByFTS(candidate, matchedTokens, filterValues, order) {
+  async *applyCandidateByFTSStream(candidate, matchedTokens, filterValues, order) {
     const keys = /* @__PURE__ */ new Set();
     for (let i = 0, len = matchedTokens.length; i < len; i++) {
       const token = matchedTokens[i];
-      const pairs = await candidate.tree.where(
+      for await (const pair of candidate.tree.whereStream(
         { primaryEqual: { v: token } },
-        {
-          order
+        { order }
+      )) {
+        const pk = pair[1].k;
+        if (filterValues && !filterValues.has(pk)) continue;
+        if (!keys.has(pk)) {
+          keys.add(pk);
+          yield pk;
         }
-      );
-      for (const pair of pairs.values()) {
-        if (filterValues && !filterValues.has(pair.k)) continue;
-        keys.add(pair.k);
       }
     }
-    return keys;
   }
   /**
    * 특정 인덱스 후보를 조회하여 PK 집합을 필터링합니다.
    */
-  async applyCandidate(candidate, filterValues, order) {
-    return await candidate.tree.keys(
+  applyCandidateStream(candidate, filterValues, order) {
+    return candidate.tree.keysStream(
       candidate.condition,
-      {
-        filterValues,
-        order
-      }
+      { filterValues, order }
     );
   }
   /**
@@ -10936,30 +11151,34 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     );
     if (!selectivity) return new Float64Array(0);
     const { driver, others, rollback } = selectivity;
-    const useIndexOrder = orderBy === void 0 || driver.field === orderBy;
+    const useIndexOrder = orderBy === void 0 || driver.isIndexOrderSupported;
     const candidates = [driver, ...others];
     let keys = void 0;
     for (let i = 0, len = candidates.length; i < len; i++) {
       const candidate = candidates[i];
       const currentOrder = useIndexOrder ? sortOrder : void 0;
       if (candidate.isFtsMatch && candidate.matchTokens && candidate.matchTokens.length > 0) {
-        keys = await this.applyCandidateByFTS(
+        const stream = this.applyCandidateByFTSStream(
           candidate,
           candidate.matchTokens,
           keys,
           currentOrder
         );
+        keys = /* @__PURE__ */ new Set();
+        for await (const pk of stream) keys.add(pk);
       } else {
-        keys = await this.applyCandidate(candidate, keys, currentOrder);
+        const stream = this.applyCandidateStream(candidate, keys, currentOrder);
+        keys = /* @__PURE__ */ new Set();
+        for await (const pk of stream) keys.add(pk);
       }
     }
     rollback();
     return new Float64Array(Array.from(keys || []));
   }
   /**
-   * 드라이버 인덱스만으로 PK를 가져옵니다. (교집합 없이)
+   * 드라이버 인덱스만으로 PK 스트림을 가져옵니다. (교집합 없이)
    * selectDocuments에서 사용하며, 나머지 조건(others)은 스트리밍 중 tree.verify()로 검증합니다.
-   * @returns 드라이버 키 배열, others 후보 목록, rollback 함수. 또는 null.
+   * @returns 드라이버 키 스트림, others 후보 목록, rollback 함수. 또는 null.
    */
   async getDriverKeys(query, orderBy, sortOrder = "asc") {
     const isQueryEmpty = Object.keys(query).length === 0;
@@ -10970,21 +11189,21 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     );
     if (!selectivity) return null;
     const { driver, others, compositeVerifyConditions, rollback } = selectivity;
-    const useIndexOrder = orderBy === void 0 || driver.field === orderBy;
+    const useIndexOrder = orderBy === void 0 || driver.isIndexOrderSupported;
     const currentOrder = useIndexOrder ? sortOrder : void 0;
-    let keys;
+    let keysStream;
     if (driver.isFtsMatch && driver.matchTokens && driver.matchTokens.length > 0) {
-      keys = await this.applyCandidateByFTS(
+      keysStream = this.applyCandidateByFTSStream(
         driver,
         driver.matchTokens,
         void 0,
         currentOrder
       );
     } else {
-      keys = await this.applyCandidate(driver, void 0, currentOrder);
+      keysStream = this.applyCandidateStream(driver, void 0, currentOrder);
     }
     return {
-      keys: new Float64Array(Array.from(keys)),
+      keysStream,
       others,
       compositeVerifyConditions,
       isDriverOrderByField: useIndexOrder,
@@ -11334,28 +11553,17 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     return currentChunkSize;
   }
   /**
-   * Prefetch 방식으로 키 배열을 청크 단위로 조회하여 문서를 순회합니다.
+   * Prefetch 방식으로 키 스트림을 청크 단위로 조회하여 문서를 순회합니다.
    * FTS 검증, 복합 인덱스 검증, others 후보에 대한 tree.verify() 검증을 통과한 문서만 yield 합니다.
    */
-  async *processChunkedKeysWithVerify(keys, startIdx, initialChunkSize, ftsConditions, compositeVerifyConditions, others, tx) {
+  async *processChunkedKeysWithVerify(keysStream, startIdx, initialChunkSize, ftsConditions, compositeVerifyConditions, others, tx) {
     const verifyOthers = others.filter((o) => !o.isFtsMatch);
-    let i = startIdx;
-    const totalKeys = keys.length;
     let currentChunkSize = initialChunkSize;
-    let nextChunkPromise = null;
-    if (i < totalKeys) {
-      const endIdx = Math.min(i + currentChunkSize, totalKeys);
-      nextChunkPromise = this.selectMany(keys.subarray(i, endIdx), false, tx);
-      i = endIdx;
-    }
-    while (nextChunkPromise) {
-      const rawResults = await nextChunkPromise;
-      nextChunkPromise = null;
-      if (i < totalKeys) {
-        const endIdx = Math.min(i + currentChunkSize, totalKeys);
-        nextChunkPromise = this.selectMany(keys.subarray(i, endIdx), false, tx);
-        i = endIdx;
-      }
+    let chunk = [];
+    let dropped = 0;
+    const processChunk = async (pks) => {
+      const docs = [];
+      const rawResults = await this.selectMany(new Float64Array(pks), false, tx);
       let chunkTotalSize = 0;
       for (let j = 0, len = rawResults.length; j < len; j++) {
         const s = rawResults[j];
@@ -11382,9 +11590,26 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           }
           if (!passed) continue;
         }
-        yield doc;
+        docs.push(doc);
       }
       currentChunkSize = this.adjustChunkSize(currentChunkSize, chunkTotalSize);
+      return docs;
+    };
+    for await (const pk of keysStream) {
+      if (dropped < startIdx) {
+        dropped++;
+        continue;
+      }
+      chunk.push(pk);
+      if (chunk.length >= currentChunkSize) {
+        const docs = await processChunk(chunk);
+        for (let j = 0; j < docs.length; j++) yield docs[j];
+        chunk = [];
+      }
+    }
+    if (chunk.length > 0) {
+      const docs = await processChunk(chunk);
+      for (let j = 0; j < docs.length; j++) yield docs[j];
     }
   }
   /**
@@ -11432,11 +11657,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       }
       const driverResult = await self.getDriverKeys(query, orderByField, sortOrder);
       if (!driverResult) return;
-      const { keys, others, compositeVerifyConditions, isDriverOrderByField, rollback } = driverResult;
-      if (keys.length === 0) {
-        rollback();
-        return;
-      }
+      const { keysStream, others, compositeVerifyConditions, isDriverOrderByField, rollback } = driverResult;
       try {
         if (!isDriverOrderByField && orderByField) {
           const topK = limit === Infinity ? Infinity : offset + limit;
@@ -11451,7 +11672,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           }
           const results = [];
           for await (const doc of self.processChunkedKeysWithVerify(
-            keys,
+            keysStream,
             0,
             self.options.pageSize,
             ftsConditions,
@@ -11487,16 +11708,23 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
             yield limitedResults[j];
           }
         } else {
+          const hasFilters = ftsConditions.length > 0 || compositeVerifyConditions.length > 0 || others.length > 0;
+          const startIdx = hasFilters ? 0 : offset;
           let yieldedCount = 0;
+          let skippedCount = hasFilters ? 0 : offset;
           for await (const doc of self.processChunkedKeysWithVerify(
-            keys,
-            offset,
+            keysStream,
+            startIdx,
             self.options.pageSize,
             ftsConditions,
             compositeVerifyConditions,
             others,
             tx2
           )) {
+            if (skippedCount < offset) {
+              skippedCount++;
+              continue;
+            }
             if (yieldedCount >= limit) break;
             yield doc;
             yieldedCount++;

package/dist/types/core/documentAPI.d.ts

@@ -127,6 +127,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             field: string;
             indexName: string;
             isFtsMatch: false;
+            isIndexOrderSupported: boolean;
         } | {
             tree: BPTreeAsync<string, V>;
             condition: Partial<DocumentDataplyCondition<U>>;
@@ -134,6 +135,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             indexName: string;
             isFtsMatch: true;
             matchTokens: string[];
+            isIndexOrderSupported: boolean;
         });
         others: ({
             tree: BPTreeAsync<number, V>;
@@ -141,6 +143,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             field: string;
             indexName: string;
             isFtsMatch: false;
+            isIndexOrderSupported: boolean;
         } | {
             tree: BPTreeAsync<string, V>;
             condition: Partial<DocumentDataplyCondition<U>>;
@@ -148,6 +151,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             indexName: string;
             isFtsMatch: true;
             matchTokens: string[];
+            isIndexOrderSupported: boolean;
         })[];
         compositeVerifyConditions: {
             field: string;
@@ -164,20 +168,20 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
         smallChunkSize: number;
     };
     private getTokenKey;
-    private applyCandidateByFTS;
+    private applyCandidateByFTSStream;
     /**
      * 특정 인덱스 후보를 조회하여 PK 집합을 필터링합니다.
      */
-    private applyCandidate;
+    private applyCandidateStream;
     /**
      * 쿼리와 인덱스 선택을 기반으로 기본 키(Primary Keys)를 가져옵니다.
      * 쿼리 최적화를 통합하기 위한 내부 공통 메서드입니다.
      */
     getKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<Float64Array>;
     /**
-     * 드라이버 인덱스만으로 PK를 가져옵니다. (교집합 없이)
+     * 드라이버 인덱스만으로 PK 스트림을 가져옵니다. (교집합 없이)
      * selectDocuments에서 사용하며, 나머지 조건(others)은 스트리밍 중 tree.verify()로 검증합니다.
-     * @returns 드라이버 키 배열, others 후보 목록, rollback 함수. 또는 null.
+     * @returns 드라이버 키 스트림, others 후보 목록, rollback 함수. 또는 null.
      */
     private getDriverKeys;
     private insertDocumentInternal;
@@ -250,7 +254,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
      */
     private adjustChunkSize;
     /**
-     * Prefetch 방식으로 키 배열을 청크 단위로 조회하여 문서를 순회합니다.
+     * Prefetch 방식으로 키 스트림을 청크 단위로 조회하여 문서를 순회합니다.
      * FTS 검증, 복합 인덱스 검증, others 후보에 대한 tree.verify() 검증을 통과한 문서만 yield 합니다.
      */
     private processChunkedKeysWithVerify;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.9-alpha.4",
+  "version": "0.0.9-alpha.6",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.24-alpha.0"
+    "dataply": "^0.0.24-alpha.2"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",

package/readme.md

@@ -54,16 +54,21 @@ async function main() {
     .Options({ wal: 'my-database.wal' })
     .Open('my-database.db');
   
-  // Register indices before init (Recommended)
-  await db.createIndex('name', { type: 'btree', fields: ['name'] });
-  await db.createIndex('tags_0', { type: 'btree', fields: ['tags.0'] });
-  
-  // Composite Index support
-  await db.createIndex('idx_name_age', { type: 'btree', fields: ['name', 'age'] });
-
   // Initialize database
   await db.init();
 
+  // Register indices
+  // use transaction to ensure atomicity
+  await db.migration(1, async (tx) => {
+    await db.createIndex('name', { type: 'btree', fields: ['name'] }, tx);
+    await db.createIndex('tags_0', { type: 'btree', fields: ['tags.0'] }, tx);
+  
+    // Composite Index support
+    await db.createIndex('idx_name_age', { type: 'btree', fields: ['name', 'age'] }, tx);
+
+    console.log('Migration completed successfully');
+  });
+
   // Insert document
   const id = await db.insert({
     name: 'John Doe',