document-dataply 0.0.9-alpha.4 → 0.0.9-alpha.5

package/dist/cjs/index.js

@@ -10572,8 +10572,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       if (metadata.lastId === 0) {
         return 0;
       }
-      const indexTxMap = {};
-      const indexEntryMap = /* @__PURE__ */ new Map();
+      let indexTxMap = {};
       for (const indexName of backfillTargets) {
         const tree = this.trees.get(indexName);
         if (tree && indexName !== "_id") {
@@ -10581,6 +10580,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
         }
       }
       let backfilledCount = 0;
+      let chunkCount = 0;
+      const CHUNK_SIZE = 1e3;
       const idTree = this.trees.get("_id");
       if (!idTree) {
         throw new Error("ID tree not found");
@@ -10609,10 +10610,6 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
               const keyToInsert = this.getTokenKey(k, token);
               const entry = { k, v: token };
               batchInsertData.push([keyToInsert, entry]);
-              if (!indexEntryMap.has(btx)) {
-                indexEntryMap.set(btx, []);
-              }
-              indexEntryMap.get(btx).push({ k: keyToInsert, v: entry });
             }
             await btx.batchInsert(batchInsertData);
           } else {
@@ -10620,34 +10617,42 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
             if (indexVal === void 0) continue;
             const entry = { k, v: indexVal };
             const batchInsertData = [[k, entry]];
-            if (!indexEntryMap.has(btx)) {
-              indexEntryMap.set(btx, []);
-            }
-            indexEntryMap.get(btx).push(entry);
             await btx.batchInsert(batchInsertData);
           }
         }
         backfilledCount++;
-      }
-      const btxs = Object.values(indexTxMap);
-      const success = [];
-      try {
-        for (const btx of btxs) {
-          await btx.commit();
-          success.push(btx);
-        }
-      } catch (err) {
-        for (const btx of btxs) {
-          await btx.rollback();
+        chunkCount++;
+        if (chunkCount >= CHUNK_SIZE) {
+          try {
+            for (const btx of Object.values(indexTxMap)) {
+              await btx.commit();
+            }
+          } catch (err) {
+            for (const btx of Object.values(indexTxMap)) {
+              await btx.rollback();
+            }
+            throw err;
+          }
+          for (const indexName of backfillTargets) {
+            const tree = this.trees.get(indexName);
+            if (tree && indexName !== "_id") {
+              indexTxMap[indexName] = await tree.createTransaction();
+            }
+          }
+          chunkCount = 0;
         }
-        for (const btx of success) {
-          const entries = indexEntryMap.get(btx);
-          if (!entries) continue;
-          for (const entry of entries) {
-            await btx.delete(entry.k, entry);
+      }
+      if (chunkCount > 0) {
+        try {
+          for (const btx of Object.values(indexTxMap)) {
+            await btx.commit();
           }
+        } catch (err) {
+          for (const btx of Object.values(indexTxMap)) {
+            await btx.rollback();
+          }
+          throw err;
         }
-        throw err;
       }
       this.pendingBackfillFields = [];
       return backfilledCount;
@@ -10805,27 +10810,61 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
         const condition = query[primaryField];
         const treeTx = await tree.createTransaction();
         let score = 0;
-        const coveredFields = config.fields.filter((f) => queryFields.has(f));
-        score += coveredFields.length;
-        if (condition) {
-          if (typeof condition !== "object" || condition === null) {
-            score += 100;
-          } else if ("primaryEqual" in condition || "equal" in condition) {
-            score += 100;
-          } else if ("primaryGte" in condition || "primaryLte" in condition || "primaryGt" in condition || "primaryLt" in condition || "gte" in condition || "lte" in condition || "gt" in condition || "lt" in condition) {
-            score += 50;
-          } else if ("primaryOr" in condition || "or" in condition) {
-            score += 20;
-          } else if ("like" in condition) {
-            score += 15;
-          } else {
-            score += 10;
+        let isConsecutive = true;
+        const coveredFields = [];
+        const compositeVerifyFields = [];
+        for (const field of config.fields) {
+          if (!queryFields.has(field)) {
+            isConsecutive = false;
+            continue;
+          }
+          coveredFields.push(field);
+          if (field !== primaryField) {
+            compositeVerifyFields.push(field);
+          }
+          score += 1;
+          if (isConsecutive) {
+            const cond = query[field];
+            if (cond !== void 0) {
+              if (typeof cond !== "object" || cond === null) {
+                score += 100;
+              } else if ("primaryEqual" in cond || "equal" in cond) {
+                score += 100;
+              } else if ("primaryGte" in cond || "primaryLte" in cond || "primaryGt" in cond || "primaryLt" in cond || "gte" in cond || "lte" in cond || "gt" in cond || "lt" in cond) {
+                score += 50;
+                isConsecutive = false;
+              } else if ("primaryOr" in cond || "or" in cond) {
+                score += 20;
+                isConsecutive = false;
+              } else if ("like" in cond) {
+                score += 15;
+                isConsecutive = false;
+              } else {
+                score += 10;
+                isConsecutive = false;
+              }
+            }
           }
         }
-        if (orderByField && primaryField === orderByField) {
-          score += 200;
+        let isIndexOrderSupported = false;
+        if (orderByField) {
+          for (const field of config.fields) {
+            if (field === orderByField) {
+              isIndexOrderSupported = true;
+              break;
+            }
+            const cond = query[field];
+            let isExactMatch = false;
+            if (cond !== void 0) {
+              if (typeof cond !== "object" || cond === null) isExactMatch = true;
+              else if ("primaryEqual" in cond || "equal" in cond) isExactMatch = true;
+            }
+            if (!isExactMatch) break;
+          }
+          if (isIndexOrderSupported) {
+            score += 200;
+          }
         }
-        const compositeVerifyFields = coveredFields.filter((f) => f !== primaryField);
         candidates.push({
           tree: treeTx,
           condition,
@@ -10833,7 +10872,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           indexName,
           isFtsMatch: false,
           score,
-          compositeVerifyFields
+          compositeVerifyFields,
+          isIndexOrderSupported
         });
       } else if (config.type === "fts") {
         const field = config.fields;
@@ -10851,7 +10891,8 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           isFtsMatch: true,
           matchTokens,
           score: 90,
-          compositeVerifyFields: []
+          compositeVerifyFields: [],
+          isIndexOrderSupported: false
         });
       }
     }
@@ -10894,33 +10935,30 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
   getTokenKey(pk, token) {
     return pk + ":" + token;
   }
-  async applyCandidateByFTS(candidate, matchedTokens, filterValues, order) {
+  async *applyCandidateByFTSStream(candidate, matchedTokens, filterValues, order) {
     const keys = /* @__PURE__ */ new Set();
     for (let i = 0, len = matchedTokens.length; i < len; i++) {
       const token = matchedTokens[i];
-      const pairs = await candidate.tree.where(
+      for await (const pair of candidate.tree.whereStream(
         { primaryEqual: { v: token } },
-        {
-          order
+        { order }
+      )) {
+        const pk = pair[1].k;
+        if (filterValues && !filterValues.has(pk)) continue;
+        if (!keys.has(pk)) {
+          keys.add(pk);
+          yield pk;
         }
-      );
-      for (const pair of pairs.values()) {
-        if (filterValues && !filterValues.has(pair.k)) continue;
-        keys.add(pair.k);
       }
     }
-    return keys;
   }
   /**
    * 특정 인덱스 후보를 조회하여 PK 집합을 필터링합니다.
    */
-  async applyCandidate(candidate, filterValues, order) {
-    return await candidate.tree.keys(
+  applyCandidateStream(candidate, filterValues, order) {
+    return candidate.tree.keysStream(
       candidate.condition,
-      {
-        filterValues,
-        order
-      }
+      { filterValues, order }
     );
   }
   /**
@@ -10936,30 +10974,34 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     );
     if (!selectivity) return new Float64Array(0);
     const { driver, others, rollback } = selectivity;
-    const useIndexOrder = orderBy === void 0 || driver.field === orderBy;
+    const useIndexOrder = orderBy === void 0 || driver.isIndexOrderSupported;
     const candidates = [driver, ...others];
     let keys = void 0;
     for (let i = 0, len = candidates.length; i < len; i++) {
       const candidate = candidates[i];
       const currentOrder = useIndexOrder ? sortOrder : void 0;
       if (candidate.isFtsMatch && candidate.matchTokens && candidate.matchTokens.length > 0) {
-        keys = await this.applyCandidateByFTS(
+        const stream = this.applyCandidateByFTSStream(
           candidate,
           candidate.matchTokens,
           keys,
           currentOrder
         );
+        keys = /* @__PURE__ */ new Set();
+        for await (const pk of stream) keys.add(pk);
       } else {
-        keys = await this.applyCandidate(candidate, keys, currentOrder);
+        const stream = this.applyCandidateStream(candidate, keys, currentOrder);
+        keys = /* @__PURE__ */ new Set();
+        for await (const pk of stream) keys.add(pk);
       }
     }
     rollback();
     return new Float64Array(Array.from(keys || []));
   }
   /**
-   * 드라이버 인덱스만으로 PK를 가져옵니다. (교집합 없이)
+   * 드라이버 인덱스만으로 PK 스트림을 가져옵니다. (교집합 없이)
    * selectDocuments에서 사용하며, 나머지 조건(others)은 스트리밍 중 tree.verify()로 검증합니다.
-   * @returns 드라이버 키 배열, others 후보 목록, rollback 함수. 또는 null.
+   * @returns 드라이버 키 스트림, others 후보 목록, rollback 함수. 또는 null.
    */
   async getDriverKeys(query, orderBy, sortOrder = "asc") {
     const isQueryEmpty = Object.keys(query).length === 0;
@@ -10970,21 +11012,21 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     );
     if (!selectivity) return null;
     const { driver, others, compositeVerifyConditions, rollback } = selectivity;
-    const useIndexOrder = orderBy === void 0 || driver.field === orderBy;
+    const useIndexOrder = orderBy === void 0 || driver.isIndexOrderSupported;
     const currentOrder = useIndexOrder ? sortOrder : void 0;
-    let keys;
+    let keysStream;
     if (driver.isFtsMatch && driver.matchTokens && driver.matchTokens.length > 0) {
-      keys = await this.applyCandidateByFTS(
+      keysStream = this.applyCandidateByFTSStream(
         driver,
         driver.matchTokens,
         void 0,
         currentOrder
       );
     } else {
-      keys = await this.applyCandidate(driver, void 0, currentOrder);
+      keysStream = this.applyCandidateStream(driver, void 0, currentOrder);
     }
     return {
-      keys: new Float64Array(Array.from(keys)),
+      keysStream,
       others,
       compositeVerifyConditions,
       isDriverOrderByField: useIndexOrder,
@@ -11334,28 +11376,17 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     return currentChunkSize;
   }
   /**
-   * Prefetch 방식으로 키 배열을 청크 단위로 조회하여 문서를 순회합니다.
+   * Prefetch 방식으로 키 스트림을 청크 단위로 조회하여 문서를 순회합니다.
    * FTS 검증, 복합 인덱스 검증, others 후보에 대한 tree.verify() 검증을 통과한 문서만 yield 합니다.
    */
-  async *processChunkedKeysWithVerify(keys, startIdx, initialChunkSize, ftsConditions, compositeVerifyConditions, others, tx) {
+  async *processChunkedKeysWithVerify(keysStream, startIdx, initialChunkSize, ftsConditions, compositeVerifyConditions, others, tx) {
     const verifyOthers = others.filter((o) => !o.isFtsMatch);
-    let i = startIdx;
-    const totalKeys = keys.length;
     let currentChunkSize = initialChunkSize;
-    let nextChunkPromise = null;
-    if (i < totalKeys) {
-      const endIdx = Math.min(i + currentChunkSize, totalKeys);
-      nextChunkPromise = this.selectMany(keys.subarray(i, endIdx), false, tx);
-      i = endIdx;
-    }
-    while (nextChunkPromise) {
-      const rawResults = await nextChunkPromise;
-      nextChunkPromise = null;
-      if (i < totalKeys) {
-        const endIdx = Math.min(i + currentChunkSize, totalKeys);
-        nextChunkPromise = this.selectMany(keys.subarray(i, endIdx), false, tx);
-        i = endIdx;
-      }
+    let chunk = [];
+    let dropped = 0;
+    const processChunk = async (pks) => {
+      const docs = [];
+      const rawResults = await this.selectMany(new Float64Array(pks), false, tx);
       let chunkTotalSize = 0;
       for (let j = 0, len = rawResults.length; j < len; j++) {
         const s = rawResults[j];
@@ -11382,9 +11413,26 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           }
           if (!passed) continue;
         }
-        yield doc;
+        docs.push(doc);
       }
       currentChunkSize = this.adjustChunkSize(currentChunkSize, chunkTotalSize);
+      return docs;
+    };
+    for await (const pk of keysStream) {
+      if (dropped < startIdx) {
+        dropped++;
+        continue;
+      }
+      chunk.push(pk);
+      if (chunk.length >= currentChunkSize) {
+        const docs = await processChunk(chunk);
+        for (let j = 0; j < docs.length; j++) yield docs[j];
+        chunk = [];
+      }
+    }
+    if (chunk.length > 0) {
+      const docs = await processChunk(chunk);
+      for (let j = 0; j < docs.length; j++) yield docs[j];
     }
   }
   /**
@@ -11432,11 +11480,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       }
       const driverResult = await self.getDriverKeys(query, orderByField, sortOrder);
       if (!driverResult) return;
-      const { keys, others, compositeVerifyConditions, isDriverOrderByField, rollback } = driverResult;
-      if (keys.length === 0) {
-        rollback();
-        return;
-      }
+      const { keysStream, others, compositeVerifyConditions, isDriverOrderByField, rollback } = driverResult;
       try {
         if (!isDriverOrderByField && orderByField) {
           const topK = limit === Infinity ? Infinity : offset + limit;
@@ -11451,7 +11495,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
           }
           const results = [];
           for await (const doc of self.processChunkedKeysWithVerify(
-            keys,
+            keysStream,
             0,
             self.options.pageSize,
             ftsConditions,
@@ -11487,16 +11531,23 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
             yield limitedResults[j];
           }
         } else {
+          const hasFilters = ftsConditions.length > 0 || compositeVerifyConditions.length > 0 || others.length > 0;
+          const startIdx = hasFilters ? 0 : offset;
           let yieldedCount = 0;
+          let skippedCount = hasFilters ? 0 : offset;
           for await (const doc of self.processChunkedKeysWithVerify(
-            keys,
-            offset,
+            keysStream,
+            startIdx,
             self.options.pageSize,
             ftsConditions,
             compositeVerifyConditions,
             others,
             tx2
           )) {
+            if (skippedCount < offset) {
+              skippedCount++;
+              continue;
+            }
             if (yieldedCount >= limit) break;
             yield doc;
             yieldedCount++;

package/dist/types/core/documentAPI.d.ts

@@ -127,6 +127,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             field: string;
             indexName: string;
             isFtsMatch: false;
+            isIndexOrderSupported: boolean;
         } | {
             tree: BPTreeAsync<string, V>;
             condition: Partial<DocumentDataplyCondition<U>>;
@@ -134,6 +135,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             indexName: string;
             isFtsMatch: true;
             matchTokens: string[];
+            isIndexOrderSupported: boolean;
         });
         others: ({
             tree: BPTreeAsync<number, V>;
@@ -141,6 +143,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             field: string;
             indexName: string;
             isFtsMatch: false;
+            isIndexOrderSupported: boolean;
         } | {
             tree: BPTreeAsync<string, V>;
             condition: Partial<DocumentDataplyCondition<U>>;
@@ -148,6 +151,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
             indexName: string;
             isFtsMatch: true;
             matchTokens: string[];
+            isIndexOrderSupported: boolean;
         })[];
         compositeVerifyConditions: {
             field: string;
@@ -164,20 +168,20 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
         smallChunkSize: number;
     };
     private getTokenKey;
-    private applyCandidateByFTS;
+    private applyCandidateByFTSStream;
     /**
      * 특정 인덱스 후보를 조회하여 PK 집합을 필터링합니다.
      */
-    private applyCandidate;
+    private applyCandidateStream;
     /**
      * 쿼리와 인덱스 선택을 기반으로 기본 키(Primary Keys)를 가져옵니다.
      * 쿼리 최적화를 통합하기 위한 내부 공통 메서드입니다.
      */
     getKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<Float64Array>;
     /**
-     * 드라이버 인덱스만으로 PK를 가져옵니다. (교집합 없이)
+     * 드라이버 인덱스만으로 PK 스트림을 가져옵니다. (교집합 없이)
      * selectDocuments에서 사용하며, 나머지 조건(others)은 스트리밍 중 tree.verify()로 검증합니다.
-     * @returns 드라이버 키 배열, others 후보 목록, rollback 함수. 또는 null.
+     * @returns 드라이버 키 스트림, others 후보 목록, rollback 함수. 또는 null.
      */
     private getDriverKeys;
     private insertDocumentInternal;
@@ -250,7 +254,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
      */
     private adjustChunkSize;
     /**
-     * Prefetch 방식으로 키 배열을 청크 단위로 조회하여 문서를 순회합니다.
+     * Prefetch 방식으로 키 스트림을 청크 단위로 조회하여 문서를 순회합니다.
      * FTS 검증, 복합 인덱스 검증, others 후보에 대한 tree.verify() 검증을 통과한 문서만 yield 합니다.
      */
     private processChunkedKeysWithVerify;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.9-alpha.4",
+  "version": "0.0.9-alpha.5",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.24-alpha.0"
+    "dataply": "^0.0.24-alpha.1"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",

package/readme.md

@@ -54,16 +54,21 @@ async function main() {
     .Options({ wal: 'my-database.wal' })
     .Open('my-database.db');
   
-  // Register indices before init (Recommended)
-  await db.createIndex('name', { type: 'btree', fields: ['name'] });
-  await db.createIndex('tags_0', { type: 'btree', fields: ['tags.0'] });
-  
-  // Composite Index support
-  await db.createIndex('idx_name_age', { type: 'btree', fields: ['name', 'age'] });
-
   // Initialize database
   await db.init();
 
+  // Register indices
+  // use transaction to ensure atomicity
+  await db.migration(1, async (tx) => {
+    await db.createIndex('name', { type: 'btree', fields: ['name'] }, tx);
+    await db.createIndex('tags_0', { type: 'btree', fields: ['tags.0'] }, tx);
+  
+    // Composite Index support
+    await db.createIndex('idx_name_age', { type: 'btree', fields: ['name', 'age'] }, tx);
+
+    console.log('Migration completed successfully');
+  });
+
   // Insert document
   const id = await db.insert({
     name: 'John Doe',