document-dataply 0.0.10-alpha.5 → 0.0.10-alpha.7

package/dist/cjs/index.js

@@ -10466,6 +10466,26 @@ function tokenize(text, options) {
 }
 
 // src/core/Optimizer.ts
+var SELECTIVITY = {
+  /** O(log N) 포인트 룩업 */
+  EQUAL: 0.01,
+  /** 양쪽 바운드(gte+lte) 범위 스캔 */
+  BOUNDED_RANGE: 0.33,
+  /** 한쪽 바운드(gte 또는 lte)만 있을 때, 중간부터 풀스캔 */
+  HALF_RANGE: 0.5,
+  /** Or 조건: B+Tree 내부 풀스캔 */
+  OR: 0.9,
+  /** Like 조건: B+Tree 내부 풀스캔 */
+  LIKE: 0.9,
+  /** 알 수 없는 조건 */
+  UNKNOWN: 0.9,
+  /** FTS 통계 없을 때 보수적 추정 */
+  FTS_DEFAULT: 0.5,
+  /** 정렬 비용 가중치 (orderBy 미지원 시) */
+  SORT_PENALTY: 0.5,
+  /** 인메모리 정렬이 유의미해지는 임계 문서 수 */
+  SORT_THRESHOLD: 1e4
+};
 var Optimizer = class {
   constructor(api) {
     this.api = api;
@@ -10477,7 +10497,7 @@ var Optimizer = class {
     const primaryField = config.fields[0];
     if (!queryFields.has(primaryField)) return null;
     const builtCondition = {};
-    let score = 0;
+    let selectivity = 1;
     let isConsecutive = true;
     const coveredFields = [];
     const compositeVerifyFields = [];
@@ -10492,13 +10512,12 @@ var Optimizer = class {
         continue;
       }
       coveredFields.push(field);
-      score += 1;
       if (isConsecutive) {
         const cond = query[field];
         if (cond !== void 0) {
           let isBounded = false;
           if (typeof cond !== "object" || cond === null) {
-            score += 100;
+            selectivity *= SELECTIVITY.EQUAL;
             startValues.push(cond);
             endValues.push(cond);
             startOperator = "primaryGte";
@@ -10506,7 +10525,7 @@ var Optimizer = class {
             isBounded = true;
           } else if ("primaryEqual" in cond || "equal" in cond) {
             const val = cond.primaryEqual?.v ?? cond.equal?.v ?? cond.primaryEqual ?? cond.equal;
-            score += 100;
+            selectivity *= SELECTIVITY.EQUAL;
             startValues.push(val);
             endValues.push(val);
             startOperator = "primaryGte";
@@ -10514,7 +10533,7 @@ var Optimizer = class {
             isBounded = true;
           } else if ("primaryGte" in cond || "gte" in cond) {
             const val = cond.primaryGte?.v ?? cond.gte?.v ?? cond.primaryGte ?? cond.gte;
-            score += 50;
+            selectivity *= SELECTIVITY.HALF_RANGE;
             isConsecutive = false;
             startValues.push(val);
             startOperator = "primaryGte";
@@ -10522,7 +10541,7 @@ var Optimizer = class {
             isBounded = true;
           } else if ("primaryGt" in cond || "gt" in cond) {
             const val = cond.primaryGt?.v ?? cond.gt?.v ?? cond.primaryGt ?? cond.gt;
-            score += 50;
+            selectivity *= SELECTIVITY.HALF_RANGE;
             isConsecutive = false;
             startValues.push(val);
             startOperator = "primaryGt";
@@ -10530,7 +10549,7 @@ var Optimizer = class {
             isBounded = true;
           } else if ("primaryLte" in cond || "lte" in cond) {
             const val = cond.primaryLte?.v ?? cond.lte?.v ?? cond.primaryLte ?? cond.lte;
-            score += 50;
+            selectivity *= SELECTIVITY.HALF_RANGE;
             isConsecutive = false;
             endValues.push(val);
             endOperator = "primaryLte";
@@ -10538,20 +10557,20 @@ var Optimizer = class {
             isBounded = true;
           } else if ("primaryLt" in cond || "lt" in cond) {
             const val = cond.primaryLt?.v ?? cond.lt?.v ?? cond.primaryLt ?? cond.lt;
-            score += 50;
+            selectivity *= SELECTIVITY.HALF_RANGE;
             isConsecutive = false;
             endValues.push(val);
             endOperator = "primaryLt";
             if (startValues.length > 0) startOperator = "primaryGte";
             isBounded = true;
           } else if ("primaryOr" in cond || "or" in cond) {
-            score += 20;
+            selectivity *= SELECTIVITY.OR;
             isConsecutive = false;
           } else if ("like" in cond) {
-            score += 15;
+            selectivity *= SELECTIVITY.LIKE;
             isConsecutive = false;
           } else {
-            score += 10;
+            selectivity *= SELECTIVITY.UNKNOWN;
             isConsecutive = false;
           }
           if (!isBounded && field !== primaryField) {
@@ -10598,9 +10617,6 @@ var Optimizer = class {
         }
         if (!isExactMatch) break;
       }
-      if (isIndexOrderSupported) {
-        score += 200;
-      }
     }
     return {
       tree: treeTx,
@@ -10608,7 +10624,7 @@ var Optimizer = class {
       field: primaryField,
       indexName,
       isFtsMatch: false,
-      score,
+      selectivity,
       compositeVerifyFields,
       coveredFields,
       isIndexOrderSupported
@@ -10616,7 +10632,7 @@ var Optimizer = class {
   }
   /**
    * FTS 타입 인덱스의 선택도를 평가합니다.
-   * FTSTermCount 통계가 있으면 토큰 빈도 기반 동적 score를 산출합니다.
+   * FTSTermCount 통계가 있으면 실측 데이터 기반으로 선택도를 산출합니다.
    */
   evaluateFTSCandidate(indexName, config, query, queryFields, treeTx) {
     const field = config.fields;
@@ -10625,18 +10641,14 @@ var Optimizer = class {
     if (!condition || typeof condition !== "object" || !("match" in condition)) return null;
     const ftsConfig = this.api.indexManager.getFtsConfig(config);
     const matchTokens = ftsConfig ? tokenize(condition.match, ftsConfig) : [];
-    const MAX_FTS_SCORE = 400;
-    const MIN_FTS_SCORE = 10;
-    const DEFAULT_FTS_SCORE = 90;
-    let score = DEFAULT_FTS_SCORE;
+    let selectivity = SELECTIVITY.FTS_DEFAULT;
     const termCountProvider = this.api.analysisManager.getProvider("fts_term_count");
     if (termCountProvider && termCountProvider.hasSampleData && ftsConfig && matchTokens.length > 0) {
       const strategy = ftsConfig.tokenizer === "ngram" ? `${ftsConfig.gramSize}gram` : ftsConfig.tokenizer;
       const minCount = termCountProvider.getMinTokenCount(field, strategy, matchTokens);
       if (minCount >= 0) {
         const sampleSize = termCountProvider.getSampleSize();
-        const selectivityRatio = Math.min(minCount / sampleSize, 1);
-        score = Math.round(MAX_FTS_SCORE * (1 - selectivityRatio) + MIN_FTS_SCORE);
+        selectivity = Math.min(minCount / sampleSize, 1);
       }
     }
     return {
@@ -10646,18 +10658,36 @@ var Optimizer = class {
       indexName,
       isFtsMatch: true,
       matchTokens,
-      score,
+      selectivity,
       compositeVerifyFields: [],
       coveredFields: [field],
       isIndexOrderSupported: false
     };
   }
   /**
-   * 실행할 최적의 인덱스를 선택합니다. (최적 드라이버 선택)
+   * 비용 계산: effectiveScanCost + sortPenalty
+   * - effectiveScanCost: 인덱스 순서 지원 + limit 존재 시 조기 종료 이점 반영
+   * - sortPenalty: 인메모리 정렬의 절대 문서 수 기반 비용
+   * - hasUncoveredFilters: 드라이버가 커버하지 못하는 비-FTS 필터 존재 여부
+   *   true일 경우 topK/N 조기 종료를 적용하지 않음
+   *   (uncovered 필터가 행을 탈락시킬 수 있어 topK개 이상 스캔 필요)
    */
-  async getSelectivityCandidate(query, orderByField) {
+  calculateCost(selectivity, isIndexOrderSupported, orderByField, N, topK, hasUncoveredFilters = false) {
+    const effectiveScanCost = isIndexOrderSupported && isFinite(topK) && N > 0 && !hasUncoveredFilters ? Math.min(topK / N, selectivity) : selectivity;
+    const estimatedSortDocs = selectivity * N;
+    const sortPenalty = orderByField && !isIndexOrderSupported ? Math.min(estimatedSortDocs / SELECTIVITY.SORT_THRESHOLD, 1) * SELECTIVITY.SORT_PENALTY : 0;
+    return effectiveScanCost + sortPenalty;
+  }
+  /**
+   * 실행할 최적의 인덱스를 선택합니다. (비용 기반 최적 드라이버 선택)
+   * cost = selectivity + sortPenalty (낮을수록 좋음)
+   */
+  async getSelectivityCandidate(query, orderByField, limit = Infinity, offset = 0) {
     const queryFields = new Set(Object.keys(query));
     const candidates = [];
+    const metadata = await this.api.getMetadata();
+    const N = metadata.rowCount;
+    const topK = isFinite(limit) ? offset + limit : Infinity;
     for (const [indexName, config] of this.api.indexManager.registeredIndices) {
       const tree = this.api.trees.get(indexName);
       if (!tree) continue;
@@ -10671,7 +10701,22 @@ var Optimizer = class {
           treeTx,
           orderByField
         );
-        if (candidate) candidates.push(candidate);
+        if (candidate) {
+          const hasUncoveredFilters = ![...queryFields].every(
+            (f) => candidate.coveredFields.includes(f)
+          );
+          candidates.push({
+            ...candidate,
+            cost: this.calculateCost(
+              candidate.selectivity,
+              candidate.isIndexOrderSupported,
+              orderByField,
+              N,
+              topK,
+              hasUncoveredFilters
+            )
+          });
+        }
       } else if (config.type === "fts") {
         const treeTx = await tree.createTransaction();
         const candidate = this.evaluateFTSCandidate(
@@ -10681,7 +10726,19 @@ var Optimizer = class {
           queryFields,
           treeTx
         );
-        if (candidate) candidates.push(candidate);
+        if (candidate) {
+          candidates.push({
+            ...candidate,
+            cost: this.calculateCost(
+              candidate.selectivity,
+              candidate.isIndexOrderSupported,
+              orderByField,
+              N,
+              topK,
+              true
+            )
+          });
+        }
       }
     }
     const rollback = () => {
@@ -10694,7 +10751,7 @@ var Optimizer = class {
       return null;
     }
     candidates.sort((a, b) => {
-      if (b.score !== a.score) return b.score - a.score;
+      if (a.cost !== b.cost) return a.cost - b.cost;
       const aConfig = this.api.indexManager.registeredIndices.get(a.indexName);
       const bConfig = this.api.indexManager.registeredIndices.get(b.indexName);
       const aFieldCount = aConfig ? Array.isArray(aConfig.fields) ? aConfig.fields.length : 1 : 0;
@@ -10709,8 +10766,8 @@ var Optimizer = class {
       const candidate = nonDriverCandidates[i];
       let isSubset = false;
       for (let j = 0, oLen = others.length; j < oLen; j++) {
-        const higher = others[j];
-        if (candidate.coveredFields.every((f) => higher.coveredFields.includes(f))) {
+        const better = others[j];
+        if (candidate.coveredFields.every((f) => better.coveredFields.includes(f))) {
           isSubset = true;
           break;
         }
@@ -10915,12 +10972,14 @@ var QueryManager = class {
     rollback();
     return new Float64Array(Array.from(keys || []));
   }
-  async getDriverKeys(query, orderBy, sortOrder = "asc") {
+  async getDriverKeys(query, orderBy, sortOrder = "asc", limit = Infinity, offset = 0) {
     const isQueryEmpty = Object.keys(query).length === 0;
     const normalizedQuery = isQueryEmpty ? { _id: { gte: 0 } } : query;
     const selectivity = await this.optimizer.getSelectivityCandidate(
       this.verboseQuery(normalizedQuery),
-      orderBy
+      orderBy,
+      limit,
+      offset
     );
     if (!selectivity) return null;
     const { driver, others, compositeVerifyConditions, rollback } = selectivity;
@@ -10996,12 +11055,24 @@ var QueryManager = class {
     }
     return true;
   }
-  adjustChunkSize(currentChunkSize, chunkTotalSize) {
-    if (chunkTotalSize <= 0) return currentChunkSize;
-    const { verySmallChunkSize, smallChunkSize } = this.getFreeMemoryChunkSize();
-    if (chunkTotalSize < verySmallChunkSize) return currentChunkSize * 2;
-    if (chunkTotalSize > smallChunkSize) return Math.max(Math.floor(currentChunkSize / 2), 20);
-    return currentChunkSize;
+  /**
+   * 최적화 공식: x = x * Math.sqrt(z / n)
+   * @param currentChunkSize 현재 청크 크기
+   * @param matchedCount 매칭된 문서 개수
+   * @param limit 최대 문서 개수
+   * @param chunkTotalSize 청크 내 문서 총 크기
+   * @returns 
+   */
+  adjustChunkSize(currentChunkSize, matchedCount, limit, chunkTotalSize) {
+    if (matchedCount <= 0 || chunkTotalSize <= 0) return currentChunkSize;
+    const n = Math.max(matchedCount, 1);
+    const z = isFinite(limit) ? limit : currentChunkSize * 10;
+    const nextChunkSize = Math.ceil(currentChunkSize * Math.sqrt(z / n));
+    const { smallChunkSize } = this.getFreeMemoryChunkSize();
+    const avgDocSize = chunkTotalSize / currentChunkSize;
+    const maxSafeChunkSize = Math.max(Math.floor(smallChunkSize / avgDocSize), 20);
+    const finalChunkSize = Math.max(Math.min(nextChunkSize, maxSafeChunkSize), 20);
+    return finalChunkSize;
   }
   async *processChunkedKeysWithVerify(keysStream, startIdx, initialChunkSize, limit, ftsConditions, compositeVerifyConditions, others, tx) {
     const verifyOthers = others.filter((o) => !o.isFtsMatch);
@@ -11017,6 +11088,7 @@ var QueryManager = class {
     let chunk = [];
     let chunkSize = 0;
     let dropped = 0;
+    let nAccumulated = 0;
     const processChunk = async (pks) => {
       const docs = [];
       const rawResults = await this.api.selectMany(new Float64Array(pks), false, tx);
@@ -11068,8 +11140,9 @@ var QueryManager = class {
         }
         docs.push(doc);
       }
+      nAccumulated += docs.length;
       if (!isReadQuotaLimited) {
-        currentChunkSize = this.adjustChunkSize(currentChunkSize, chunkTotalSize);
+        currentChunkSize = this.adjustChunkSize(currentChunkSize, nAccumulated, limit, chunkTotalSize);
       }
       return docs;
     };
@@ -11139,12 +11212,13 @@ var QueryManager = class {
           }
         }
       }
-      const driverResult = await self.getDriverKeys(query, orderByField, sortOrder);
+      const driverResult = await self.getDriverKeys(query, orderByField, sortOrder, limit, offset);
       if (!driverResult) return;
       const { keysStream, others, compositeVerifyConditions, isDriverOrderByField, rollback } = driverResult;
       const initialChunkSize = self.api.options.pageSize;
+      const isInMemorySort = !isDriverOrderByField && orderByField;
       try {
-        if (!isDriverOrderByField && orderByField) {
+        if (isInMemorySort) {
           const topK = limit === Infinity ? Infinity : offset + limit;
           let heap = null;
           if (topK !== Infinity) {

package/dist/types/core/Optimizer.d.ts

@@ -13,31 +13,41 @@ export declare class Optimizer<T extends Record<string, any>> {
         readonly field: any;
         readonly indexName: string;
         readonly isFtsMatch: false;
-        readonly score: number;
+        readonly selectivity: number;
         readonly compositeVerifyFields: string[];
         readonly coveredFields: string[];
         readonly isIndexOrderSupported: boolean;
     } | null;
     /**
      * FTS 타입 인덱스의 선택도를 평가합니다.
-     * FTSTermCount 통계가 있으면 토큰 빈도 기반 동적 score를 산출합니다.
+     * FTSTermCount 통계가 있으면 실측 데이터 기반으로 선택도를 산출합니다.
      */
     evaluateFTSCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(indexName: string, config: any, query: Partial<DocumentDataplyQuery<V>>, queryFields: Set<string>, treeTx: BPTreeAsync<string | number, V>): {
-        readonly tree: BPTreeAsync<string | number, V>;
-        readonly condition: any;
-        readonly field: any;
-        readonly indexName: string;
-        readonly isFtsMatch: true;
-        readonly matchTokens: string[];
-        readonly score: number;
-        readonly compositeVerifyFields: readonly [];
-        readonly coveredFields: readonly [any];
-        readonly isIndexOrderSupported: false;
+        tree: BPTreeAsync<string | number, V>;
+        condition: any;
+        field: any;
+        indexName: string;
+        isFtsMatch: boolean;
+        matchTokens: string[];
+        selectivity: number;
+        compositeVerifyFields: never[];
+        coveredFields: any[];
+        isIndexOrderSupported: boolean;
     } | null;
     /**
-     * 실행할 최적의 인덱스를 선택합니다. (최적 드라이버 선택)
+     * 비용 계산: effectiveScanCost + sortPenalty
+     * - effectiveScanCost: 인덱스 순서 지원 + limit 존재 시 조기 종료 이점 반영
+     * - sortPenalty: 인메모리 정렬의 절대 문서 수 기반 비용
+     * - hasUncoveredFilters: 드라이버가 커버하지 못하는 비-FTS 필터 존재 여부
+     *   true일 경우 topK/N 조기 종료를 적용하지 않음
+     *   (uncovered 필터가 행을 탈락시킬 수 있어 topK개 이상 스캔 필요)
+     */
+    private calculateCost;
+    /**
+     * 실행할 최적의 인덱스를 선택합니다. (비용 기반 최적 드라이버 선택)
+     * cost = selectivity + sortPenalty (낮을수록 좋음)
      */
-    getSelectivityCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(query: Partial<DocumentDataplyQuery<V>>, orderByField?: string): Promise<{
+    getSelectivityCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(query: Partial<DocumentDataplyQuery<V>>, orderByField?: string, limit?: number, offset?: number): Promise<{
         driver: ({
             tree: BPTreeAsync<number, V>;
             condition: Partial<DocumentDataplyCondition<U>>;

package/dist/types/core/QueryManager.d.ts

@@ -18,7 +18,7 @@ export declare class QueryManager<T extends DocumentJSON> {
     private applyCandidateByFTSStream;
     private applyCandidateStream;
     getKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<Float64Array>;
-    getDriverKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<{
+    getDriverKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc', limit?: number, offset?: number): Promise<{
         keysStream: AsyncIterableIterator<number>;
         others: {
             tree: BPTreeAsync<string | number, DataplyTreeValue<Primitive>>;
@@ -45,7 +45,15 @@ export declare class QueryManager<T extends DocumentJSON> {
         condition: any;
     }[]): boolean;
     verifyValue(value: Primitive, condition: any): boolean;
-    adjustChunkSize(currentChunkSize: number, chunkTotalSize: number): number;
+    /**
+     * 최적화 공식: x = x * Math.sqrt(z / n)
+     * @param currentChunkSize 현재 청크 크기
+     * @param matchedCount 매칭된 문서 개수
+     * @param limit 최대 문서 개수
+     * @param chunkTotalSize 청크 내 문서 총 크기
+     * @returns
+     */
+    adjustChunkSize(currentChunkSize: number, matchedCount: number, limit: number, chunkTotalSize: number): number;
     processChunkedKeysWithVerify(keysStream: AsyncIterableIterator<number>, startIdx: number, initialChunkSize: number, limit: number, ftsConditions: {
         field: string;
         matchTokens: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.10-alpha.5",
+  "version": "0.0.10-alpha.7",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",