document-dataply 0.0.9 → 0.0.10-alpha.1

package/dist/cjs/index.js

@@ -6478,65 +6478,22 @@ var require_cjs = __commonJS({
       }
       return (crc ^ -1) >>> 0;
     }
-    function calcThreshold(sortedGaps, n) {
-      const gLen = sortedGaps.length;
-      if (gLen === 0) return 0;
-      const median = sortedGaps[Math.floor(gLen * 0.5)];
-      const q1 = sortedGaps[Math.floor(gLen * 0.25)];
-      const q3 = sortedGaps[Math.floor(gLen * 0.75)];
-      const iqr = q3 - q1;
-      const logN = Math.max(1, Math.log10(n));
-      if (iqr > 0) {
-        const threshold2 = q3 + iqr * 1.5 * logN;
-        const minJump = Math.max(median * 5, 20);
-        return Math.max(threshold2, minJump);
-      }
-      const baseGap = median > 0 ? median : 1;
-      const p90 = sortedGaps[Math.floor(gLen * 0.9)];
-      if (p90 > baseGap) {
-        const threshold2 = baseGap + (p90 - baseGap) * 0.5 * logN;
-        return Math.max(threshold2, baseGap * 5, 20);
-      }
-      let mean = 0;
-      for (let i = 0; i < gLen; i++) mean += sortedGaps[i];
-      mean /= gLen;
-      let variance = 0;
-      for (let i = 0; i < gLen; i++) {
-        const d = sortedGaps[i] - mean;
-        variance += d * d;
-      }
-      const stddev = Math.sqrt(variance / gLen);
-      if (stddev === 0) {
-        return baseGap * 2;
-      }
-      const threshold = mean + stddev * logN;
-      return Math.max(threshold, baseGap * 5, 20);
-    }
-    function clusterNumbers(numbers, maxGap) {
+    function clusterNumbersByPagination(numbers, pagingSize, startPageId = 0) {
       const n = numbers.length;
       if (n === 0) return [];
       if (n === 1) return [new Float64Array([numbers[0]])];
       const sorted = (numbers instanceof Float64Array ? numbers.slice() : Float64Array.from(numbers)).sort();
-      const gaps = new Float64Array(n - 1);
-      for (let i = 0, len = n - 1; i < len; i++) {
-        gaps[i] = sorted[i + 1] - sorted[i];
-      }
-      const sortedGaps = gaps.slice().sort();
-      let threshold;
-      if (maxGap !== void 0) {
-        threshold = maxGap;
-      } else {
-        threshold = calcThreshold(sortedGaps, n);
-      }
       const clusters = [];
-      let clusterStart = 0;
+      let start = 0;
       for (let i = 0, len = n - 1; i < len; i++) {
-        if (gaps[i] > threshold) {
-          clusters.push(sorted.subarray(clusterStart, i + 1));
-          clusterStart = i + 1;
+        const paginationIndex = Math.floor((sorted[i] - startPageId) / pagingSize);
+        const nextPaginationIndex = Math.floor((sorted[i + 1] - startPageId) / pagingSize);
+        if (paginationIndex !== nextPaginationIndex) {
+          clusters.push(sorted.subarray(start, i + 1));
+          start = i + 1;
         }
       }
-      clusters.push(sorted.subarray(clusterStart));
+      clusters.push(sorted.subarray(start));
       return clusters;
     }
     var Row = class _Row {
@@ -9196,10 +9153,15 @@ var require_cjs = __commonJS({
         }
         const metadataPage = await this.pfs.getMetadata(tx);
         const manager = this.factory.getManagerFromType(MetadataPageManager.CONSTANT.PAGE_TYPE_METADATA);
+        const pageSize = manager.getPageSize(metadataPage);
+        const pageCount = manager.getPageCount(metadataPage);
+        const rowCount = manager.getRowCount(metadataPage);
+        const usage = pageCount / Math.pow(2, 32);
         return {
-          pageSize: manager.getPageSize(metadataPage),
-          pageCount: manager.getPageCount(metadataPage),
-          rowCount: manager.getRowCount(metadataPage)
+          pageSize,
+          pageCount,
+          rowCount,
+          usage
         };
       }
       /**
@@ -9470,19 +9432,34 @@ var require_cjs = __commonJS({
        * @returns Array of raw data of the rows in the same order as input PKs
        */
       async selectMany(pks, tx) {
+        const collections = await this.collectItemsByPage(pks, tx);
+        return this.fetchRowsByRids(collections, pks.length, tx);
+      }
+      /**
+       * Collects items by page ID to minimize I/O.
+       * @param pks Array of PKs to look up
+       * @param tx Transaction
+       * @returns Map of page ID to array of {pk, slotIndex, index} pairs
+       */
+      async collectItemsByPage(pks, tx) {
         if (pks.length === 0) {
-          return [];
+          return /* @__PURE__ */ new Map();
         }
         const pkIndexMap = /* @__PURE__ */ new Map();
         for (let i = 0, len = pks.length; i < len; i++) {
           pkIndexMap.set(pks[i], i);
         }
-        const validCount = pks.length;
-        const pkArray = new Float64Array(validCount).fill(0);
-        const ridArray = new Float64Array(validCount).fill(0);
-        const indexArray = new Float64Array(validCount).fill(0);
         const btx = await this.getBPTreeTransaction(tx);
-        const clusters = clusterNumbers(pks, this.order / 2);
+        const clusters = clusterNumbersByPagination(pks, this.order, 1);
+        const collections = /* @__PURE__ */ new Map();
+        const insertToCollections = (pk, rid, index) => {
+          const slotIndex = rid % 65536;
+          const pageId = Math.floor(rid / 65536);
+          if (!collections.has(pageId)) {
+            collections.set(pageId, []);
+          }
+          collections.get(pageId).push({ pk, slotIndex, index });
+        };
         for (let i = 0, len = clusters.length; i < len; i++) {
           const cluster = clusters[i];
           const minPk = cluster[0];
@@ -9493,9 +9470,7 @@ var require_cjs = __commonJS({
               const rid = keys.values().next().value;
               const index = pkIndexMap.get(minPk);
               if (index !== void 0) {
-                pkArray[index] = minPk;
-                ridArray[index] = rid;
-                indexArray[index] = index;
+                insertToCollections(minPk, rid, index);
               }
             }
             continue;
@@ -9504,13 +9479,11 @@ var require_cjs = __commonJS({
           for await (const [rid, pk] of stream) {
             const index = pkIndexMap.get(pk);
             if (index !== void 0) {
-              pkArray[index] = pk;
-              ridArray[index] = rid;
-              indexArray[index] = index;
+              insertToCollections(pk, rid, index);
             }
           }
         }
-        return this.fetchRowsByRids(validCount, pkArray, ridArray, indexArray, tx);
+        return collections;
       }
       /**
        * Fetches multiple rows by their RID and PK combinations, grouping by page ID to minimize I/O.
@@ -9518,25 +9491,12 @@ var require_cjs = __commonJS({
        * @param tx Transaction
        * @returns Array of row data in the same order as input PKs
        */
-      async fetchRowsByRids(validCount, pkArray, ridArray, indexArray, tx) {
-        const result = new Array(validCount).fill(null);
-        if (validCount === 0) return result;
-        const pageGroupMap = /* @__PURE__ */ new Map();
-        for (let i = 0; i < validCount; i++) {
-          const pk = pkArray[i];
-          const rid = ridArray[i];
-          const index = indexArray[i];
-          if (pk === 0 && rid === 0 && index === 0) continue;
-          const slotIndex = rid % 65536;
-          const pageId = Math.floor(rid / 65536);
-          if (!pageGroupMap.has(pageId)) {
-            pageGroupMap.set(pageId, []);
-          }
-          pageGroupMap.get(pageId).push({ pk, slotIndex, index });
-        }
-        const sortedPageIds = Array.from(pageGroupMap.keys()).sort((a, b) => a - b);
+      async fetchRowsByRids(collections, itemsCount, tx) {
+        const result = new Array(itemsCount).fill(null);
+        if (itemsCount === 0) return result;
+        const sortedPageIds = Array.from(collections.keys()).sort((a, b) => a - b);
         await Promise.all(sortedPageIds.map(async (pageId) => {
-          const items = pageGroupMap.get(pageId);
+          const items = collections.get(pageId);
           const page = await this.pfs.get(pageId, tx);
           if (!this.factory.isDataPage(page)) {
             throw new Error(`Page ${pageId} is not a data page`);
@@ -10283,9 +10243,11 @@ var require_cjs = __commonJS({
         if (!this.initialized) {
           throw new Error("Dataply instance is not initialized");
         }
-        return this.hook.trigger("close", void 0, async () => {
-          await this.pfs.close();
-          import_node_fs3.default.closeSync(this.fileHandle);
+        return this.runWithDefaultWrite(() => {
+          return this.hook.trigger("close", void 0, async () => {
+            await this.pfs.close();
+            import_node_fs3.default.closeSync(this.fileHandle);
+          });
         });
       }
     };
@@ -10502,38 +10464,50 @@ var DocumentSerializeStrategyAsync = class extends import_dataply.SerializeStrat
 
 // src/core/bptree/documentComparator.ts
 var import_dataply2 = __toESM(require_cjs());
-function compareDiff(a, b) {
-  if (typeof a !== "string" && typeof b !== "string") {
-    return +a - +b;
-  }
-  return (a + "").localeCompare(b + "");
-}
-function compareValue(a, b) {
-  const aArr = Array.isArray(a);
-  const bArr = Array.isArray(b);
-  if (!aArr && !bArr) {
-    return compareDiff(a, b);
-  }
-  const aList = aArr ? a : [a];
-  const bList = bArr ? b : [b];
-  const len = Math.min(aList.length, bList.length);
-  for (let i = 0; i < len; i++) {
-    const diff = compareDiff(aList[i], bList[i]);
-    if (diff !== 0) return diff;
-  }
-  return 0;
-}
 var DocumentValueComparator = class extends import_dataply2.ValueComparator {
+  _intlComparator = new Intl.Collator(void 0, {
+    numeric: true,
+    sensitivity: "variant",
+    usage: "sort"
+  });
   primaryAsc(a, b) {
-    return compareValue(a.v, b.v);
+    return this._compareValue(a.v, b.v);
   }
   asc(a, b) {
-    const diff = compareValue(a.v, b.v);
+    const diff = this._compareValue(a.v, b.v);
     return diff === 0 ? a.k - b.k : diff;
   }
   match(value) {
     return value.v + "";
   }
+  /**
+   * 두 Primitive 값을 비교합니다.
+   */
+  _compareDiff(a, b) {
+    if (typeof a === "string" && typeof b === "string") {
+      return this._intlComparator.compare(a, b);
+    }
+    return +a - +b;
+  }
+  /**
+   * 두 v 값을 비교합니다. v는 Primitive 또는 Primitive[] (복합 인덱스)일 수 있습니다.
+   * 배열인 경우 element-by-element로 비교합니다.
+   */
+  _compareValue(a, b) {
+    const aArr = Array.isArray(a);
+    const bArr = Array.isArray(b);
+    if (!aArr && !bArr) {
+      return this._compareDiff(a, b);
+    }
+    const aList = aArr ? a : [a];
+    const bList = bArr ? b : [b];
+    const len = Math.min(aList.length, bList.length);
+    for (let i = 0; i < len; i++) {
+      const diff = this._compareDiff(aList[i], bList[i]);
+      if (diff !== 0) return diff;
+    }
+    return 0;
+  }
 };
 
 // src/utils/catchPromise.ts
@@ -12000,7 +11974,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
       orderBy: orderByField
     } = options;
     const self = this;
-    const stream = this.streamWithDefault(async function* (tx2) {
+    const stream = () => this.streamWithDefault(async function* (tx2) {
       const ftsConditions = [];
       for (const field in query) {
         const q = query[field];
@@ -12102,7 +12076,7 @@ var DocumentDataplyAPI = class extends import_dataply3.DataplyAPI {
     }, tx);
     const drain = async () => {
       const result = [];
-      for await (const document of stream) {
+      for await (const document of stream()) {
         result.push(document);
       }
       return result;

package/dist/types/core/bptree/documentComparator.d.ts

@@ -1,7 +1,17 @@
 import type { DataplyTreeValue, Primitive } from '../../types';
 import { ValueComparator } from 'dataply';
 export declare class DocumentValueComparator<T extends DataplyTreeValue<U>, U extends Primitive> extends ValueComparator<T> {
+    private readonly _intlComparator;
     primaryAsc(a: T, b: T): number;
     asc(a: T, b: T): number;
     match(value: T): string;
+    /**
+     * 두 Primitive 값을 비교합니다.
+     */
+    private _compareDiff;
+    /**
+     * 두 v 값을 비교합니다. v는 Primitive 또는 Primitive[] (복합 인덱스)일 수 있습니다.
+     * 배열인 경우 element-by-element로 비교합니다.
+     */
+    private _compareValue;
 }

package/dist/types/core/document.d.ts

@@ -122,7 +122,7 @@ export declare class DocumentDataply<T extends DocumentJSON> {
      * @throws Error if query or orderBy contains non-indexed fields
      */
     select(query: Partial<DocumentDataplyQuery<T>>, options?: DocumentDataplyQueryOptions, tx?: Transaction): {
-        stream: AsyncIterableIterator<DataplyDocument<T>>;
+        stream: () => AsyncIterableIterator<DataplyDocument<T>>;
         drain: () => Promise<DataplyDocument<T>[]>;
     };
     /**

package/dist/types/core/documentAPI.d.ts

@@ -290,7 +290,7 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
      * @throws Error if query or orderBy contains non-indexed fields
      */
     selectDocuments(query: Partial<DocumentDataplyQuery<T>>, options?: DocumentDataplyQueryOptions, tx?: Transaction): {
-        stream: AsyncIterableIterator<DataplyDocument<T>>;
+        stream: () => AsyncIterableIterator<DataplyDocument<T>>;
         drain: () => Promise<DataplyDocument<T>[]>;
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.9",
+  "version": "0.0.10-alpha.1",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.24"
+    "dataply": "^0.0.25-alpha.1"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",

package/readme.md

@@ -8,7 +8,7 @@
 > **This project is currently in the Alpha stage.**
 > APIs and internal structures may change significantly between versions. Use with caution in production environments.
 
-`document-dataply` is a **pure JavaScript** high-performance document-oriented database library built on top of the [`dataply`](https://github.com/izure1/dataply) record storage engine. It is designed to handle **millions of rows** with high stability, providing a structured way to store, index, and query JSON-style documents.
+`document-dataply` is a **pure JavaScript** high-performance document-oriented database library built on top of the [`dataply`](https://github.com/izure1/dataply) record storage engine. It is designed to handle at least millions of documents with high stability, providing a structured way to store, index, and query JSON-style documents.
 
 ## Key Features
 
@@ -85,7 +85,7 @@ async function main() {
   // Get all results
   const allResults = await query.drain();
   // Or iterate through results
-  for await (const doc of query.stream) {
+  for await (const doc of query.stream()) {
     console.log(doc);
   }
 
@@ -161,7 +161,7 @@ For detailed usage and error handling patterns, see the [Transaction Guide (TRAN
 
 ### Updating and Deleting
 
-`document-dataply` provides flexible ways to update or delete documents based on query results. All these operations are **Stream-based**, allowing you to handle millions of records without memory concerns.
+`document-dataply` provides flexible ways to update or delete documents based on query results. All these operations are **Stream-based**, allowing you to handle at least millions of records without memory concerns.
 
 - **Partial Update**: Modify only specific fields or use a function for dynamic updates.
 - **Full Update**: Replace the entire document while preserving the original `_id`.
@@ -188,7 +188,7 @@ For more information on performance optimization and advanced features, see [TIP
 
 - **Query Optimization**: Automatic index selection for maximum performance.
 - **Sorting and Pagination**: Detailed usage of `limit`, `orderBy`, and `sortOrder`.
-- **Memory Management**: When to use `stream` vs `drain()`.
+- **Memory Management**: When to use `stream()` vs `drain()`.
 - **Performance**: Optimizing bulk data insertion using `insertBatch`.
 - **Indexing Policies**: Dynamic index creation and automatic backfilling.
 - **Composite Indexes**: Indexing multiple fields for complex queries.
@@ -226,7 +226,7 @@ Inserts multiple documents efficiently. Returns an array of `_ids` (`number[]`).
 ### `db.select(query, options?, tx?)`
 Searches for documents matching the query. Passing an empty object (`{}`) as the `query` retrieves all documents.
 Returns an object `{ stream, drain }`.
-- `stream`: An async iterator to traverse results one by one.
+- `stream()`: An async iterator to traverse results one by one.
 - `drain()`: A promise that resolves to an array of all matching documents.
 
 ### `db.partialUpdate(query, newFields, tx?)`