document-dataply 0.0.4-alpha.3 → 0.0.4-alpha.5

package/dist/cjs/index.js

@@ -7434,9 +7434,6 @@ var require_cjs = __commonJS({
           promises.push(writePage(pageId, data));
         }
         await Promise.all(promises);
-        if (restoredPages.size > 0) {
-          await this.clear();
-        }
       }
       /**
        * WAL에 페이지 데이터를 기록합니다 (Phase 1: Prepare).
@@ -7595,6 +7592,8 @@ var require_cjs = __commonJS({
       }
       /** LRU 캐시 (페이지 ID -> 페이지 데이터) */
       cache;
+      /** 디스크에 기록되지 않은 변경된 페이지들 (페이지 ID -> 페이지 데이터) */
+      dirtyPages = /* @__PURE__ */ new Map();
       /** 파일 크기 (논리적) */
       fileSize;
       /**
@@ -7604,12 +7603,16 @@ var require_cjs = __commonJS({
        * @returns 페이지 데이터
        */
       async read(pageId) {
-        const cached = this.cache.get(pageId);
-        if (cached) {
+        const dirty = this.dirtyPages.get(pageId);
+        if (dirty) {
           const copy = new Uint8Array(this.pageSize);
-          copy.set(cached);
+          copy.set(dirty);
           return copy;
         }
+        const cached = this.cache.get(pageId);
+        if (cached) {
+          return cached;
+        }
         const buffer = new Uint8Array(this.pageSize);
         const pageStartPos = pageId * this.pageSize;
         if (pageStartPos >= this.fileSize) {
@@ -7631,21 +7634,50 @@ var require_cjs = __commonJS({
         if (pageStartPos + this.pageSize > 512 * 1024 * 1024) {
           throw new Error(`[Safety Limit] File write exceeds 512MB limit at position ${pageStartPos}`);
         }
-        await this._writeToDisk(data, pageStartPos);
-        const cacheCopy = new Uint8Array(this.pageSize);
-        cacheCopy.set(data);
-        this.cache.set(pageId, cacheCopy);
+        const dataCopy = new Uint8Array(this.pageSize);
+        dataCopy.set(data);
+        this.dirtyPages.set(pageId, dataCopy);
+        this.cache.set(pageId, dataCopy);
         const endPosition = pageStartPos + this.pageSize;
         if (endPosition > this.fileSize) {
           this.fileSize = endPosition;
         }
       }
+      /**
+       * 더티 페이지들을 메인 디스크 파일에 일괄 기록합니다.
+       * WAL 체크포인트 시점에 호출되어야 합니다.
+       */
+      async flush() {
+        if (this.dirtyPages.size === 0) {
+          return;
+        }
+        const snapshot = new Map(this.dirtyPages);
+        const sortedPageIds = Array.from(snapshot.keys()).sort((a, b) => a - b);
+        for (const pageId of sortedPageIds) {
+          const data = snapshot.get(pageId);
+          const position = pageId * this.pageSize;
+          await this._writeToDisk(data, position);
+          this.dirtyPages.delete(pageId);
+        }
+      }
+      /**
+       * 메인 DB 파일의 물리적 동기화를 수행합니다 (fsync).
+       */
+      async sync() {
+        return new Promise((resolve, reject) => {
+          import_node_fs2.default.fsync(this.fileHandle, (err) => {
+            if (err) return reject(err);
+            resolve();
+          });
+        });
+      }
       /**
        * 페이지 삭제 (실제로는 캐시에서만 제거)
        * 실제 페이지 해제는 상위 레이어(FreeList)에서 관리합니다.
        * @param pageId 페이지 ID
        */
       async delete(pageId) {
+        this.dirtyPages.delete(pageId);
         this.cache.delete(pageId);
       }
       /**
@@ -7654,6 +7686,9 @@ var require_cjs = __commonJS({
        * @returns 존재하면 true
        */
       async exists(pageId) {
+        if (this.dirtyPages.has(pageId)) {
+          return true;
+        }
         const pageStartPos = pageId * this.pageSize;
         return pageStartPos < this.fileSize;
       }
@@ -7708,6 +7743,25 @@ var require_cjs = __commonJS({
       walManager;
       pageManagerFactory;
       pageStrategy;
+      /** 글로벌 동기화(체크포인트/커밋)를 위한 Mutex */
+      lockPromise = Promise.resolve();
+      /**
+       * 글로벌 동기화 범위 내에서 작업을 실행합니다.
+       * @param task 수행할 비동기 작업
+       */
+      async runGlobalLock(task) {
+        const previous = this.lockPromise;
+        let resolveLock;
+        this.lockPromise = new Promise((resolve) => {
+          resolveLock = resolve;
+        });
+        await previous;
+        try {
+          return await task();
+        } finally {
+          resolveLock();
+        }
+      }
       /**
        * Initializes the page file system.
        * Performs WAL recovery if necessary.
@@ -7717,6 +7771,7 @@ var require_cjs = __commonJS({
           await this.walManager.recover(async (pageId, data) => {
             await this.pageStrategy.write(pageId, data);
           });
+          await this.checkpoint();
         }
       }
       /**
@@ -8016,19 +8071,35 @@ var require_cjs = __commonJS({
       async commitToWAL(dirtyPages) {
         if (this.walManager) {
           await this.walManager.prepareCommit(dirtyPages);
-          await this.walManager.finalizeCommit(false);
+          await this.walManager.finalizeCommit(true);
         }
       }
+      /**
+       * 체크포인트를 수행합니다.
+       * 1. 메모리의 더티 페이지를 DB 파일에 기록 (Flush)
+       * 2. DB 파일 물리적 동기화 (Sync/fsync)
+       * 3. WAL 로그 파일 비우기 (Clear/Truncate)
+       */
+      async checkpoint() {
+        await this.runGlobalLock(async () => {
+          await this.pageStrategy.flush();
+          await this.pageStrategy.sync();
+          if (this.walManager) {
+            await this.walManager.clear();
+          }
+        });
+      }
       /**
        * Closes the page file system.
        */
       async close() {
+        await this.checkpoint();
         if (this.walManager) {
-          await this.walManager.clear();
           this.walManager.close();
         }
       }
     };
+    var import_node_os = __toESM2(require("node:os"));
     var TextCodec = class _TextCodec {
       static TextEncoder = new TextEncoder();
       static TextDecoder = new TextDecoder();
@@ -8268,11 +8339,14 @@ var require_cjs = __commonJS({
         this.maxBodySize = this.pfs.pageSize - DataPageManager.CONSTANT.SIZE_PAGE_HEADER;
         this.order = this.getOptimalOrder(pfs.pageSize, IndexPageManager.CONSTANT.SIZE_KEY, IndexPageManager.CONSTANT.SIZE_VALUE);
         this.strategy = new RowIdentifierStrategy(this.order, pfs, txContext);
+        const budget = import_node_os.default.freemem() * 0.05;
+        const nodeMemory = this.order * 24 + 256;
+        const capacity = Math.max(1e3, Math.min(1e6, Math.floor(budget / nodeMemory)));
         this.bptree = new BPTreeAsync2(
           this.strategy,
           new NumericComparator(),
           {
-            capacity: this.options.pageCacheCapacity
+            capacity
           }
         );
       }
@@ -8639,6 +8713,30 @@ var require_cjs = __commonJS({
         }
         return this.fetchRowByRid(pk, rid, tx);
       }
+      /**
+       * Selects multiple rows by their PKs in a single B+ Tree traversal.
+       * @param pks Array of PKs to look up
+       * @param tx Transaction
+       * @returns Array of raw data of the rows in the same order as input PKs
+       */
+      async selectMany(pks, tx) {
+        if (pks.length === 0) {
+          return [];
+        }
+        const minPk = Math.min(...pks);
+        const maxPk = Math.max(...pks);
+        const pkSet = new Set(pks);
+        const resultMap = /* @__PURE__ */ new Map();
+        const btx = await this.getBPTreeTransaction(tx);
+        const stream = btx.whereStream({ gte: minPk, lte: maxPk });
+        for await (const [rid, pk] of stream) {
+          if (pkSet.has(pk)) {
+            const rowData = await this.fetchRowByRid(pk, rid, tx);
+            resultMap.set(pk, rowData);
+          }
+        }
+        return pks.map((pk) => resultMap.get(pk) ?? null);
+      }
       async fetchRowByRid(pk, rid, tx) {
         this.keyManager.setBufferFromKey(rid, this.ridBuffer);
         const pageId = this.keyManager.getPageId(this.ridBuffer);
@@ -8836,18 +8934,24 @@ var require_cjs = __commonJS({
             await hook();
           }
         });
-        if (this.pfs.wal && this.dirtyPages.size > 0) {
-          await this.pfs.wal.prepareCommit(this.dirtyPages);
-          await this.pfs.wal.writeCommitMarker();
-        }
-        for (const [pageId, data] of this.dirtyPages) {
-          await this.pageStrategy.write(pageId, data);
-        }
-        if (this.pfs.wal) {
-          this.pfs.wal.incrementWrittenPages(this.dirtyPages.size);
-          if (this.pfs.wal.shouldCheckpoint(this.pfs.options.walCheckpointThreshold)) {
-            await this.pfs.wal.clear();
+        let shouldTriggerCheckpoint = false;
+        await this.pfs.runGlobalLock(async () => {
+          if (this.pfs.wal && this.dirtyPages.size > 0) {
+            await this.pfs.wal.prepareCommit(this.dirtyPages);
+            await this.pfs.wal.writeCommitMarker();
           }
+          for (const [pageId, data] of this.dirtyPages) {
+            await this.pageStrategy.write(pageId, data);
+          }
+          if (this.pfs.wal) {
+            this.pfs.wal.incrementWrittenPages(this.dirtyPages.size);
+            if (this.pfs.wal.shouldCheckpoint(this.pfs.options.walCheckpointThreshold)) {
+              shouldTriggerCheckpoint = true;
+            }
+          }
+        });
+        if (shouldTriggerCheckpoint) {
+          await this.pfs.checkpoint();
         }
         this.dirtyPages.clear();
         this.undoPages.clear();
@@ -9258,6 +9362,19 @@ var require_cjs = __commonJS({
           return this.textCodec.decode(data);
         }, tx);
       }
+      async selectMany(pks, asRaw = false, tx) {
+        if (!this.initialized) {
+          throw new Error("Dataply instance is not initialized");
+        }
+        return this.runWithDefault(async (tx2) => {
+          const results = await this.rowTableEngine.selectMany(pks, tx2);
+          return results.map((data) => {
+            if (data === null) return null;
+            if (asRaw) return data;
+            return this.textCodec.decode(data);
+          });
+        }, tx);
+      }
       /**
        * Closes the dataply file.
        */
@@ -9346,6 +9463,9 @@ var require_cjs = __commonJS({
       async select(pk, asRaw = false, tx) {
         return this.api.select(pk, asRaw, tx);
       }
+      async selectMany(pks, asRaw = false, tx) {
+        return this.api.selectMany(pks, asRaw, tx);
+      }
       /**
        * Closes the dataply file.
        */
@@ -9406,6 +9526,7 @@ __export(src_exports, {
 module.exports = __toCommonJS(src_exports);
 
 // src/core/document.ts
+var os = __toESM(require("node:os"));
 var import_dataply3 = __toESM(require_cjs());
 
 // src/core/bptree/documentStrategy.ts
@@ -10118,7 +10239,9 @@ var DocumentDataply = class _DocumentDataply {
     } = options;
     const self = this;
     const stream = this.api.streamWithDefault(async function* (tx2) {
-      const keys = await self.getKeys(query, orderByField, sortOrder);
+      const keySet = await self.getKeys(query, orderByField, sortOrder);
+      const keys = new Uint32Array(keySet);
+      const totalKeys = keys.length;
       const selectivity = await self.getSelectivityCandidate(
         self.verboseQuery(query),
         orderByField
@@ -10127,12 +10250,29 @@ var DocumentDataply = class _DocumentDataply {
       if (selectivity) {
         selectivity.rollback();
       }
+      let CHUNK_SIZE = 100;
       if (!isDriverOrderByField && orderByField) {
         const results = [];
-        for (const key of keys) {
-          const stringified = await self.api.select(key, false, tx2);
-          if (!stringified) continue;
-          results.push(JSON.parse(stringified));
+        let i = 0;
+        while (i < totalKeys) {
+          const chunk = Array.from(keys.subarray(i, i + CHUNK_SIZE));
+          const stringifiedResults = await self.api.selectMany(chunk, false, tx2);
+          if (i === 0) {
+            let totalBytes = 0;
+            let count = 0;
+            for (const s of stringifiedResults) {
+              if (s) {
+                totalBytes += s.length;
+                count++;
+              }
+            }
+            const avgSize = count > 0 ? totalBytes / count : 1024;
+            CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+          }
+          for (const stringified of stringifiedResults) {
+            if (stringified) results.push(JSON.parse(stringified));
+          }
+          i += chunk.length;
         }
         results.sort((a, b) => {
           const aVal = a[orderByField] ?? a._id;
@@ -10147,19 +10287,36 @@ var DocumentDataply = class _DocumentDataply {
           yield doc;
         }
       } else {
-        let i = 0;
         let yieldedCount = 0;
-        for (const key of keys) {
-          if (yieldedCount >= limit) break;
-          if (i < offset) {
-            i++;
-            continue;
+        let i = offset;
+        let isFirst = true;
+        while (i < totalKeys && yieldedCount < limit) {
+          const currentChunkLimit = isFirst ? 100 : CHUNK_SIZE;
+          const remainingLimit = limit - yieldedCount;
+          const pksToFetchCount = Math.min(currentChunkLimit, remainingLimit);
+          const chunk = Array.from(keys.subarray(i, i + pksToFetchCount));
+          const stringifiedResults = await self.api.selectMany(chunk, false, tx2);
+          if (isFirst) {
+            let totalBytes = 0;
+            let count = 0;
+            for (const s of stringifiedResults) {
+              if (s) {
+                totalBytes += s.length;
+                count++;
+              }
+            }
+            const avgSize = count > 0 ? totalBytes / count : 1024;
+            CHUNK_SIZE = Math.max(32, Math.floor(os.freemem() * 0.1 / avgSize));
+            isFirst = false;
+          }
+          for (const stringified of stringifiedResults) {
+            if (stringified) {
+              yield JSON.parse(stringified);
+              yieldedCount++;
+              if (yieldedCount >= limit) break;
+            }
           }
-          const stringified = await self.api.select(key, false, tx2);
-          if (!stringified) continue;
-          yield JSON.parse(stringified);
-          yieldedCount++;
-          i++;
+          i += chunk.length;
         }
       }
     }, tx);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.4-alpha.3",
+  "version": "0.0.4-alpha.5",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -42,7 +42,7 @@
     "dataply"
   ],
   "dependencies": {
-    "dataply": "^0.0.20-alpha.2"
+    "dataply": "^0.0.20-alpha.6"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",
@@ -51,4 +51,4 @@
     "ts-jest": "^29.4.6",
     "typescript": "^5.9.3"
   }
-}
+}