document-dataply 0.0.14-alpha.1 → 0.0.14-alpha.2

package/dist/cjs/index.js

@@ -82,6 +82,8 @@ var require_cjs = __commonJS({
       IndexPageManager: () => IndexPageManager,
       InvertedWeakMap: () => InvertedWeakMap,
       LRUMap: () => LRUMap2,
+      Logger: () => Logger2,
+      LoggerManager: () => LoggerManager,
       MVCCStrategy: () => MVCCStrategy2,
       MVCCTransaction: () => MVCCTransaction2,
       MetadataPageManager: () => MetadataPageManager,
@@ -476,11 +478,7 @@ var require_cjs = __commonJS({
        */
       rollback() {
         const { created, updated, deleted } = this.getResultEntries();
-        this.writeBuffer.clear();
-        this.deleteBuffer.clear();
-        this.createdKeys.clear();
-        this.deletedValues.clear();
-        this.originallyExisted.clear();
+        this._cleanupAll();
         this.committed = true;
         if (this.root !== this) {
           this.root.activeTransactions.delete(this);
@@ -516,6 +514,19 @@ var require_cjs = __commonJS({
         }
         return Array.from(conflicts);
       }
+      /**
+       * Cleans up all buffers and history.
+       * This method is called by the commit method.
+       */
+      _cleanupAll() {
+        this.writeBuffer.clear();
+        this.deleteBuffer.clear();
+        this.createdKeys.clear();
+        this.deletedValues.clear();
+        this.originallyExisted.clear();
+        this.keyVersions.clear();
+        this.bufferHistory.clear();
+      }
       /**
        * Cleans up both deletedCache and versionIndex based on minActiveVersion.
        * Root transactions call this after commit to reclaim memory.
@@ -707,6 +718,7 @@ var require_cjs = __commonJS({
         if (this.parent) {
           const failure = this.parent._merge(this);
           if (failure) {
+            this.rollback();
             return {
               label,
               success: false,
@@ -717,11 +729,13 @@ var require_cjs = __commonJS({
               deleted
             };
           }
+          this._cleanupAll();
           this.committed = true;
         } else {
           if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
             const failure = this._merge(this);
             if (failure) {
+              this.rollback();
               return {
                 label,
                 success: false,
@@ -732,13 +746,7 @@ var require_cjs = __commonJS({
                 deleted: []
               };
             }
-            this.writeBuffer.clear();
-            this.deleteBuffer.clear();
-            this.createdKeys.clear();
-            this.deletedValues.clear();
-            this.originallyExisted.clear();
-            this.keyVersions.clear();
-            this.bufferHistory.clear();
+            this._cleanupAll();
             this.localVersion = 0;
             this.snapshotVersion = this.version;
           }
@@ -1372,6 +1380,7 @@ var require_cjs = __commonJS({
         if (this.parent) {
           const failure = await this.parent._merge(this);
           if (failure) {
+            this.rollback();
             return {
               label,
               success: false,
@@ -1382,11 +1391,13 @@ var require_cjs = __commonJS({
               deleted
             };
           }
+          this._cleanupAll();
           this.committed = true;
         } else {
           if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
             const failure = await this._merge(this);
             if (failure) {
+              this.rollback();
               return {
                 label,
                 success: false,
@@ -1397,13 +1408,7 @@ var require_cjs = __commonJS({
                 deleted: []
               };
             }
-            this.writeBuffer.clear();
-            this.deleteBuffer.clear();
-            this.createdKeys.clear();
-            this.deletedValues.clear();
-            this.originallyExisted.clear();
-            this.keyVersions.clear();
-            this.bufferHistory.clear();
+            this._cleanupAll();
             this.localVersion = 0;
             this.snapshotVersion = this.version;
           }
@@ -2958,8 +2963,12 @@ var require_cjs = __commonJS({
             result = this.rootTx.commit(label);
             if (result.success) {
               this.rootTx.rootId = this.rootId;
+            } else {
+              this.mvcc.rollback();
             }
           }
+        } else {
+          this.mvcc.rollback();
         }
         return result;
       }
@@ -4160,8 +4169,12 @@ var require_cjs = __commonJS({
             result = await this.rootTx.commit(label);
             if (result.success) {
               this.rootTx.rootId = this.rootId;
+            } else {
+              this.mvcc.rollback();
             }
           }
+        } else {
+          this.mvcc.rollback();
         }
         return result;
       }
@@ -5426,11 +5439,7 @@ var require_cjs = __commonJS({
        */
       rollback() {
         const { created, updated, deleted } = this.getResultEntries();
-        this.writeBuffer.clear();
-        this.deleteBuffer.clear();
-        this.createdKeys.clear();
-        this.deletedValues.clear();
-        this.originallyExisted.clear();
+        this._cleanupAll();
         this.committed = true;
         if (this.root !== this) {
           this.root.activeTransactions.delete(this);
@@ -5466,6 +5475,19 @@ var require_cjs = __commonJS({
         }
         return Array.from(conflicts);
       }
+      /**
+       * Cleans up all buffers and history.
+       * This method is called by the commit method.
+       */
+      _cleanupAll() {
+        this.writeBuffer.clear();
+        this.deleteBuffer.clear();
+        this.createdKeys.clear();
+        this.deletedValues.clear();
+        this.originallyExisted.clear();
+        this.keyVersions.clear();
+        this.bufferHistory.clear();
+      }
       /**
        * Cleans up both deletedCache and versionIndex based on minActiveVersion.
        * Root transactions call this after commit to reclaim memory.
@@ -5657,6 +5679,7 @@ var require_cjs = __commonJS({
         if (this.parent) {
           const failure = this.parent._merge(this);
           if (failure) {
+            this.rollback();
             return {
               label,
               success: false,
@@ -5667,11 +5690,13 @@ var require_cjs = __commonJS({
               deleted
             };
           }
+          this._cleanupAll();
           this.committed = true;
         } else {
           if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
             const failure = this._merge(this);
             if (failure) {
+              this.rollback();
               return {
                 label,
                 success: false,
@@ -5682,13 +5707,7 @@ var require_cjs = __commonJS({
                 deleted: []
               };
             }
-            this.writeBuffer.clear();
-            this.deleteBuffer.clear();
-            this.createdKeys.clear();
-            this.deletedValues.clear();
-            this.originallyExisted.clear();
-            this.keyVersions.clear();
-            this.bufferHistory.clear();
+            this._cleanupAll();
             this.localVersion = 0;
             this.snapshotVersion = this.version;
           }
@@ -6322,6 +6341,7 @@ var require_cjs = __commonJS({
         if (this.parent) {
           const failure = await this.parent._merge(this);
           if (failure) {
+            this.rollback();
             return {
               label,
               success: false,
@@ -6332,11 +6352,13 @@ var require_cjs = __commonJS({
               deleted
             };
           }
+          this._cleanupAll();
           this.committed = true;
         } else {
           if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
             const failure = await this._merge(this);
             if (failure) {
+              this.rollback();
               return {
                 label,
                 success: false,
@@ -6347,13 +6369,7 @@ var require_cjs = __commonJS({
                 deleted: []
               };
             }
-            this.writeBuffer.clear();
-            this.deleteBuffer.clear();
-            this.createdKeys.clear();
-            this.deletedValues.clear();
-            this.originallyExisted.clear();
-            this.keyVersions.clear();
-            this.bufferHistory.clear();
+            this._cleanupAll();
             this.localVersion = 0;
             this.snapshotVersion = this.version;
           }
@@ -9967,10 +9983,10 @@ var require_cjs = __commonJS({
           if (shouldTriggerCheckpoint) {
             await this.pfs.checkpoint();
           }
+        } finally {
           this.dirtyPages.clear();
           this.undoPages.clear();
           this.releaseAllLocks();
-        } finally {
           if (this._writeLockRelease) {
             this._writeLockRelease();
             this._writeLockRelease = null;
@@ -10053,10 +10069,10 @@ var require_cjs = __commonJS({
         return level >= this.level;
       }
       create(moduleName) {
-        return new Logger(this, moduleName);
+        return new Logger2(this, moduleName);
       }
     };
-    var Logger = class {
+    var Logger2 = class {
       constructor(parent, moduleName) {
         this.parent = parent;
         this.moduleName = moduleName;
@@ -11699,6 +11715,71 @@ var DocumentSerializeStrategyAsync = class extends import_dataply2.SerializeStra
   }
 };
 
+// src/utils/eventLoopManager.ts
+function yieldEventLoop() {
+  return new Promise(setImmediate);
+}
+var DeadlineChunker = class {
+  /**
+   * 이벤트 루프를 막을 최대 허용 시간
+   */
+  targetMs;
+  /**
+   * 현재 chunk size
+   */
+  chunkSize;
+  /**
+   * Exponential Weighted Moving Average
+   */
+  ewmaMs;
+  /**
+   * EWMA 평활화 계수
+   */
+  alpha;
+  constructor(startChunkSize = 0, targetMs = 5, alpha = 0.5) {
+    this.chunkSize = startChunkSize;
+    this.targetMs = targetMs;
+    this.alpha = alpha;
+    this.ewmaMs = null;
+  }
+  /**
+   * EWMA 평활화 계수를 사용하여 평균 처리 시간을 업데이트합니다.
+   */
+  updateEstimate(elapsed, count) {
+    const msPerItem = elapsed / count;
+    this.ewmaMs = this.ewmaMs === null ? msPerItem : this.alpha * msPerItem + (1 - this.alpha) * this.ewmaMs;
+  }
+  /**
+   * 현재 chunk size를 업데이트합니다.
+   */
+  nextChunkSize() {
+    if (!this.ewmaMs || this.ewmaMs === 0) return this.chunkSize;
+    const next = Math.floor(this.targetMs / this.ewmaMs);
+    return Math.max(1, next);
+  }
+  /**
+   * 주어진 items를 chunk로 분할하여 처리합니다.
+   */
+  async processInChunks(items, processFn) {
+    let i = 0;
+    let len = items.length;
+    if (this.chunkSize === 0) {
+      this.chunkSize = Math.floor(items.length / 100 * 5);
+    }
+    while (i < len) {
+      const chunk = items.slice(i, i + this.chunkSize);
+      const count = chunk.length;
+      const start = performance.now();
+      await processFn(chunk);
+      const elapsed = performance.now() - start;
+      this.updateEstimate(elapsed, count);
+      this.chunkSize = this.nextChunkSize();
+      i += count;
+      await yieldEventLoop();
+    }
+  }
+};
+
 // src/core/IndexManager.ts
 var IndexManager = class {
   constructor(api, logger) {
@@ -11878,7 +11959,7 @@ var IndexManager = class {
       this.indices = metadata.indices;
       this.registeredIndices.delete(name);
       const fields = this.getFieldsFromConfig(config);
-      for (let i = 0; i < fields.length; i++) {
+      for (let i = 0, len = fields.length; i < len; i++) {
         const field = fields[i];
         const indexNames = this.fieldToIndices.get(field);
         if (indexNames) {
@@ -11972,6 +12053,7 @@ var IndexManager = class {
           await treeTx.rollback();
           throw err;
         }
+        await yieldEventLoop();
       }
       this.pendingBackfillFields = [];
       return backfilledCount;
@@ -12043,7 +12125,9 @@ var IndexManager = class {
             indexName
           ),
           this.api.comparator,
-          { capacity: perIndexCapacity }
+          {
+            capacity: perIndexCapacity
+          }
         );
         await tree.init();
         this.trees.set(indexName, tree);
@@ -12061,6 +12145,7 @@ var IndexManager = class {
             await treeTx.rollback();
             throw err;
           }
+          await yieldEventLoop();
         }
       }
       return docCount;
@@ -12171,77 +12256,15 @@ async function catchPromise(promise) {
   return promise.then((res) => [void 0, res]).catch((reason) => [reason]);
 }
 
-// src/utils/DeadlineChunker.ts
-var DeadlineChunker = class {
-  /**
-   * 이벤트 루프를 막을 최대 허용 시간
-   */
-  targetMs;
-  /**
-   * 현재 chunk size
-   */
-  chunkSize;
-  /**
-   * Exponential Weighted Moving Average
-   */
-  ewmaMs;
-  /**
-   * EWMA 평활화 계수
-   */
-  alpha;
-  constructor(startChunkSize = 0, targetMs = 5, alpha = 0.5) {
-    this.chunkSize = startChunkSize;
-    this.targetMs = targetMs;
-    this.alpha = alpha;
-    this.ewmaMs = null;
-  }
-  /**
-   * EWMA 평활화 계수를 사용하여 평균 처리 시간을 업데이트합니다.
-   */
-  updateEstimate(elapsed, count) {
-    const msPerItem = elapsed / count;
-    this.ewmaMs = this.ewmaMs === null ? msPerItem : this.alpha * msPerItem + (1 - this.alpha) * this.ewmaMs;
-  }
-  /**
-   * 현재 chunk size를 업데이트합니다.
-   */
-  nextChunkSize() {
-    if (!this.ewmaMs || this.ewmaMs === 0) return this.chunkSize;
-    const next = Math.floor(this.targetMs / this.ewmaMs);
-    return Math.max(1, next);
-  }
-  /**
-   * 주어진 items를 chunk로 분할하여 처리합니다.
-   */
-  async processInChunks(items, processFn) {
-    let i = 0;
-    let len = items.length;
-    if (this.chunkSize === 0) {
-      this.chunkSize = Math.floor(items.length / 100 * 5);
-    }
-    while (i < len) {
-      const chunk = items.slice(i, i + this.chunkSize);
-      const count = chunk.length;
-      const start = performance.now();
-      await processFn(chunk);
-      const elapsed = performance.now() - start;
-      this.updateEstimate(elapsed, count);
-      this.chunkSize = this.nextChunkSize();
-      i += count;
-      await new Promise(setImmediate);
-    }
-  }
-};
-
 // src/core/MutationManager.ts
 var MutationManager = class {
   constructor(api, logger) {
     this.api = api;
     this.logger = logger;
   }
-  async isTreeEmpty(tree) {
+  async isTreeEmpty(tree, tx) {
     try {
-      const root = await tree.getNode(tree.rootId);
+      const root = await tree.getNode(tree.rootId, tx);
       return root.leaf && root.values.length === 0;
     } catch {
       return true;
@@ -12288,16 +12311,19 @@ var MutationManager = class {
         }
       }
       await this.api.analysisManager.notifyInsert([flattenDocument], tx2);
+      this.logger.debug(`Inserted single document with ID: ${dataplyDocument._id}`);
       return dataplyDocument._id;
     }, tx);
   }
   async insertBatchDocuments(documents, tx) {
     return this.api.runWithDefaultWrite(async (tx2) => {
+      this.logger.debug(`Batch inserting ${documents.length} documents`);
       const metadata = await this.api.getDocumentInnerMetadata(tx2);
-      const startId = metadata.lastId + 1;
+      let startId = metadata.lastId + 1;
       metadata.lastId += documents.length;
       await this.api.updateDocumentInnerMetadata(metadata, tx2);
-      const ids = [];
+      const ids = new Float64Array(documents.length);
+      const pks = new Float64Array(documents.length);
       const dataplyDocuments = [];
       const flattenedData = [];
       for (let i = 0, len = documents.length; i < len; i++) {
@@ -12309,17 +12335,15 @@ var MutationManager = class {
         dataplyDocuments.push(stringified);
         const flattenDocument = this.api.flattenDocument(dataplyDocument);
         flattenedData.push({ pk: -1, data: flattenDocument });
-        ids.push(id);
+        ids[i] = id;
       }
-      const pks = [];
-      const documentChunker = new DeadlineChunker(1e4);
-      await documentChunker.processInChunks(dataplyDocuments, async (chunk) => {
-        const res = await this.api.insertBatch(chunk, true, tx2);
-        pks.push(...res);
-      });
-      for (let i = 0, len = pks.length; i < len; i++) {
-        flattenedData[i].pk = pks[i];
+      const res = await this.api.insertBatch(dataplyDocuments, true, tx2);
+      for (let i = 0, len = res.length; i < len; i++) {
+        const index = i;
+        pks[index] = res[i];
+        flattenedData[index].pk = res[i];
       }
+      await yieldEventLoop();
       for (const [indexName, config] of this.api.indexManager.registeredIndices) {
         const tree = this.api.trees.get(indexName);
         if (!tree) continue;
@@ -12346,42 +12370,39 @@ var MutationManager = class {
             batchInsertData.push([item.pk, { k: item.pk, v: indexVal }]);
           }
         }
-        const isEmptyTree = await this.isTreeEmpty(tree);
+        const isEmptyTree = await this.isTreeEmpty(tree, tx2);
         if (isEmptyTree) {
           const [error] = await catchPromise(treeTx.bulkLoad(batchInsertData));
           if (error) {
             throw error;
           }
         } else {
-          const initMaxSize = 5e4;
-          const initChunkSize = Math.min(
-            initMaxSize,
-            Math.max(initMaxSize, Math.floor(batchInsertData.length / 100 * 5))
-          );
-          const chunker = new DeadlineChunker(initChunkSize);
-          await chunker.processInChunks(batchInsertData, async (chunk) => {
-            const [error] = await catchPromise(treeTx.batchInsert(chunk));
-            if (error) {
-              throw error;
-            }
-          });
+          const [error] = await catchPromise(treeTx.batchInsert(batchInsertData));
+          if (error) {
+            throw error;
+          }
         }
-        const res = await treeTx.commit();
-        if (!res.success) {
+        const res2 = await treeTx.commit();
+        if (!res2.success) {
           await treeTx.rollback();
-          throw res.error;
+          this.logger.error(`Failed to commit batch insert for index ${indexName}: ${res2.error}`);
+          throw res2.error;
         }
+        await yieldEventLoop();
       }
       const flatDocs = [];
       for (let i = 0, len = flattenedData.length; i < len; i++) {
         flatDocs.push(flattenedData[i].data);
       }
       await this.api.analysisManager.notifyInsert(flatDocs, tx2);
-      return ids;
+      await yieldEventLoop();
+      this.logger.debug(`Successfully batch inserted ${documents.length} documents`);
+      return Array.from(ids);
     }, tx);
   }
   async updateInternal(query, computeUpdatedDoc, tx) {
     const pks = await this.api.queryManager.getKeys(query);
+    this.logger.debug(`Found ${pks.length} documents to update`);
     let updatedCount = 0;
     const updatePairs = [];
     const treeTxs = /* @__PURE__ */ new Map();
@@ -12430,9 +12451,11 @@ var MutationManager = class {
             await treeTx.batchInsert([[pk, { k: pk, v: newIndexVal }]]);
           }
         }
+        await yieldEventLoop();
       }
       updatePairs.push({ oldDocument: oldFlatDoc, newDocument: newFlatDoc });
       await this.api.update(pk, JSON.stringify(updatedDoc), tx);
+      await yieldEventLoop();
       updatedCount++;
     }
     for (const [indexName, treeTx] of treeTxs) {
@@ -12441,10 +12464,14 @@ var MutationManager = class {
         for (const rollbackTx of treeTxs.values()) {
           rollbackTx.rollback();
         }
+        await yieldEventLoop();
+        this.logger.error(`Failed to commit update for index ${indexName}: ${result.error}`);
         throw result.error;
       }
     }
     await this.api.analysisManager.notifyUpdate(updatePairs, tx);
+    await yieldEventLoop();
+    this.logger.debug(`Successfully updated ${updatedCount} documents`);
     return updatedCount;
   }
   async fullUpdate(query, newRecord, tx) {
@@ -12468,6 +12495,7 @@ var MutationManager = class {
   async deleteDocuments(query, tx) {
     return this.api.runWithDefaultWrite(async (tx2) => {
       const pks = await this.api.queryManager.getKeys(query);
+      this.logger.debug(`Found ${pks.length} documents to delete`);
       let deletedCount = 0;
       const deletedFlatDocs = [];
       for (let i = 0, len = pks.length; i < len; i++) {
@@ -12492,12 +12520,16 @@ var MutationManager = class {
             if (indexVal === void 0) continue;
             await tree.delete(pk, { k: pk, v: indexVal });
           }
+          await yieldEventLoop();
         }
         deletedFlatDocs.push(flatDoc);
         await this.api.delete(pk, true, tx2);
+        await yieldEventLoop();
         deletedCount++;
       }
       await this.api.analysisManager.notifyDelete(deletedFlatDocs, tx2);
+      await yieldEventLoop();
+      this.logger.debug(`Successfully deleted ${deletedCount} documents`);
       return deletedCount;
     }, tx);
   }
@@ -13701,6 +13733,22 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
   get indexedFields() {
     return this.indexManager.indexedFields;
   }
+  /**
+   * Method for reliably processing large batches (chunks) of data.
+   * Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
+   * @param items The items to process
+   * @param callback The callback to process each chunk
+   * @param options The options for the chunk splitter
+   * @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
+   * @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
+   * @returns The processed items
+   */
+  processInChunks(items, callback, options) {
+    this.logger.debug(`Processing ${items.length} items in chunks`);
+    const firstChunkSize = options?.firstChunkSize ?? 0;
+    const alpha = options?.alpha ?? 0.5;
+    return new DeadlineChunker(firstChunkSize, alpha).processInChunks(items, callback);
+  }
   /**
    * Register an index.
    * @param name The name of the index
@@ -13950,6 +13998,19 @@ var DocumentDataply = class _DocumentDataply {
   constructor(file, options) {
     this.api = new DocumentDataplyAPI(file, options ?? {});
   }
+  /**
+   * Method for reliably processing large batches (chunks) of data.
+   * Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
+   * @param items The items to process
+   * @param callback The callback to process each chunk
+   * @param options The options for the chunk splitter
+   * @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
+   * @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
+   * @returns The processed items
+   */
+  processInChunks(items, callback, options) {
+    return this.api.processInChunks(items, callback, options);
+  }
   /**
    * Create a named index on the database.
    * Can be called before or after init().

package/dist/types/core/AnalysisManager.d.ts

@@ -1,7 +1,7 @@
 import type { AnalysisHeader, DocumentJSON, FlattenedDocumentJSON } from '../types';
 import type { DocumentDataplyAPI } from './documentAPI';
 import type { AnalysisProvider } from './AnalysisProvider';
-import { Transaction } from 'dataply';
+import { Transaction, Logger } from 'dataply';
 export declare class AnalysisManager<T extends DocumentJSON> {
     private api;
     readonly sampleSize: number;
@@ -9,7 +9,7 @@ export declare class AnalysisManager<T extends DocumentJSON> {
     private providers;
     private cron;
     private flushing;
-    constructor(api: DocumentDataplyAPI<T>, schedule: string, sampleSize: number, logger: any);
+    constructor(api: DocumentDataplyAPI<T>, schedule: string, sampleSize: number, logger: Logger);
     /**
      * Stop the background analysis cron job.
      */

package/dist/types/core/IndexManager.d.ts

@@ -1,6 +1,6 @@
 import type { DataplyTreeValue, DocumentDataplyInnerMetadata, Primitive, CreateIndexOption, IndexMetaConfig, FTSConfig, DocumentJSON, FlattenedDocumentJSON } from '../types';
 import type { DocumentDataplyAPI } from './documentAPI';
-import { BPTreeAsync, Transaction } from 'dataply';
+import { BPTreeAsync, Transaction, Logger } from 'dataply';
 export declare class IndexManager<T extends DocumentJSON> {
     private api;
     private logger;
@@ -23,7 +23,7 @@ export declare class IndexManager<T extends DocumentJSON> {
      */
     fieldToIndices: Map<string, string[]>;
     pendingBackfillFields: string[];
-    constructor(api: DocumentDataplyAPI<T>, logger: any);
+    constructor(api: DocumentDataplyAPI<T>, logger: Logger);
     /**
      * Validate and apply indices from DB metadata and pending indices.
      * Called during database initialization.

package/dist/types/core/MetadataManager.d.ts

@@ -1,10 +1,10 @@
 import type { DocumentDataplyMetadata, DocumentDataplyInnerMetadata, DocumentJSON } from '../types';
 import type { DocumentDataplyAPI } from './documentAPI';
-import { Transaction } from 'dataply';
+import { Transaction, Logger } from 'dataply';
 export declare class MetadataManager<T extends DocumentJSON> {
     private api;
     private logger;
-    constructor(api: DocumentDataplyAPI<T>, logger: any);
+    constructor(api: DocumentDataplyAPI<T>, logger: Logger);
     getDocumentMetadata(tx: Transaction): Promise<DocumentDataplyMetadata>;
     getDocumentInnerMetadata(tx: Transaction): Promise<DocumentDataplyInnerMetadata>;
     updateDocumentInnerMetadata(metadata: DocumentDataplyInnerMetadata, tx: Transaction): Promise<void>;

package/dist/types/core/MutationManager.d.ts

@@ -1,10 +1,10 @@
 import type { DocumentJSON, DataplyDocument, DocumentDataplyQuery } from '../types';
 import type { DocumentDataplyAPI } from './documentAPI';
-import { Transaction } from 'dataply';
+import { Transaction, Logger } from 'dataply';
 export declare class MutationManager<T extends DocumentJSON> {
     private api;
     private logger;
-    constructor(api: DocumentDataplyAPI<T>, logger: any);
+    constructor(api: DocumentDataplyAPI<T>, logger: Logger);
     private isTreeEmpty;
     private insertDocumentInternal;
     insertSingleDocument(document: T, tx?: Transaction): Promise<number>;

package/dist/types/core/QueryManager.d.ts

@@ -1,13 +1,13 @@
 import type { DataplyTreeValue, DocumentDataplyQuery, DocumentDataplyQueryOptions, DataplyDocument, Primitive, DocumentJSON } from '../types';
 import type { DocumentDataplyAPI } from './documentAPI';
 import type { Optimizer } from './Optimizer';
-import { BPTreeAsync } from 'dataply';
+import { BPTreeAsync, Logger } from 'dataply';
 export declare class QueryManager<T extends DocumentJSON> {
     private api;
     private optimizer;
     private logger;
     private readonly operatorConverters;
-    constructor(api: DocumentDataplyAPI<T>, optimizer: Optimizer<T>, logger: any);
+    constructor(api: DocumentDataplyAPI<T>, optimizer: Optimizer<T>, logger: Logger);
     /**
      * Transforms a query object into a verbose query object
      */

package/dist/types/core/document.d.ts

@@ -30,6 +30,20 @@ export declare class DocumentDataply<T extends DocumentJSON> {
     private static Open;
     protected readonly api: DocumentDataplyAPI<T>;
     protected constructor(file: string, options?: DocumentDataplyOptions);
+    /**
+     * Method for reliably processing large batches (chunks) of data.
+     * Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
+     * @param items The items to process
+     * @param callback The callback to process each chunk
+     * @param options The options for the chunk splitter
+     * @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
+     * @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
+     * @returns The processed items
+     */
+    processInChunks<T>(items: T[], callback: (chunk: T[]) => Promise<void>, options?: {
+        firstChunkSize?: number;
+        alpha?: number;
+    }): Promise<void>;
     /**
      * Create a named index on the database.
      * Can be called before or after init().

package/dist/types/core/documentAPI.d.ts

@@ -31,6 +31,20 @@ export declare class DocumentDataplyAPI<T extends DocumentJSON> extends DataplyA
     };
     get trees(): Map<string, import("dataply").BPTreeAsync<string | number, import("../types").DataplyTreeValue<import("../types").Primitive>>>;
     get indexedFields(): Set<string>;
+    /**
+     * Method for reliably processing large batches (chunks) of data.
+     * Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
+     * @param items The items to process
+     * @param callback The callback to process each chunk
+     * @param options The options for the chunk splitter
+     * @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
+     * @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
+     * @returns The processed items
+     */
+    processInChunks<T>(items: T[], callback: (chunk: T[]) => Promise<void>, options?: {
+        firstChunkSize?: number;
+        alpha?: number;
+    }): Promise<void>;
     /**
      * Register an index.
      * @param name The name of the index

package/dist/types/utils/{DeadlineChunker.d.ts → eventLoopManager.d.ts}

@@ -1,3 +1,4 @@
+export declare function yieldEventLoop(): Promise<unknown>;
 /**
  * 이벤트 루프를 막지 않고 대량의 데이터를 처리하기 위한 유틸리티 클래스
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-dataply",
-  "version": "0.0.14-alpha.1",
+  "version": "0.0.14-alpha.2",
   "description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -43,7 +43,7 @@
   ],
   "dependencies": {
     "croner": "^10.0.1",
-    "dataply": "^0.0.26-alpha.2"
+    "dataply": "^0.0.26-alpha.4"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",