dataply 0.0.1 → 0.0.2

package/dist/cjs/index.js

@@ -3776,11 +3776,16 @@ var MetadataPageManager = class _MetadataPageManager extends PageManager {
     OFFSET_ROOT_INDEX_ORDER: 126,
     OFFSET_LAST_INSERT_PAGE_ID: 130,
     OFFSET_LAST_ROW_PK: 134,
+    OFFSET_BITMAP_PAGE_ID: 140,
+    OFFSET_FREE_PAGE_ID: 144,
     SIZE_PAGE_COUNT: 4,
     SIZE_PAGE_SIZE: 4,
     SIZE_ROOT_INDEX_PAGE_ID: 4,
     SIZE_ROOT_INDEX_ORDER: 4,
-    SIZE_LAST_INSERT_PAGE_ID: 4
+    SIZE_LAST_INSERT_PAGE_ID: 4,
+    SIZE_ROW_PK: 6,
+    SIZE_BITMAP_PAGE_ID: 4,
+    SIZE_FREE_PAGE_ID: 4
   };
   /**
    * Checks if the page type is `MetadataPage`.
@@ -3882,7 +3887,7 @@ var MetadataPageManager = class _MetadataPageManager extends PageManager {
     return bytesToNumber(
       page,
       _MetadataPageManager.CONSTANT.OFFSET_LAST_ROW_PK,
-      Row.CONSTANT.SIZE_PK
+      _MetadataPageManager.CONSTANT.SIZE_ROW_PK
     );
   }
   /**
@@ -3894,9 +3899,34 @@ var MetadataPageManager = class _MetadataPageManager extends PageManager {
     return bytesToNumber(
       page,
       _MetadataPageManager.CONSTANT.OFFSET_ROW_COUNT,
-      Row.CONSTANT.SIZE_PK
+      _MetadataPageManager.CONSTANT.SIZE_ROW_PK
+    );
+  }
+  /**
+   * Returns the ID of the bitmap page.
+   * @param page Page data
+   * @returns Bitmap page ID
+   */
+  getBitmapPageId(page) {
+    return bytesToNumber(
+      page,
+      _MetadataPageManager.CONSTANT.OFFSET_BITMAP_PAGE_ID,
+      _MetadataPageManager.CONSTANT.SIZE_BITMAP_PAGE_ID
     );
   }
+  /**
+   * Returns the ID of the free page.
+   * @param page Page data
+   * @returns Free page ID
+   */
+  getFreePageId(page) {
+    const id = bytesToNumber(
+      page,
+      _MetadataPageManager.CONSTANT.OFFSET_FREE_PAGE_ID,
+      _MetadataPageManager.CONSTANT.SIZE_FREE_PAGE_ID
+    );
+    return id === 4294967295 ? -1 : id;
+  }
   /**
    * Sets the number of pages stored in the database.
    * @param page Page data
@@ -3997,6 +4027,32 @@ var MetadataPageManager = class _MetadataPageManager extends PageManager {
       Row.CONSTANT.SIZE_PK
     );
   }
+  /**
+   * Sets the ID of the bitmap page.
+   * @param page Page data
+   * @param bitmapPageId Bitmap page ID
+   */
+  setBitmapPageId(page, bitmapPageId) {
+    numberToBytes(
+      bitmapPageId,
+      page,
+      _MetadataPageManager.CONSTANT.OFFSET_BITMAP_PAGE_ID,
+      _MetadataPageManager.CONSTANT.SIZE_BITMAP_PAGE_ID
+    );
+  }
+  /**
+   * Sets the ID of the free page.
+   * @param page Page data
+   * @param pageId Free page ID
+   */
+  setFreePageId(page, pageId) {
+    numberToBytes(
+      pageId,
+      page,
+      _MetadataPageManager.CONSTANT.OFFSET_FREE_PAGE_ID,
+      _MetadataPageManager.CONSTANT.SIZE_FREE_PAGE_ID
+    );
+  }
 };
 var BitmapPageManager = class _BitmapPageManager extends PageManager {
   get pageType() {
@@ -4027,6 +4083,31 @@ var BitmapPageManager = class _BitmapPageManager extends PageManager {
   isEmptyPage(page, index) {
     return bytesToNumber(page, index, 1) === 0;
   }
+  /**
+   * Gets a bit from the bitmap page.
+   * @param page Page data
+   * @param index Bit index
+   * @returns boolean indicating if the bit is set
+   */
+  getBit(page, index) {
+    const bitOffset = Math.floor(index / 8);
+    const offset = _BitmapPageManager.CONSTANT.SIZE_PAGE_HEADER + bitOffset;
+    const value = bytesToNumber(page, offset, 1);
+    return getBit(value, index % 8);
+  }
+  /**
+   * Sets a bit in the bitmap page.
+   * @param page Page data
+   * @param index Bit index
+   * @param flag boolean indicating if the bit is set
+   */
+  setBit(page, index, flag) {
+    const bitOffset = Math.floor(index / 8);
+    const offset = _BitmapPageManager.CONSTANT.SIZE_PAGE_HEADER + bitOffset;
+    const value = bytesToNumber(page, offset, 1);
+    const newValue = setBit(value, index % 8, flag);
+    numberToBytes(newValue, page, offset, 1);
+  }
 };
 var OverflowPageManager = class _OverflowPageManager extends PageManager {
   get pageType() {
@@ -4448,8 +4529,8 @@ var VirtualFileSystem = class {
    * @param position Start position in file
    */
   _writeAsync(handle, buffer, offset, length, position) {
-    if (position + length > 100 * 1024 * 1024) {
-      return Promise.reject(new Error(`[Safety Limit] File write exceeds 100MB limit at position ${position}`));
+    if (position + length > 512 * 1024 * 1024) {
+      return Promise.reject(new Error(`[Safety Limit] File write exceeds 512MB limit at position ${position}`));
     }
     return new Promise((resolve, reject) => {
       import_node_fs2.default.write(handle, buffer, offset, length, position, (err, bytesWritten) => {
@@ -4627,6 +4708,43 @@ var PageFileSystem = class {
   pageFactory = new PageManagerFactory();
   vfs;
   pageManagerFactory;
+  /**
+   * Updates the bitmap status for a specific page.
+   * @param pageId The ID of the page to update
+   * @param isFree True to mark as free, false to mark as used
+   * @param tx Transaction
+   */
+  async updateBitmap(pageId, isFree, tx) {
+    const metadata = await this.getMetadata(tx);
+    const metadataManager = this.pageFactory.getManager(metadata);
+    const bitmapPageId = metadataManager.getBitmapPageId(metadata);
+    const headerSize = PageManager.CONSTANT.SIZE_PAGE_HEADER;
+    const capacityPerBitmapPage = (this.pageSize - headerSize) * 8;
+    let currentBitmapPageId = bitmapPageId;
+    let targetBitIndex = pageId;
+    while (targetBitIndex >= capacityPerBitmapPage) {
+      const currentBitmapPage = await this.get(currentBitmapPageId, tx);
+      const manager = this.pageFactory.getManager(currentBitmapPage);
+      targetBitIndex -= capacityPerBitmapPage;
+      const nextPageId = manager.getNextPageId(currentBitmapPage);
+      if (nextPageId === -1) {
+        if (!isFree) {
+          throw new Error("Bitmap page not found for reused page");
+        }
+        const newBitmapPageId = await this.appendNewPage(PageManager.CONSTANT.PAGE_TYPE_BITMAP, tx);
+        manager.setNextPageId(currentBitmapPage, newBitmapPageId);
+        await this.setPage(currentBitmapPageId, currentBitmapPage, tx);
+        currentBitmapPageId = newBitmapPageId;
+      } else {
+        currentBitmapPageId = nextPageId;
+      }
+    }
+    await tx.__acquireWriteLock(currentBitmapPageId);
+    const targetBitmapPage = await this.get(currentBitmapPageId, tx);
+    const bitmapManager = this.pageFactory.getManager(targetBitmapPage);
+    bitmapManager.setBit(targetBitmapPage, targetBitIndex, isFree);
+    await this.setPage(currentBitmapPageId, targetBitmapPage, tx);
+  }
   /**
    * VFS 인스턴스를 반환합니다.
    * Transaction 생성 시 사용됩니다.
@@ -4735,12 +4853,28 @@ var PageFileSystem = class {
   }
   /**
    * Appends and inserts a new page.
-   * @returns Created page ID
+   * If a free page is available in the free list, it reuses it.
+   * Otherwise, it appends a new page to the end of the file.
+   * @returns Created or reused page ID
    */
   async appendNewPage(pageType = PageManager.CONSTANT.PAGE_TYPE_EMPTY, tx) {
     await tx.__acquireWriteLock(0);
     const metadata = await this.getMetadata(tx);
     const metadataManager = this.pageFactory.getManager(metadata);
+    const freePageId = metadataManager.getFreePageId(metadata);
+    if (freePageId !== -1) {
+      const reusedPageId = freePageId;
+      const reusedPage = await this.get(reusedPageId, tx);
+      const reusedPageManager = this.pageFactory.getManager(reusedPage);
+      const nextFreePageId = reusedPageManager.getNextPageId(reusedPage);
+      metadataManager.setFreePageId(metadata, nextFreePageId);
+      await this.setPage(0, metadata, tx);
+      await this.updateBitmap(reusedPageId, false, tx);
+      const manager2 = this.pageFactory.getManagerFromType(pageType);
+      const newPage2 = manager2.create(this.pageSize, reusedPageId);
+      await this.setPage(reusedPageId, newPage2, tx);
+      return reusedPageId;
+    }
     const pageCount = metadataManager.getPageCount(metadata);
     const newPageIndex = pageCount;
     const newTotalCount = pageCount + 1;
@@ -4805,6 +4939,26 @@ var PageFileSystem = class {
       }
     }
   }
+  /**
+   * Frees the page and marks it as available in the bitmap.
+   * It also adds the page to the linked list of free pages in metadata.
+   * @param pageId Page ID
+   * @param tx Transaction
+   */
+  async setFreePage(pageId, tx) {
+    await tx.__acquireWriteLock(0);
+    await tx.__acquireWriteLock(pageId);
+    const metadata = await this.getMetadata(tx);
+    const metadataManager = this.pageFactory.getManager(metadata);
+    const currentHeadFreePageId = metadataManager.getFreePageId(metadata);
+    const emptyPageManager = this.pageFactory.getManagerFromType(PageManager.CONSTANT.PAGE_TYPE_EMPTY);
+    const emptyPage = emptyPageManager.create(this.pageSize, pageId);
+    emptyPageManager.setNextPageId(emptyPage, currentHeadFreePageId);
+    await this.setPage(pageId, emptyPage, tx);
+    await this.updateBitmap(pageId, true, tx);
+    metadataManager.setFreePageId(metadata, pageId);
+    await this.setPage(0, metadata, tx);
+  }
   /**
    * Closes the page file system.
    */
@@ -4944,8 +5098,7 @@ var RowIdentifierStrategy = class extends SerializeStrategyAsync {
     while (true) {
       const page = await this.pfs.get(pageId, tx);
       const nextPageId = manager.getNextPageId(page);
-      manager.setNextPageId(page, -1);
-      manager.setRemainingCapacity(page, this.pfs.pageSize - PageManager.CONSTANT.SIZE_PAGE_HEADER);
+      await this.pfs.setFreePage(pageId, tx);
       if (nextPageId === -1) {
         break;
       }
@@ -5337,13 +5490,41 @@ var RowTableEngine = class {
     if (this.rowManager.getDeletedFlag(row)) {
       return;
     }
+    if (this.rowManager.getOverflowFlag(row)) {
+      let overflowPageId = bytesToNumber(this.rowManager.getBody(row));
+      while (overflowPageId !== -1) {
+        const overflowPage = await this.pfs.get(overflowPageId, tx);
+        const manager = this.factory.getManager(overflowPage);
+        const nextPageId = manager.getNextPageId(overflowPage);
+        await this.pfs.setFreePage(overflowPageId, tx);
+        overflowPageId = nextPageId;
+      }
+    }
     this.rowManager.setDeletedFlag(row, true);
     await this.pfs.setPage(pageId, page, tx);
     if (decrementRowCount) {
-      const metadataPage = await this.pfs.getMetadata(tx);
-      const currentRowCount = this.metadataPageManager.getRowCount(metadataPage);
-      this.metadataPageManager.setRowCount(metadataPage, currentRowCount - 1);
-      await this.pfs.setMetadata(metadataPage, tx);
+      const metadataPage2 = await this.pfs.getMetadata(tx);
+      const currentRowCount = this.metadataPageManager.getRowCount(metadataPage2);
+      this.metadataPageManager.setRowCount(metadataPage2, currentRowCount - 1);
+      await this.pfs.setMetadata(metadataPage2, tx);
+    }
+    const insertedRowCount = this.dataPageManager.getInsertedRowCount(page);
+    let allDeleted = true;
+    const metadataPage = await this.pfs.getMetadata(tx);
+    const lastInsertPageId = this.metadataPageManager.getLastInsertPageId(metadataPage);
+    if (pageId === lastInsertPageId) {
+      allDeleted = false;
+    } else {
+      for (let i = 0; i < insertedRowCount; i++) {
+        const slotRow = this.dataPageManager.getRow(page, i);
+        if (!this.rowManager.getDeletedFlag(slotRow)) {
+          allDeleted = false;
+          break;
+        }
+      }
+    }
+    if (allDeleted) {
+      await this.pfs.setFreePage(pageId, tx);
     }
   }
   /**
@@ -5648,6 +5829,7 @@ var DataplyAPI = class {
    */
   static InitializeFile(fileHandle, options) {
     const metadataPageManager = new MetadataPageManager();
+    const bitmapPageManager = new BitmapPageManager();
     const dataPageManager = new DataPageManager();
     const metadataPage = new Uint8Array(options.pageSize);
     const dataPage = new Uint8Array(options.pageSize);
@@ -5659,19 +5841,32 @@ var DataplyAPI = class {
       options.pageSize - MetadataPageManager.CONSTANT.SIZE_PAGE_HEADER
     );
     metadataPageManager.setMagicString(metadataPage);
-    metadataPageManager.setPageCount(metadataPage, 2);
     metadataPageManager.setPageSize(metadataPage, options.pageSize);
     metadataPageManager.setRootIndexPageId(metadataPage, -1);
-    metadataPageManager.setLastInsertPageId(metadataPage, 1);
+    metadataPageManager.setBitmapPageId(metadataPage, 1);
+    metadataPageManager.setLastInsertPageId(metadataPage, 2);
+    metadataPageManager.setPageCount(metadataPage, 3);
+    metadataPageManager.setFreePageId(metadataPage, -1);
+    const bitmapPage = new Uint8Array(options.pageSize);
+    bitmapPageManager.initial(
+      bitmapPage,
+      BitmapPageManager.CONSTANT.PAGE_TYPE_BITMAP,
+      1,
+      -1,
+      options.pageSize - BitmapPageManager.CONSTANT.SIZE_PAGE_HEADER
+    );
     dataPageManager.initial(
       dataPage,
       DataPageManager.CONSTANT.PAGE_TYPE_DATA,
-      1,
+      2,
       -1,
       options.pageSize - DataPageManager.CONSTANT.SIZE_PAGE_HEADER
     );
-    import_node_fs3.default.appendFileSync(fileHandle, metadataPage);
-    import_node_fs3.default.appendFileSync(fileHandle, dataPage);
+    import_node_fs3.default.appendFileSync(fileHandle, new Uint8Array([
+      ...metadataPage,
+      ...bitmapPage,
+      ...dataPage
+    ]));
   }
   /**
    * Opens the database file. If the file does not exist, it initializes it.

package/dist/types/core/Page.d.ts

@@ -421,11 +421,16 @@ export declare class MetadataPageManager extends PageManager {
         readonly OFFSET_ROOT_INDEX_ORDER: 126;
         readonly OFFSET_LAST_INSERT_PAGE_ID: 130;
         readonly OFFSET_LAST_ROW_PK: 134;
+        readonly OFFSET_BITMAP_PAGE_ID: 140;
+        readonly OFFSET_FREE_PAGE_ID: 144;
         readonly SIZE_PAGE_COUNT: 4;
         readonly SIZE_PAGE_SIZE: 4;
         readonly SIZE_ROOT_INDEX_PAGE_ID: 4;
         readonly SIZE_ROOT_INDEX_ORDER: 4;
         readonly SIZE_LAST_INSERT_PAGE_ID: 4;
+        readonly SIZE_ROW_PK: 6;
+        readonly SIZE_BITMAP_PAGE_ID: 4;
+        readonly SIZE_FREE_PAGE_ID: 4;
         readonly PAGE_TYPE_UNKNOWN: 0;
         readonly PAGE_TYPE_EMPTY: 1;
         readonly PAGE_TYPE_METADATA: 2;
@@ -508,6 +513,18 @@ export declare class MetadataPageManager extends PageManager {
      * @returns Number of rows
      */
     getRowCount(page: MetadataPage): number;
+    /**
+     * Returns the ID of the bitmap page.
+     * @param page Page data
+     * @returns Bitmap page ID
+     */
+    getBitmapPageId(page: MetadataPage): number;
+    /**
+     * Returns the ID of the free page.
+     * @param page Page data
+     * @returns Free page ID
+     */
+    getFreePageId(page: MetadataPage): number;
     /**
      * Sets the number of pages stored in the database.
      * @param page Page data
@@ -555,6 +572,18 @@ export declare class MetadataPageManager extends PageManager {
      * @param rowCount Number of rows
      */
     setRowCount(page: MetadataPage, rowCount: number): void;
+    /**
+     * Sets the ID of the bitmap page.
+     * @param page Page data
+     * @param bitmapPageId Bitmap page ID
+     */
+    setBitmapPageId(page: MetadataPage, bitmapPageId: number): void;
+    /**
+     * Sets the ID of the free page.
+     * @param page Page data
+     * @param pageId Free page ID
+     */
+    setFreePageId(page: MetadataPage, pageId: number): void;
 }
 /**
  * Represents a bitmap page.
@@ -581,6 +610,20 @@ export declare class BitmapPageManager extends PageManager {
      * @returns boolean indicating if the page is empty
      */
     isEmptyPage(page: BitmapPage, index: number): boolean;
+    /**
+     * Gets a bit from the bitmap page.
+     * @param page Page data
+     * @param index Bit index
+     * @returns boolean indicating if the bit is set
+     */
+    getBit(page: BitmapPage, index: number): boolean;
+    /**
+     * Sets a bit in the bitmap page.
+     * @param page Page data
+     * @param index Bit index
+     * @param flag boolean indicating if the bit is set
+     */
+    setBit(page: BitmapPage, index: number, flag: boolean): void;
 }
 /**
  * Represents an overflow page.

package/dist/types/core/PageFileSystem.d.ts

@@ -21,6 +21,13 @@ export declare class PageFileSystem {
      * @param walPath WAL 파일 경로 (기본값: null)
      */
     constructor(fileHandle: number, pageSize: number, pageCacheCapacity: number, walPath?: string | undefined | null);
+    /**
+     * Updates the bitmap status for a specific page.
+     * @param pageId The ID of the page to update
+     * @param isFree True to mark as free, false to mark as used
+     * @param tx Transaction
+     */
+    private updateBitmap;
     /**
      * VFS 인스턴스를 반환합니다.
      * Transaction 생성 시 사용됩니다.
@@ -80,7 +87,9 @@ export declare class PageFileSystem {
     setPage(pageIndex: number, page: Uint8Array, tx: Transaction): Promise<void>;
     /**
      * Appends and inserts a new page.
-     * @returns Created page ID
+     * If a free page is available in the free list, it reuses it.
+     * Otherwise, it appends a new page to the end of the file.
+     * @returns Created or reused page ID
      */
     appendNewPage(pageType: number | undefined, tx: Transaction): Promise<number>;
     /**
@@ -91,6 +100,13 @@ export declare class PageFileSystem {
      * @param tx Transaction
      */
     writePageContent(pageId: number, data: Uint8Array, offset: number | undefined, tx: Transaction): Promise<void>;
+    /**
+     * Frees the page and marks it as available in the bitmap.
+     * It also adds the page to the linked list of free pages in metadata.
+     * @param pageId Page ID
+     * @param tx Transaction
+     */
+    setFreePage(pageId: number, tx: Transaction): Promise<void>;
     /**
      * Closes the page file system.
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dataply",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "A lightweight storage engine for Node.js with support for MVCC, WAL.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -17,7 +17,7 @@
     "dist/**/*"
   ],
   "scripts": {
-    "test": "jest --runInBand --forceExit",
+    "test": "jest -i",
     "benchmark": "npx tsx benchmark/benchmark.ts",
     "build": "node build/index.js && tsc"
   },

package/readme.md

@@ -3,7 +3,7 @@
 # Dataply
 
 > [!WARNING]
-> **Dataply is currently in Alpha version.** It is experimental and not yet suitable for production use.
+> **Dataply is currently in Alpha version.** It is experimental and not yet suitable for production use. Internal data structures and file formats are subject to change at any time.
 
 **Dataply** is a lightweight, high-performance **Record Store** designed for Node.js. It focuses on storing arbitrary data and providing an auto-generated Primary Key (PK) for ultra-fast retrieval, while supporting core enterprise features like MVCC, WAL, and atomic transactions.
 
@@ -177,9 +177,17 @@ graph TD
 ```
 
 ### 2. Page-Based Storage and VFS Caching
+
 - **Fixed-size Pages**: All data is managed in fixed-size units (default 8KB) called pages.
 - **VFS Cache**: Minimizes disk I/O by caching frequently accessed pages in memory.
 - **Dirty Page Tracking**: Tracks modified pages (Dirty) to synchronize them with disk efficiently only at the time of commit.
+- **Detailed Structure**: For technical details on the physical layout, see [structure.md](structure.md).
+
+#### Page & Row Layout
+Dataply uses a **Slotted Page** architecture to manage records efficiently:
+- **Pages**: Consists of a 100-byte header (containing `type`, `id`, `checksum`, etc.) and a body where rows are stored. Slot offsets are stored at the end of the page to track row positions.
+- **Rows**: Each row has a 9-byte header (`flags`, `size`, `PK`) followed by the actual data. Large records automatically trigger **Overflow Pages** to handle data exceeding page capacity.
+- **Keys & Identifiers**: Uses a 6-byte **Primary Key (PK)** for logical mapping and a 6-byte **Record Identifier (RID)** (Slot + Page ID) for direct physical addressing.
 
 ### 3. MVCC and Snapshot Isolation
 - **Non-blocking Reads**: Read operations are not blocked by write operations.