dataply 0.0.1 → 0.0.3

package/dist/types/core/DataplyAPI.d.ts

@@ -1,50 +1,63 @@
 import type { DataplyOptions, DataplyMetadata } from '../types';
+import { type IHookall, type IHookallSync } from 'hookall';
 import { PageFileSystem } from './PageFileSystem';
 import { RowTableEngine } from './RowTableEngine';
 import { TextCodec } from '../utils/TextCodec';
 import { LockManager } from './transaction/LockManager';
 import { Transaction } from './transaction/Transaction';
+interface DataplyAPISyncHook {
+    create: (fileData: Uint8Array, file: string, fileHandle: number, options: Required<DataplyOptions>) => Uint8Array;
+}
+interface DataplyAPIAsyncHook {
+    init: () => Promise<void>;
+    close: () => Promise<void>;
+}
 /**
  * Class for managing Dataply files.
  */
 export declare class DataplyAPI {
-    protected file: string;
-    protected fileHandle: number;
+    protected readonly file: string;
     readonly options: Required<DataplyOptions>;
+    protected readonly fileHandle: number;
     protected readonly pfs: PageFileSystem;
     protected readonly rowTableEngine: RowTableEngine;
     protected readonly lockManager: LockManager;
     protected readonly textCodec: TextCodec;
+    protected readonly hook: {
+        sync: IHookallSync<DataplyAPISyncHook>;
+        async: IHookall<DataplyAPIAsyncHook>;
+    };
     protected initialized: boolean;
     private txIdCounter;
-    protected constructor(file: string, fileHandle: number, options: Required<DataplyOptions>);
+    constructor(file: string, options: DataplyOptions);
     /**
      * Verifies if the page file is a valid Dataply file.
      * The metadata page must be located at the beginning of the Dataply file.
      * @param fileHandle File handle
      * @returns Whether the page file is a valid Dataply file
      */
-    static VerifyFormat(fileHandle: number): boolean;
+    private verifyFormat;
     /**
      * Fills missing options with default values.
      * @param options Options
      * @returns Options filled without omissions
      */
-    static VerboseOptions(options?: DataplyOptions): Required<DataplyOptions>;
+    private verboseOptions;
     /**
      * Initializes the database file.
      * The first page is initialized as the metadata page.
      * The second page is initialized as the first data page.
+     * @param file Database file path
      * @param fileHandle File handle
      */
-    static InitializeFile(fileHandle: number, options: Required<DataplyOptions>): void;
+    private initializeFile;
     /**
      * Opens the database file. If the file does not exist, it initializes it.
      * @param file Database file path
      * @param options Options
-     * @returns Dataply instance
+     * @returns File handle
      */
-    static Use(file: string, options?: DataplyOptions): DataplyAPI;
+    private createOrOpen;
     /**
      * Initializes the dataply instance.
      * Must be called before using the dataply instance.
@@ -119,3 +132,4 @@ export declare class DataplyAPI {
      */
     close(): Promise<void>;
 }
+export {};

package/dist/types/core/LogManager.d.ts

@@ -25,9 +25,15 @@ export declare class LogManager {
      * @param pages Map of changed pages (Page ID -> Data)
      */
     append(pages: Map<number, Uint8Array>): Promise<void>;
+    /**
+     * Writes a commit marker to the log file.
+     * This indicates that the preceding logs are part of a committed transaction.
+     */
+    writeCommitMarker(): Promise<void>;
     /**
      * Reads the log file to recover the page map.
      * Runs synchronously as it is called by the VFS constructor.
+     * Only returns pages from committed transactions (ended with a commit marker).
      * @returns Recovered page map
      */
     readAllSync(): Map<number, Uint8Array>;

package/dist/types/core/Page.d.ts

@@ -421,11 +421,16 @@ export declare class MetadataPageManager extends PageManager {
         readonly OFFSET_ROOT_INDEX_ORDER: 126;
         readonly OFFSET_LAST_INSERT_PAGE_ID: 130;
         readonly OFFSET_LAST_ROW_PK: 134;
+        readonly OFFSET_BITMAP_PAGE_ID: 140;
+        readonly OFFSET_FREE_PAGE_ID: 144;
         readonly SIZE_PAGE_COUNT: 4;
         readonly SIZE_PAGE_SIZE: 4;
         readonly SIZE_ROOT_INDEX_PAGE_ID: 4;
         readonly SIZE_ROOT_INDEX_ORDER: 4;
         readonly SIZE_LAST_INSERT_PAGE_ID: 4;
+        readonly SIZE_ROW_PK: 6;
+        readonly SIZE_BITMAP_PAGE_ID: 4;
+        readonly SIZE_FREE_PAGE_ID: 4;
         readonly PAGE_TYPE_UNKNOWN: 0;
         readonly PAGE_TYPE_EMPTY: 1;
         readonly PAGE_TYPE_METADATA: 2;
@@ -508,6 +513,18 @@ export declare class MetadataPageManager extends PageManager {
      * @returns Number of rows
      */
     getRowCount(page: MetadataPage): number;
+    /**
+     * Returns the ID of the bitmap page.
+     * @param page Page data
+     * @returns Bitmap page ID
+     */
+    getBitmapPageId(page: MetadataPage): number;
+    /**
+     * Returns the ID of the free page.
+     * @param page Page data
+     * @returns Free page ID
+     */
+    getFreePageId(page: MetadataPage): number;
     /**
      * Sets the number of pages stored in the database.
      * @param page Page data
@@ -555,6 +572,18 @@ export declare class MetadataPageManager extends PageManager {
      * @param rowCount Number of rows
      */
     setRowCount(page: MetadataPage, rowCount: number): void;
+    /**
+     * Sets the ID of the bitmap page.
+     * @param page Page data
+     * @param bitmapPageId Bitmap page ID
+     */
+    setBitmapPageId(page: MetadataPage, bitmapPageId: number): void;
+    /**
+     * Sets the ID of the free page.
+     * @param page Page data
+     * @param pageId Free page ID
+     */
+    setFreePageId(page: MetadataPage, pageId: number): void;
 }
 /**
  * Represents a bitmap page.
@@ -581,6 +610,20 @@ export declare class BitmapPageManager extends PageManager {
      * @returns boolean indicating if the page is empty
      */
     isEmptyPage(page: BitmapPage, index: number): boolean;
+    /**
+     * Gets a bit from the bitmap page.
+     * @param page Page data
+     * @param index Bit index
+     * @returns boolean indicating if the bit is set
+     */
+    getBit(page: BitmapPage, index: number): boolean;
+    /**
+     * Sets a bit in the bitmap page.
+     * @param page Page data
+     * @param index Bit index
+     * @param flag boolean indicating if the bit is set
+     */
+    setBit(page: BitmapPage, index: number, flag: boolean): void;
 }
 /**
  * Represents an overflow page.

package/dist/types/core/PageFileSystem.d.ts

@@ -21,6 +21,13 @@ export declare class PageFileSystem {
      * @param walPath WAL 파일 경로 (기본값: null)
      */
     constructor(fileHandle: number, pageSize: number, pageCacheCapacity: number, walPath?: string | undefined | null);
+    /**
+     * Updates the bitmap status for a specific page.
+     * @param pageId The ID of the page to update
+     * @param isFree True to mark as free, false to mark as used
+     * @param tx Transaction
+     */
+    private updateBitmap;
     /**
      * VFS 인스턴스를 반환합니다.
      * Transaction 생성 시 사용됩니다.
@@ -80,7 +87,9 @@ export declare class PageFileSystem {
     setPage(pageIndex: number, page: Uint8Array, tx: Transaction): Promise<void>;
     /**
      * Appends and inserts a new page.
-     * @returns Created page ID
+     * If a free page is available in the free list, it reuses it.
+     * Otherwise, it appends a new page to the end of the file.
+     * @returns Created or reused page ID
      */
     appendNewPage(pageType: number | undefined, tx: Transaction): Promise<number>;
     /**
@@ -91,6 +100,13 @@ export declare class PageFileSystem {
      * @param tx Transaction
      */
     writePageContent(pageId: number, data: Uint8Array, offset: number | undefined, tx: Transaction): Promise<void>;
+    /**
+     * Frees the page and marks it as available in the bitmap.
+     * It also adds the page to the linked list of free pages in metadata.
+     * @param pageId Page ID
+     * @param tx Transaction
+     */
+    setFreePage(pageId: number, tx: Transaction): Promise<void>;
     /**
      * Closes the page file system.
      */

package/dist/types/core/VirtualFileSystem.d.ts

@@ -29,7 +29,20 @@ export declare class VirtualFileSystem {
      */
     private recover;
     /**
-     * Commits the transaction.
+     * Prepares the transaction for commit (Phase 1).
+     * Writes dirty pages to WAL but does not update the main database file.
+     * @param tx Transaction
+     */
+    prepareCommit(tx: Transaction): Promise<void>;
+    /**
+     * Finalizes the transaction commit (Phase 2).
+     * Writes commit marker to WAL and updates the main database file (Checkpoint).
+     * @param tx Transaction
+     */
+    finalizeCommit(tx: Transaction): Promise<void>;
+    /**
+     * Commits the transaction (Single Phase).
+     * Wrapper for prepare + finalize for backward compatibility.
      * @param tx Transaction
      */
     commit(tx: Transaction): Promise<void>;

package/dist/types/core/transaction/GlobalTransaction.d.ts

@@ -0,0 +1,25 @@
+import { Transaction } from './Transaction';
+/**
+ * Global Transaction Manager.
+ * Coordinates transactions across multiple instances (shards) using 2-Phase Commit (2PC).
+ */
+export declare class GlobalTransaction {
+    private transactions;
+    private isCommitted;
+    private isRolledBack;
+    /**
+     * Adds a transaction to the global transaction.
+     * @param tx Transaction to add
+     */
+    add(tx: Transaction): void;
+    /**
+     * Commits all transactions atomically.
+     * Phase 1: Prepare (Write WAL)
+     * Phase 2: Commit (Write Commit Marker & Checkpoint)
+     */
+    commit(): Promise<void>;
+    /**
+     * Rolls back all transactions.
+     */
+    rollback(): Promise<void>;
+}

package/dist/types/core/transaction/Transaction.d.ts

@@ -86,6 +86,11 @@ export declare class Transaction {
      * @param pageId Page ID
      */
     __acquireWriteLock(pageId: number): Promise<void>;
+    /**
+     * Prepares the transaction for commit (Phase 1 of 2PC).
+     * Writes dirty pages to WAL but does not update the database file yet.
+     */
+    prepare(): Promise<void>;
     /**
      * Commits the transaction.
      */

package/dist/types/index.d.ts

@@ -4,3 +4,4 @@ export * from 'cache-entanglement';
 export type { DataplyOptions } from './types';
 export { Dataply } from './core/Dataply';
 export { DataplyAPI } from './core/DataplyAPI';
+export { GlobalTransaction } from './core/transaction/GlobalTransaction';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dataply",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "A lightweight storage engine for Node.js with support for MVCC, WAL.",
   "license": "MIT",
   "author": "izure <admin@izure.org>",
@@ -17,7 +17,7 @@
     "dist/**/*"
   ],
   "scripts": {
-    "test": "jest --runInBand --forceExit",
+    "test": "jest -i",
     "benchmark": "npx tsx benchmark/benchmark.ts",
     "build": "node build/index.js && tsc"
   },
@@ -45,7 +45,8 @@
   },
   "dependencies": {
     "cache-entanglement": "^1.7.1",
+    "hookall": "^2.2.0",
     "ryoiki": "^1.2.0",
     "serializable-bptree": "^5.2.1"
   }
-}
+}

package/readme.md

@@ -3,7 +3,7 @@
 # Dataply
 
 > [!WARNING]
-> **Dataply is currently in Alpha version.** It is experimental and not yet suitable for production use.
+> **Dataply is currently in Alpha version.** It is experimental and not yet suitable for production use. Internal data structures and file formats are subject to change at any time.
 
 **Dataply** is a lightweight, high-performance **Record Store** designed for Node.js. It focuses on storing arbitrary data and providing an auto-generated Primary Key (PK) for ultra-fast retrieval, while supporting core enterprise features like MVCC, WAL, and atomic transactions.
 
@@ -82,6 +82,36 @@ try {
 }
 ```
 
+### Global Transactions
+You can perform atomic operations across multiple `Dataply` instances using the `GlobalTransaction` class. This uses a **2-Phase Commit (2PC)** mechanism to ensure that either all instances commit successfully or all are rolled back.
+
+```typescript
+import { Dataply, GlobalTransaction } from 'dataply'
+
+const db1 = new Dataply('./db1.db', { wal: './db1.wal' })
+const db2 = new Dataply('./db2.db', { wal: './db2.wal' })
+
+await db1.init()
+await db2.init()
+
+const tx1 = db1.createTransaction()
+const tx2 = db2.createTransaction()
+
+const globalTx = new GlobalTransaction()
+globalTx.add(tx1)
+globalTx.add(tx2)
+
+try {
+  await db1.insert('Data for DB1', tx1)
+  await db2.insert('Data for DB2', tx2)
+  
+  // Phase 1: Prepare (WAL write) -> Phase 2: Commit (Marker write)
+  await globalTx.commit() 
+} catch (error) {
+  await globalTx.rollback()
+}
+```
+
 ### Auto-Transaction
 If you omit the `tx` argument when calling methods like `insert`, `update`, or `delete`, Dataply internally **creates an individual transaction automatically**.
 
@@ -133,6 +163,17 @@ Permanently reflects all changes made during the transaction to disk and release
 #### `async rollback(): Promise<void>`
 Cancels all changes made during the transaction and restores the original state.
 
+### GlobalTransaction Class
+
+#### `add(tx: Transaction): void`
+Adds an individual transaction from a `Dataply` instance to the global transaction.
+
+#### `async commit(): Promise<void>`
+Atomically commits all added transactions using a 2-Phase Commit (2PC) process.
+
+#### `async rollback(): Promise<void>`
+Rolls back all added transactions.
+
 ## Extending Dataply
 
 If you want to extend Dataply's functionality, use the `DataplyAPI` class. Unlike the standard `Dataply` class, `DataplyAPI` provides direct access to internal components like `PageFileSystem` or `RowTableEngine`, offering much more flexibility for custom implementations.
@@ -152,7 +193,7 @@ class CustomDataply extends DataplyAPI {
   }
 }
 
-const custom = CustomDataply.Use('./data.db')
+const custom = new CustomDataply('./data.db')
 await custom.init()
 
 const stats = await custom.getInternalStats()
@@ -177,9 +218,17 @@ graph TD
 ```
 
 ### 2. Page-Based Storage and VFS Caching
+
 - **Fixed-size Pages**: All data is managed in fixed-size units (default 8KB) called pages.
 - **VFS Cache**: Minimizes disk I/O by caching frequently accessed pages in memory.
 - **Dirty Page Tracking**: Tracks modified pages (Dirty) to synchronize them with disk efficiently only at the time of commit.
+- **Detailed Structure**: For technical details on the physical layout, see [structure.md](structure.md).
+
+#### Page & Row Layout
+Dataply uses a **Slotted Page** architecture to manage records efficiently:
+- **Pages**: Consists of a 100-byte header (containing `type`, `id`, `checksum`, etc.) and a body where rows are stored. Slot offsets are stored at the end of the page to track row positions.
+- **Rows**: Each row has a 9-byte header (`flags`, `size`, `PK`) followed by the actual data. Large records automatically trigger **Overflow Pages** to handle data exceeding page capacity.
+- **Keys & Identifiers**: Uses a 6-byte **Primary Key (PK)** for logical mapping and a 6-byte **Record Identifier (RID)** (Slot + Page ID) for direct physical addressing.
 
 ### 3. MVCC and Snapshot Isolation
 - **Non-blocking Reads**: Read operations are not blocked by write operations.

package/dist/types/core/RowIndexStategy.d.ts

@@ -1,19 +0,0 @@
-import { type BPTreeNode, type SerializeStrategyHead, SerializeStrategyAsync } from 'serializable-bptree';
-import { PageFileSystem } from './PageFileSystem';
-import { IndexPageManager, PageManagerFactory } from './Page';
-import { TextCodec } from '../utils/TextCodec';
-export declare class RowIdentifierStrategy extends SerializeStrategyAsync<number, number> {
-    readonly order: number;
-    protected readonly pfs: PageFileSystem;
-    protected rootPageId: number;
-    protected factory: PageManagerFactory;
-    protected indexPageManger: IndexPageManager;
-    protected codec: TextCodec;
-    constructor(order: number, pfs: PageFileSystem);
-    id(isLeaf: boolean): Promise<string>;
-    read(id: string): Promise<BPTreeNode<number, number>>;
-    write(id: string, node: BPTreeNode<number, number>): Promise<void>;
-    delete(id: string): Promise<void>;
-    readHead(): Promise<SerializeStrategyHead | null>;
-    writeHead(head: SerializeStrategyHead): Promise<void>;
-}

package/dist/types/core/Shard.d.ts

@@ -1,75 +0,0 @@
-import type { ShardOptions, ShardMetadata } from '../types';
-import { ShardAPI } from './ShareAPI';
-import { Transaction } from './transaction/Transaction';
-/**
- * Class for managing Shard files.
- */
-export declare class Shard {
-    protected readonly api: ShardAPI;
-    constructor(file: string, options?: ShardOptions);
-    /**
-     * Gets the options used to open the shard.
-     * @returns Options used to open the shard.
-     */
-    get options(): Required<ShardOptions>;
-    /**
-     * Creates a transaction.
-     * The created transaction object can be used to add or modify data.
-     * A transaction must be terminated by calling either `commit` or `rollback`.
-     * @returns Transaction object
-     */
-    createTransaction(): Transaction;
-    /**
-     * Initializes the shard instance.
-     * Must be called before using the shard instance.
-     * If not called, the shard instance cannot be used.
-     */
-    init(): Promise<void>;
-    /**
-     * Retrieves metadata from the shard.
-     * @returns Metadata of the shard.
-     */
-    getMetadata(): Promise<ShardMetadata>;
-    /**
-     * Inserts data. Returns the PK of the added row.
-     * @param data Data to add
-     * @param tx Transaction
-     * @returns PK of the added data
-     */
-    insert(data: string | Uint8Array, tx?: Transaction): Promise<number>;
-    /**
-     * Inserts multiple data in batch.
-     * If a transaction is not provided, it internally creates a single transaction to process.
-     * @param dataList Array of data to add
-     * @param tx Transaction
-     * @returns Array of PKs of the added data
-     */
-    insertBatch(dataList: (string | Uint8Array)[], tx?: Transaction): Promise<number[]>;
-    /**
-     * Updates data.
-     * @param pk PK of the data to update
-     * @param data Data to update
-     * @param tx Transaction
-     */
-    update(pk: number, data: string | Uint8Array, tx?: Transaction): Promise<void>;
-    /**
-     * Deletes data.
-     * @param pk PK of the data to delete
-     * @param tx Transaction
-     */
-    delete(pk: number, tx?: Transaction): Promise<void>;
-    /**
-     * Selects data.
-     * @param pk PK of the data to select
-     * @param asRaw Whether to return the selected data as raw
-     * @param tx Transaction
-     * @returns Selected data
-     */
-    select(pk: number, asRaw: true, tx?: Transaction): Promise<Uint8Array | null>;
-    select(pk: number, asRaw: false, tx?: Transaction): Promise<string | null>;
-    select(pk: number, asRaw?: boolean, tx?: Transaction): Promise<string | null>;
-    /**
-     * Closes the shard file.
-     */
-    close(): Promise<void>;
-}

package/dist/types/core/ShareAPI.d.ts

@@ -1,121 +0,0 @@
-import type { ShardOptions, ShardMetadata } from '../types';
-import { PageFileSystem } from './PageFileSystem';
-import { RowTableEngine } from './RowTableEngine';
-import { TextCodec } from '../utils/TextCodec';
-import { LockManager } from './transaction/LockManager';
-import { Transaction } from './transaction/Transaction';
-/**
- * Class for managing Shard files.
- */
-export declare class ShardAPI {
-    protected file: string;
-    protected fileHandle: number;
-    readonly options: Required<ShardOptions>;
-    protected readonly pfs: PageFileSystem;
-    protected readonly rowTableEngine: RowTableEngine;
-    protected readonly lockManager: LockManager;
-    protected readonly textCodec: TextCodec;
-    protected initialized: boolean;
-    private txIdCounter;
-    protected constructor(file: string, fileHandle: number, options: Required<ShardOptions>);
-    /**
-     * Verifies if the page file is a valid Shard file.
-     * The metadata page must be located at the beginning of the Shard file.
-     * @param fileHandle File handle
-     * @returns Whether the page file is a valid Shard file
-     */
-    static VerifyFormat(fileHandle: number): boolean;
-    /**
-     * Fills missing options with default values.
-     * @param options Options
-     * @returns Options filled without omissions
-     */
-    static VerboseOptions(options?: ShardOptions): Required<ShardOptions>;
-    /**
-     * Initializes the database file.
-     * The first page is initialized as the metadata page.
-     * The second page is initialized as the first data page.
-     * @param fileHandle File handle
-     */
-    static InitializeFile(fileHandle: number, options: Required<ShardOptions>): void;
-    /**
-     * Opens the database file. If the file does not exist, it initializes it.
-     * @param file Database file path
-     * @param options Options
-     * @returns Shard instance
-     */
-    static Use(file: string, options?: ShardOptions): ShardAPI;
-    /**
-     * Initializes the shard instance.
-     * Must be called before using the shard instance.
-     * If not called, the shard instance cannot be used.
-     */
-    init(): Promise<void>;
-    /**
-     * Creates a transaction.
-     * The created transaction object can be used to add or modify data.
-     * A transaction must be terminated by calling either `commit` or `rollback`.
-     * @returns Transaction object
-     */
-    createTransaction(): Transaction;
-    /**
-     * Runs a callback function within a transaction context.
-     * If no transaction is provided, a new transaction is created.
-     * The transaction is committed if the callback completes successfully,
-     * or rolled back if an error occurs.
-     * @param callback The callback function to run within the transaction context.
-     * @param tx The transaction to use. If not provided, a new transaction is created.
-     * @returns The result of the callback function.
-     */
-    private runWithDefault;
-    /**
-     * Retrieves metadata from the shard.
-     * @returns Metadata of the shard.
-     */
-    getMetadata(): Promise<ShardMetadata>;
-    /**
-     * Inserts data. Returns the PK of the added row.
-     * @param data Data to add
-     * @param incrementRowCount Whether to increment the row count to metadata
-     * @param tx Transaction
-     * @returns PK of the added data
-     */
-    insert(data: string | Uint8Array, incrementRowCount?: boolean, tx?: Transaction): Promise<number>;
-    /**
-     * Inserts multiple data in batch.
-     * If a transaction is not provided, it internally creates a single transaction to process.
-     * @param dataList Array of data to add
-     * @param incrementRowCount Whether to increment the row count to metadata
-     * @param tx Transaction
-     * @returns Array of PKs of the added data
-     */
-    insertBatch(dataList: (string | Uint8Array)[], incrementRowCount?: boolean, tx?: Transaction): Promise<number[]>;
-    /**
-     * Updates data.
-     * @param pk PK of the data to update
-     * @param data Data to update
-     * @param tx Transaction
-     */
-    update(pk: number, data: string | Uint8Array, tx?: Transaction): Promise<void>;
-    /**
-     * Deletes data.
-     * @param pk PK of the data to delete
-     * @param decrementRowCount Whether to decrement the row count to metadata
-     * @param tx Transaction
-     */
-    delete(pk: number, decrementRowCount?: boolean, tx?: Transaction): Promise<void>;
-    /**
-     * Selects data.
-     * @param pk PK of the data to select
-     * @param asRaw Whether to return the selected data as raw
-     * @param tx Transaction
-     * @returns Selected data
-     */
-    select(pk: number, asRaw: true, tx?: Transaction): Promise<Uint8Array | null>;
-    select(pk: number, asRaw: false, tx?: Transaction): Promise<string | null>;
-    select(pk: number, asRaw?: boolean, tx?: Transaction): Promise<string | null>;
-    /**
-     * Closes the shard file.
-     */
-    close(): Promise<void>;
-}