lba 3.2.2 → 3.5.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lba",
-  "version": "3.2.2",
+  "version": "3.5.1",
   "description": "Lightweight, high-performance, file-based key-value store with NoSQL query support.",
   "main": "src/index.js",
   "types": "src/types",

package/src/index.js

@@ -13,7 +13,10 @@ class LBA {
   constructor(dbDir = "lba_storage", options = {}) {
     this.dbDir = path.resolve(dbDir);
     this.shardCount = options.shardCount || 32;
+    this.loadingPromises = new Array(this.shardCount).fill(null);
+
     this.cacheLimit = options.cacheLimit || 10000;
+    this.fastMode = options.fastMode === true;
     this.syncOnWrite = options.syncOnWrite !== false;
     this.compressionLevel = options.compressionLevel || 6;
     this.maxDecompressedSize = options.maxDecompressedSize || 100 * 1024 * 1024;
@@ -22,11 +25,10 @@ class LBA {
     this.vacuumThreshold = options.vacuumThreshold || 500;
 
     const cpuCores = os.cpus().length;
-    if (options.workerCount === "auto" || !options.workerCount) {
-      this.workerLimit = Math.max(1, Math.floor(cpuCores * 0.75));
-    } else {
-      this.workerLimit = options.workerCount;
-    }
+    this.workerLimit =
+      options.workerCount === "auto" || !options.workerCount
+        ? Math.max(1, Math.floor(cpuCores * 0.75))
+        : options.workerCount;
 
     this.dirtyCounts = new Array(this.shardCount).fill(0);
     this.isVacuuming = new Array(this.shardCount).fill(false);
@@ -50,43 +52,56 @@ class LBA {
   }
 
   async _ensureShardLoaded(sIdx) {
-    if (this.isLoaded[sIdx]) return;
-    const fPath = path.join(this.dbDir, `shard_${sIdx}.lba`);
-    const handle = await fs.promises.open(fPath, "a+");
-    this.fileHandles[sIdx] = handle;
+    if (this.isLoaded[sIdx] && this.fileHandles[sIdx]) return;
+    if (this.loadingPromises[sIdx]) return this.loadingPromises[sIdx];
 
-    const { size } = await handle.stat();
-    let offset = 0;
-    const head = Buffer.allocUnsafe(11);
+    this.loadingPromises[sIdx] = (async () => {
+      try {
+        const fPath = path.join(this.dbDir, `shard_${sIdx}.lba`);
+        if (!this.fileHandles[sIdx]) {
+          this.fileHandles[sIdx] = await fs.promises.open(fPath, "a+");
+        }
 
-    while (offset + 11 <= size) {
-      await handle.read(head, 0, 11, offset);
-      if (head[0] !== 0x4c || head[1] !== 0x42) {
-        offset++;
-        continue;
-      }
-      const vLen = head.readUInt32BE(6);
-      const kLen = head[10];
-      const recordSize = 11 + kLen + vLen;
-      if (offset + recordSize > size) break;
-
-      const kBuf = Buffer.allocUnsafe(kLen);
-      await handle.read(kBuf, 0, kLen, offset + 11);
-      const key = kBuf.toString();
-
-      if (vLen > 0) {
-        this.indices[sIdx].set(key, {
-          offset: offset + 11 + kLen,
-          length: vLen,
-          crc: head.readUInt32BE(2),
-          kLen,
-        });
-      } else {
-        this.indices[sIdx].delete(key);
+        const handle = this.fileHandles[sIdx];
+        const { size } = await handle.stat();
+        let offset = 0;
+        const head = Buffer.allocUnsafe(11);
+
+        while (offset + 11 <= size) {
+          await handle.read(head, 0, 11, offset);
+          if (head[0] !== 0x4c || head[1] !== 0x42) {
+            offset++;
+            continue;
+          }
+          const vLen = head.readUInt32BE(6);
+          const kLen = head[10];
+          const recordSize = 11 + kLen + vLen;
+
+          if (offset + recordSize > size) break;
+
+          const kBuf = Buffer.allocUnsafe(kLen);
+          await handle.read(kBuf, 0, kLen, offset + 11);
+          const key = kBuf.toString();
+
+          if (vLen > 0) {
+            this.indices[sIdx].set(key, {
+              offset: offset + 11 + kLen,
+              length: vLen,
+              crc: head.readUInt32BE(2),
+              kLen,
+            });
+          } else {
+            this.indices[sIdx].delete(key);
+          }
+          offset += recordSize;
+        }
+        this.isLoaded[sIdx] = true;
+      } finally {
+        this.loadingPromises[sIdx] = null;
       }
-      offset += recordSize;
-    }
-    this.isLoaded[sIdx] = true;
+    })();
+
+    return this.loadingPromises[sIdx];
   }
 
   async get(key) {
@@ -94,6 +109,7 @@ class LBA {
     return this._enqueue(sIdx, async () => {
       const kStr = String(key);
       if (this.cache.has(kStr)) return structuredClone(this.cache.get(kStr));
+
       const meta = this.indices[sIdx].get(kStr);
       if (!meta) return null;
 
@@ -110,64 +126,69 @@ class LBA {
   async set(key, value) {
     const sIdx = getShard(key, this.shardCount);
     return this._enqueue(sIdx, async () => {
-      const kStr = String(key);
-      const kBuf = Buffer.from(kStr);
-      let vBuf = null,
-        vLen = 0;
+      await this._internalSet(key, value, sIdx);
+    });
+  }
 
-      if (value !== null && value !== undefined) {
-        vBuf = await deflateRawAsync(JSON.stringify(value), {
-          level: this.compressionLevel,
-        });
-        vLen = vBuf.length;
-      }
+  async _internalSet(key, value, sIdx) {
+    const kStr = String(key);
+    const kBuf = Buffer.from(kStr);
+    let vBuf = null,
+      vLen = 0;
 
-      const metaBuf = Buffer.allocUnsafe(5);
-      metaBuf.writeUInt32BE(vLen, 0);
-      metaBuf[4] = kBuf.length;
-      const checksum = calculateCRC32([metaBuf, kBuf, vBuf]);
-
-      const head = Buffer.allocUnsafe(11);
-      head[0] = 0x4c;
-      head[1] = 0x42;
-      head.writeUInt32BE(checksum, 2);
-      head.writeUInt32BE(vLen, 6);
-      head[10] = kBuf.length;
-
-      const { size: pos } = await this.fileHandles[sIdx].stat();
-      await this.fileHandles[sIdx].write(
-        vBuf ? Buffer.concat([head, kBuf, vBuf]) : Buffer.concat([head, kBuf]),
-        0,
-        11 + kBuf.length + vLen,
-        null,
-      );
-      if (this.syncOnWrite) await this.fileHandles[sIdx].datasync();
+    if (value !== null && value !== undefined) {
+      vBuf = await deflateRawAsync(JSON.stringify(value), {
+        level: this.compressionLevel,
+      });
+      vLen = vBuf.length;
+    }
 
-      if (vLen > 0) {
-        this.indices[sIdx].set(kStr, {
-          offset: pos + 11 + kBuf.length,
-          length: vLen,
-          crc: checksum,
-          kLen: kBuf.length,
-        });
-        this._addToCache(kStr, value);
-      } else {
-        this.indices[sIdx].delete(kStr);
-        this.cache.delete(kStr);
-      }
+    const metaBuf = Buffer.allocUnsafe(5);
+    metaBuf.writeUInt32BE(vLen, 0);
+    metaBuf[4] = kBuf.length;
+    const checksum = calculateCRC32([metaBuf, kBuf, vBuf]);
 
-      this.dirtyCounts[sIdx]++;
-      if (this.autoVacuum && this.dirtyCounts[sIdx] >= this.vacuumThreshold) {
-        this.vacuum(sIdx).catch(() => {});
-      }
-    });
+    const head = Buffer.allocUnsafe(11);
+    head[0] = 0x4c;
+    head[1] = 0x42;
+    head.writeUInt32BE(checksum, 2);
+    head.writeUInt32BE(vLen, 6);
+    head[10] = kBuf.length;
+
+    const { size: pos } = await this.fileHandles[sIdx].stat();
+    const writeBuf = vBuf
+      ? Buffer.concat([head, kBuf, vBuf])
+      : Buffer.concat([head, kBuf]);
+
+    await this.fileHandles[sIdx].write(writeBuf, 0, writeBuf.length, null);
+
+    if (!this.fastMode && this.syncOnWrite)
+      await this.fileHandles[sIdx].datasync();
+
+    if (vLen > 0) {
+      this.indices[sIdx].set(kStr, {
+        offset: pos + 11 + kBuf.length,
+        length: vLen,
+        crc: checksum,
+        kLen: kBuf.length,
+      });
+      this._addToCache(kStr, value);
+    } else {
+      this.indices[sIdx].delete(kStr);
+      this.cache.delete(kStr);
+    }
+
+    this.dirtyCounts[sIdx]++;
+    if (this.autoVacuum && this.dirtyCounts[sIdx] >= this.vacuumThreshold) {
+      this._enqueue(sIdx, () => this._internalVacuum(sIdx)).catch(() => {});
+    }
   }
 
   async delete(key) {
     return this.set(key, null);
   }
 
-  async vacuum(sIdx) {
+  async _internalVacuum(sIdx) {
     if (this.isVacuuming[sIdx]) return;
     this.isVacuuming[sIdx] = true;
     try {
@@ -181,14 +202,11 @@ class LBA {
         const vBuf = Buffer.allocUnsafe(meta.length);
         await this.fileHandles[sIdx].read(vBuf, 0, meta.length, meta.offset);
         const kBuf = Buffer.from(key);
-        const metaBuf = Buffer.allocUnsafe(5);
-        metaBuf.writeUInt32BE(meta.length, 0);
-        metaBuf[4] = kBuf.length;
-        const checksum = calculateCRC32([metaBuf, kBuf, vBuf]);
+
         const head = Buffer.allocUnsafe(11);
         head[0] = 0x4c;
         head[1] = 0x42;
-        head.writeUInt32BE(checksum, 2);
+        head.writeUInt32BE(meta.crc, 2);
         head.writeUInt32BE(meta.length, 6);
         head[10] = kBuf.length;
 
@@ -197,7 +215,7 @@ class LBA {
         newIndices.set(key, {
           offset: currentPos + 11 + kBuf.length,
           length: meta.length,
-          crc: checksum,
+          crc: meta.crc,
           kLen: kBuf.length,
         });
         currentPos += block.length;
@@ -213,6 +231,10 @@ class LBA {
     }
   }
 
+  async vacuum(sIdx) {
+    return this._enqueue(sIdx, () => this._internalVacuum(sIdx));
+  }
+
   async find(query = {}) {
     const res = [];
     for (let i = 0; i < this.shardCount; i++) {
@@ -235,6 +257,81 @@ class LBA {
     return targets.length;
   }
 
+  async batchSet(dataObj) {
+    const shardTasks = {};
+    for (const [key, value] of Object.entries(dataObj)) {
+      const sIdx = getShard(key, this.shardCount);
+      if (!shardTasks[sIdx]) shardTasks[sIdx] = [];
+      shardTasks[sIdx].push({ key, value });
+    }
+
+    const tasks = Object.entries(shardTasks).map(([sIdx, items]) => {
+      return this._enqueue(parseInt(sIdx), async () => {
+        for (const { key, value } of items) {
+          await this._internalSet(key, value, parseInt(sIdx));
+        }
+      });
+    });
+    await Promise.all(tasks);
+  }
+
+  async *entries() {
+    for (let i = 0; i < this.shardCount; i++) {
+      await this._enqueue(i, async () => {});
+      for (const key of this.indices[i].keys()) {
+        yield [key, await this.get(key)];
+      }
+    }
+  }
+
+  async clear() {
+    const clearTasks = [];
+    for (let i = 0; i < this.shardCount; i++) {
+      clearTasks.push(
+        this._enqueue(i, async () => {
+          const fPath = path.join(this.dbDir, `shard_${i}.lba`);
+          if (this.fileHandles[i]) await this.fileHandles[i].close();
+          await fs.promises.writeFile(fPath, "");
+          this.fileHandles[i] = await fs.promises.open(fPath, "a+");
+          this.indices[i].clear();
+          this.dirtyCounts[i] = 0;
+        }),
+      );
+    }
+    await Promise.all(clearTasks);
+    this.cache.clear();
+  }
+
+  size() {
+    return this.indices.reduce((acc, idxMap) => acc + idxMap.size, 0);
+  }
+
+  async backup(backupPath) {
+    const targetDir = path.resolve(backupPath);
+    if (!fs.existsSync(targetDir)) fs.mkdirSync(targetDir, { recursive: true });
+
+    const backupTasks = this.indices.map((_, sIdx) => {
+      return this._enqueue(sIdx, async () => {
+        const bPath = path.join(targetDir, `shard_${sIdx}.lba.bak`);
+        const bHandle = await fs.promises.open(bPath, "w");
+        for (const [key, meta] of this.indices[sIdx].entries()) {
+          const vBuf = Buffer.allocUnsafe(meta.length);
+          await this.fileHandles[sIdx].read(vBuf, 0, meta.length, meta.offset);
+          const kBuf = Buffer.from(key);
+          const head = Buffer.allocUnsafe(11);
+          head[0] = 0x4c;
+          head[1] = 0x42;
+          head.writeUInt32BE(meta.crc, 2);
+          head.writeUInt32BE(meta.length, 6);
+          head[10] = kBuf.length;
+          await bHandle.write(Buffer.concat([head, kBuf, vBuf]));
+        }
+        await bHandle.close();
+      });
+    });
+    await Promise.all(backupTasks);
+  }
+
   _addToCache(k, v) {
     if (this.cache.has(k)) this.cache.delete(k);
     else if (this.cache.size >= this.cacheLimit)
@@ -244,13 +341,28 @@ class LBA {
 
   _enqueue(sIdx, task) {
     return (this.queues[sIdx] = this.queues[sIdx]
-      .then(() => this._ensureShardLoaded(sIdx))
-      .then(task));
+      .catch(() => {})
+      .then(async () => {
+        await this._ensureShardLoaded(sIdx);
+
+        if (!this.fileHandles[sIdx]) {
+          const fPath = path.join(this.dbDir, `shard_${sIdx}.lba`);
+          this.fileHandles[sIdx] = await fs.promises.open(fPath, "a+");
+        }
+
+        return task();
+      }));
   }
 
   async close() {
     await Promise.all(this.queues);
-    for (const h of this.fileHandles) if (h) await h.close();
+    await Promise.all(this.loadingPromises.filter((p) => p !== null));
+    for (let i = 0; i < this.shardCount; i++) {
+      if (this.fileHandles[i]) {
+        await this.fileHandles[i].close();
+        this.fileHandles[i] = null;
+      }
+    }
   }
 }
 

package/src/types.ts

@@ -1,26 +1,82 @@
 export type QueryOperator<T = any> = {
-    $eq?: T; $ne?: T; $gt?: T; $gte?: T; $lt?: T; $lte?: T; $in?: T[]; $nin?: T[]; $exists?: boolean;
+    /** Equal to */
+    $eq?: T;
+    /** Not equal to */
+    $ne?: T;
+    /** Greater than */
+    $gt?: T;
+    /** Greater than or equal to */
+    $gte?: T;
+    /** Less than */
+    $lt?: T;
+    /** Less than or equal to */
+    $lte?: T;
+    /** Included in an array */
+    $in?: T[];
+    /** Not included in an array */
+    $nin?: T[];
+    /** Property exists or not */
+    $exists?: boolean;
 };
 
 export type Query<T = any> = { [K in keyof T]?: T[K] | QueryOperator<T[K]>; } & { [key: string]: any };
 
 export interface LBAOptions {
+    /** Number of shards for data distribution. Default: 32 */
     shardCount?: number;
+    /** Maximum number of items to keep in memory cache. Default: 10000 */
     cacheLimit?: number;
+    /** Whether to wait for hardware disk sync on every write. Default: true */
     syncOnWrite?: boolean;
+    /** Zlib compression level (0-9). Default: 6 */
     compressionLevel?: number;
+    /** Automatically run vacuum to clean up stale data. Default: true */
     autoVacuum?: boolean;
+    /** Number of dirty writes before triggering auto-vacuum. Default: 500 */
     vacuumThreshold?: number;
-    // 워커 스레드 설정: 'auto'면 CPU 코어의 75% 사용
+    /** Number of concurrent workers. 'auto' uses 75% of CPU cores. */
     workerCount?: number | 'auto';
+    /** Enable OS-level write buffering for extreme performance. Default: false */
+    fastMode?: boolean;
 }
 
+/**
+ * LBA: A lightweight, high-performance, file-based key-value store for Node.js.
+ * Supports sharding, atomic updates, and zlib compression.
+ */
 export declare class LBA<T = any> {
     constructor(dbDir?: string, options?: LBAOptions);
+
+    /** Retrieves the value associated with the key. Returns null if not found. */
     get(key: string | number): Promise<T | null>;
+
+    /** Stores a value for the given key. Set value to null to delete. */
     set(key: string | number, value: T | null): Promise<void>;
+
+    /** Deletes the entry for the given key. */
     delete(key: string | number): Promise<void>;
+
+    /** Finds records matching the given query. Supports MongoDB-like operators. */
     find(query?: Query<T>): Promise<(T & { _key: string })[]>;
+
+    /** Updates multiple records matching the query with new data. */
     updateMany(query: Query<T>, updateData: Partial<T>): Promise<number>;
+
+    /** Gracefully closes the database and ensures all writes are finished. */
     close(): Promise<void>;
+
+    /** Saves multiple key-value pairs at once for optimized I/O. */
+    batchSet(data: Record<string | number, T>): Promise<void>;
+
+    /** Async iterator that yields all key-value pairs in the database. */
+    entries(): AsyncIterableIterator<[string, T]>;
+
+    /** Wipes all data from the database and resets shards. Use with caution. */
+    clear(): Promise<void>;
+
+    /** Returns the total number of records currently stored in the database. */
+    size(): number;
+
+    /** Creates an optimized incremental backup at the specified path. */
+    backup(backupPath: string): Promise<void>;
 }