lba 1.0.0 → 2.0.0

package/README.md

@@ -9,44 +9,69 @@ It supports sharding, compression (zlib), and atomic updates.
 npm install lba
 ```
 
+## Features
+
+- **High-Concurrency Sharding:** Uses FNV-1a hashing to distribute data across multiple shards, reducing file lock contention and achieving over 28,000+ reads/sec.
+
+- **Memory-Efficient Streaming:** Iterate through millions of records without memory spikes using Async Generators.
+
+- **Atomic Updates:** Built-in update() method ensures thread-safe Read-Modify-Write operations.
+
+- **Smart LRU Cache:** Internal memory cache provides sub-millisecond latency for frequent data access.
+
+- **Automatic Compression:** Integrated zlib compression reduces disk footprint by up to 90%.
+
+- **Data Integrity:** Magic Byte verification detects and recovers from unexpected process terminations.
+
 ## Usage
 
 ```js
-const LBA = require("@your-username/lba");
+const LBA = require("lba");
 
-// Initialize DB (creates 'lba_storage' folder by default)
-const db = new LBA("./my-data");
+// Initialize DB (supports optional config: shardCount, cacheLimit, fastMode)
+const db = new LBA("./my-data", { shardCount: 16 });
 
 (async () => {
-  // 1. Set Data
-  await db.set("user:1001", {
-    name: "Alice",
-    age: 30,
-    items: ["sword", "shield"],
-  });
-
-  // 2. Get Data
+  // 1. Set & Get (Auto-compressed)
+  await db.set("user:1001", { name: "Alice", age: 30 });
   const user = await db.get("user:1001");
-  console.log(user);
-  // Output: { name: 'Alice', age: 30, items: ['sword', 'shield'] }
 
-  // 3. Atomic Update (Thread-safe)
+  // 2. Atomic Update (Prevents race conditions)
   await db.update("user:1001", (data) => {
     data.age += 1;
-    return data; // Return modified data
+    return data;
   });
 
-  // 4. Delete
-  await db.delete("user:1001");
+  // 3. Batch Operations (High-speed bulk processing)
+  await db.batchSet({
+    key1: "value1",
+    key2: "value2",
+  });
+  const results = await db.batchGet(["key1", "key2"]);
 })();
 ```
 
-## Features
+## Memory-Efficient Iteration
 
-- Sharding: Distributes data across multiple files to reduce lock contention.
+For large datasets, use the Async Generator to keep memory usage low.
 
-- Compression: Automatically compresses data using zlib.
+```js
+// Extremely fast: up to 1.7M items/sec processing
+for await (const [key, value] of db.entries({ batchSize: 50 })) {
+  console.log(key, value);
+}
+```
 
-- Atomic Updates: Prevents race conditions during read-modify-write operations.
+## Maintenance
+
+Since LBA uses a log-structured approach, use vacuum() to reclaim disk space.
+
+```js
+await db.vacuum(); // Compacts files and removes deleted entries
+```
 
-- LRU Cache: Built-in memory caching for fast reads.
+| Operation       |      Throughput      |
+| :-------------- | :------------------: |
+| **Batch Write** |   10,000+ ops/sec    |
+| **Batch Read**  |   28,000+ ops/sec    |
+| **Streaming**   | 1,700,000+ items/sec |

package/index.js

@@ -6,12 +6,23 @@ const util = require("util");
 const inflateAsync = util.promisify(zlib.inflateRaw);
 const deflateAsync = util.promisify(zlib.deflateRaw);
 
+const MAGIC_BYTES = [0x4c, 0x42];
+const HEADER_SIZE = 11;
+const DEFAULT_SHARD_COUNT = 32;
+const DEFAULT_CACHE_LIMIT = 10000;
+const DEFAULT_COMPRESSION_LEVEL = 6;
+const FAST_COMPRESSION_LEVEL = 1;
+const FNV_OFFSET = 2166136261;
+const FNV_PRIME = 16777619;
+
 class LBA {
   constructor(dbDir = "lba_storage", options = {}) {
     this.dbDir = path.resolve(dbDir);
-    this.shardCount = options.shardCount || 32;
-    this.cacheLimit = options.cacheLimit || 10000;
-    this.compressionLevel = options.fastMode ? 1 : 6;
+    this.shardCount = options.shardCount || DEFAULT_SHARD_COUNT;
+    this.cacheLimit = options.cacheLimit || DEFAULT_CACHE_LIMIT;
+    this.compressionLevel = options.fastMode
+      ? FAST_COMPRESSION_LEVEL
+      : DEFAULT_COMPRESSION_LEVEL;
 
     this.indices = Array.from({ length: this.shardCount }, () => new Map());
     this.cache = new Map();
@@ -23,18 +34,35 @@ class LBA {
     this.fileHandles = new Array(this.shardCount).fill(null);
     this.isLoaded = new Array(this.shardCount).fill(false);
 
-    if (!fs.existsSync(this.dbDir))
+    this.stats = {
+      cacheHits: 0,
+      cacheMisses: 0,
+      reads: 0,
+      writes: 0,
+    };
+
+    this.headerBuffer = Buffer.allocUnsafe(HEADER_SIZE);
+
+    this._ensureDbDir();
+  }
+
+  _ensureDbDir() {
+    if (!fs.existsSync(this.dbDir)) {
       fs.mkdirSync(this.dbDir, { recursive: true });
+    }
   }
 
   _getShard(key) {
-    let hash = 2166136261;
+    let hash = FNV_OFFSET;
     const sKey = String(key);
-    for (let i = 0; i < sKey.length; i++) {
+    const len = sKey.length;
+
+    for (let i = 0; i < len; i++) {
       hash ^= sKey.charCodeAt(i);
-      hash = Math.imul(hash, 16777619);
+      hash = Math.imul(hash, FNV_PRIME);
     }
-    return Math.abs(hash) % this.shardCount;
+
+    return (hash >>> 0) % this.shardCount;
   }
 
   async _ensureShardLoaded(sIdx) {
@@ -45,44 +73,52 @@ class LBA {
 
     const stat = await this.fileHandles[sIdx].stat();
     const size = stat.size;
+
+    if (size === 0) {
+      this.isLoaded[sIdx] = true;
+      return;
+    }
+
     let offset = 0;
-    const headerSize = 11;
-    const headerBuf = Buffer.alloc(headerSize);
 
-    while (offset + headerSize <= size) {
+    while (offset + HEADER_SIZE <= size) {
       const { bytesRead } = await this.fileHandles[sIdx].read(
-        headerBuf,
+        this.headerBuffer,
         0,
-        headerSize,
+        HEADER_SIZE,
         offset,
       );
-      if (bytesRead < headerSize) break;
 
-      if (headerBuf[0] !== 0x4c || headerBuf[1] !== 0x42) {
+      if (bytesRead < HEADER_SIZE) break;
+
+      if (
+        this.headerBuffer[0] !== MAGIC_BYTES[0] ||
+        this.headerBuffer[1] !== MAGIC_BYTES[1]
+      ) {
         console.warn(
-          `Shard ${sIdx}: 데이터 손상 감지 (Offset: ${offset}). 복구를 위해 이후 데이터를 무시합니다.`,
+          `Shard ${sIdx}: Data corruption detected (Offset: ${offset}). Subsequent data will be ignored for recovery.`,
         );
         break;
       }
 
-      const vLen = headerBuf.readUInt32BE(6);
-      const kLen = headerBuf[10];
-      const recordTotalSize = headerSize + kLen + vLen;
+      const vLen = this.headerBuffer.readUInt32BE(6);
+      const kLen = this.headerBuffer[10];
+      const recordTotalSize = HEADER_SIZE + kLen + vLen;
 
       if (offset + recordTotalSize > size) {
         console.warn(
-          `Shard ${sIdx}: 불완전한 기록 발견 (Offset: ${offset}). 마지막 데이터를 폐기합니다.`,
+          `Shard ${sIdx}: Incomplete record detected (Offset: ${offset}). Discarding last data.`,
         );
         break;
       }
 
-      const kBuf = Buffer.alloc(kLen);
-      await this.fileHandles[sIdx].read(kBuf, 0, kLen, offset + headerSize);
+      const kBuf = Buffer.allocUnsafe(kLen);
+      await this.fileHandles[sIdx].read(kBuf, 0, kLen, offset + HEADER_SIZE);
       const key = kBuf.toString();
 
       if (vLen > 0) {
         this.indices[sIdx].set(key, {
-          offset: offset + headerSize + kLen,
+          offset: offset + HEADER_SIZE + kLen,
           length: vLen,
         });
       } else {
@@ -96,28 +132,30 @@ class LBA {
   }
 
   async _readImpl(sIdx, key) {
-    if (this.cache.has(key)) {
-      const val = this.cache.get(key);
-      this.cache.delete(key);
-      this.cache.set(key, val);
+    const keyStr = String(key);
+
+    if (this.cache.has(keyStr)) {
+      this.stats.cacheHits++;
+      const val = this.cache.get(keyStr);
+      this.cache.delete(keyStr);
+      this.cache.set(keyStr, val);
       return val;
     }
 
-    const meta = this.indices[sIdx].get(String(key));
+    this.stats.cacheMisses++;
+    this.stats.reads++;
+
+    const meta = this.indices[sIdx].get(keyStr);
     if (!meta) return null;
 
-    const buf = Buffer.alloc(meta.length);
+    const buf = Buffer.allocUnsafe(meta.length);
     await this.fileHandles[sIdx].read(buf, 0, meta.length, meta.offset);
 
     try {
       const decompressed = await inflateAsync(buf);
       const data = JSON.parse(decompressed.toString());
 
-      if (this.cache.size >= this.cacheLimit) {
-        const firstKey = this.cache.keys().next().value;
-        this.cache.delete(firstKey);
-      }
-      this.cache.set(String(key), data);
+      this._addToCache(keyStr, data);
 
       return data;
     } catch (err) {
@@ -126,7 +164,22 @@ class LBA {
     }
   }
 
+  _addToCache(key, value) {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    }
+
+    while (this.cache.size >= this.cacheLimit) {
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+
+    this.cache.set(key, value);
+  }
+
   async _writeImpl(sIdx, key, value) {
+    this.stats.writes++;
+
     const kStr = String(key);
     const kBuf = Buffer.from(kStr);
 
@@ -139,9 +192,9 @@ class LBA {
       vLen = vBuf.length;
     }
 
-    const head = Buffer.alloc(11);
-    head[0] = 0x4c;
-    head[1] = 0x42;
+    const head = Buffer.allocUnsafe(HEADER_SIZE);
+    head[0] = MAGIC_BYTES[0];
+    head[1] = MAGIC_BYTES[1];
     head.writeUInt32BE(0, 2);
     head.writeUInt32BE(vLen, 6);
     head[10] = kBuf.length;
@@ -157,12 +210,11 @@ class LBA {
 
     if (vLen > 0) {
       this.indices[sIdx].set(kStr, {
-        offset: pos + 11 + kBuf.length,
+        offset: pos + HEADER_SIZE + kBuf.length,
         length: vLen,
       });
 
-      this.cache.delete(kStr);
-      this.cache.set(kStr, value);
+      this._addToCache(kStr, value);
     } else {
       this.indices[sIdx].delete(kStr);
       this.cache.delete(kStr);
@@ -173,7 +225,10 @@ class LBA {
     const next = this.queues[sIdx]
       .then(() => this._ensureShardLoaded(sIdx))
       .then(task)
-      .catch((err) => console.error(`LBA Error (Shard ${sIdx}):`, err));
+      .catch((err) => {
+        console.error(`LBA Error (Shard ${sIdx}):`, err);
+        throw err;
+      });
 
     this.queues[sIdx] = next;
     return next;
@@ -201,11 +256,219 @@ class LBA {
       if (next !== undefined) {
         await this._writeImpl(sIdx, key, next);
       }
+      return next;
     });
   }
 
+  async forEach(callback, options = {}) {
+    const batchSize = options.batchSize || 100;
+    let processed = 0;
+    let batch = [];
+
+    for (let i = 0; i < this.shardCount; i++) {
+      await this._enqueue(i, async () => {
+        for (const key of this.indices[i].keys()) {
+          try {
+            const value = await this._readImpl(i, key);
+            if (value !== null) {
+              batch.push({ key, value });
+
+              if (batch.length >= batchSize) {
+                for (const item of batch) {
+                  await callback(item.key, item.value);
+                  processed++;
+                }
+                batch = [];
+              }
+            }
+          } catch (err) {
+            console.error(`Error reading key ${key} from shard ${i}:`, err);
+          }
+        }
+      });
+    }
+
+    for (const item of batch) {
+      await callback(item.key, item.value);
+      processed++;
+    }
+
+    return processed;
+  }
+
+  async *entries(options = {}) {
+    const batchSize = options.batchSize || 50;
+
+    for (let i = 0; i < this.shardCount; i++) {
+      const entries = await this._enqueue(i, async () => {
+        const batch = [];
+
+        for (const key of this.indices[i].keys()) {
+          try {
+            const value = await this._readImpl(i, key);
+            if (value !== null) {
+              batch.push([key, value]);
+
+              if (batch.length >= batchSize) {
+                const result = [...batch];
+                batch.length = 0;
+                return result;
+              }
+            }
+          } catch (err) {
+            console.error(`Error reading key ${key} from shard ${i}:`, err);
+          }
+        }
+
+        return batch;
+      });
+
+      for (const entry of entries) {
+        yield entry;
+      }
+    }
+  }
+
+  async getAll(options = {}) {
+    const maxSize = options.maxSize || Infinity;
+    const results = {};
+    let count = 0;
+
+    await this.forEach(
+      (key, value) => {
+        if (count >= maxSize) {
+          return;
+        }
+        results[key] = value;
+        count++;
+      },
+      { batchSize: options.batchSize || 100 },
+    );
+
+    return results;
+  }
+
+  async keys() {
+    const allKeys = [];
+
+    for (let i = 0; i < this.shardCount; i++) {
+      await this._enqueue(i, async () => {
+        allKeys.push(...this.indices[i].keys());
+      });
+    }
+
+    return allKeys;
+  }
+
+  async count() {
+    let total = 0;
+
+    const counts = await Promise.all(
+      Array.from({ length: this.shardCount }, (_, i) =>
+        this._enqueue(i, async () => this.indices[i].size),
+      ),
+    );
+
+    return counts.reduce((sum, count) => sum + count, 0);
+  }
+
+  async batchGet(keys) {
+    const results = {};
+    const keysByShard = new Map();
+
+    for (const key of keys) {
+      const sIdx = this._getShard(key);
+      if (!keysByShard.has(sIdx)) {
+        keysByShard.set(sIdx, []);
+      }
+      keysByShard.get(sIdx).push(key);
+    }
+
+    const promises = [];
+    for (const [sIdx, shardKeys] of keysByShard) {
+      promises.push(
+        this._enqueue(sIdx, async () => {
+          const shardResults = {};
+          for (const key of shardKeys) {
+            try {
+              const value = await this._readImpl(sIdx, key);
+              if (value !== null) {
+                shardResults[key] = value;
+              }
+            } catch (err) {
+              console.error(`Error in batchGet for key ${key}:`, err);
+            }
+          }
+          return shardResults;
+        }),
+      );
+    }
+
+    const shardResults = await Promise.all(promises);
+
+    for (const shardResult of shardResults) {
+      Object.assign(results, shardResult);
+    }
+
+    return results;
+  }
+
+  async batchSet(entries) {
+    const entriesByShard = new Map();
+
+    for (const [key, value] of Object.entries(entries)) {
+      const sIdx = this._getShard(key);
+      if (!entriesByShard.has(sIdx)) {
+        entriesByShard.set(sIdx, []);
+      }
+      entriesByShard.get(sIdx).push([key, value]);
+    }
+
+    const promises = [];
+    for (const [sIdx, shardEntries] of entriesByShard) {
+      promises.push(
+        this._enqueue(sIdx, async () => {
+          for (const [key, value] of shardEntries) {
+            await this._writeImpl(sIdx, key, value);
+          }
+        }),
+      );
+    }
+
+    await Promise.all(promises);
+  }
+
+  getStats() {
+    const hitRate =
+      this.stats.cacheHits + this.stats.cacheMisses > 0
+        ? (
+            (this.stats.cacheHits /
+              (this.stats.cacheHits + this.stats.cacheMisses)) *
+            100
+          ).toFixed(2)
+        : 0;
+
+    return {
+      ...this.stats,
+      cacheSize: this.cache.size,
+      cacheLimit: this.cacheLimit,
+      cacheHitRate: `${hitRate}%`,
+      shardCount: this.shardCount,
+    };
+  }
+
+  resetStats() {
+    this.stats = {
+      cacheHits: 0,
+      cacheMisses: 0,
+      reads: 0,
+      writes: 0,
+    };
+  }
+
   async vacuum() {
     const tasks = [];
+
     for (let i = 0; i < this.shardCount; i++) {
       tasks.push(
         this._enqueue(i, async () => {
@@ -214,49 +477,84 @@ class LBA {
           const tmpPath = path.join(this.dbDir, `vacuum_${i}.tmp`);
           const oldPath = path.join(this.dbDir, `shard_${i}.lba`);
 
-          const tmpHandle = await fs.promises.open(tmpPath, "w");
-          let newPos = 0;
+          let tmpHandle = null;
 
-          for (const [key, meta] of this.indices[i]) {
-            const vBuf = Buffer.alloc(meta.length);
-            await this.fileHandles[i].read(vBuf, 0, meta.length, meta.offset);
+          try {
+            tmpHandle = await fs.promises.open(tmpPath, "w");
+            let newPos = 0;
 
-            const kBuf = Buffer.from(key);
+            for (const [key, meta] of this.indices[i]) {
+              const vBuf = Buffer.allocUnsafe(meta.length);
+              await this.fileHandles[i].read(vBuf, 0, meta.length, meta.offset);
 
-            const head = Buffer.alloc(11);
-            head[0] = 0x4c;
-            head[1] = 0x42;
-            head.writeUInt32BE(0, 2);
-            head.writeUInt32BE(vBuf.length, 6);
-            head[10] = kBuf.length;
+              const kBuf = Buffer.from(key);
 
-            const block = Buffer.concat([head, kBuf, vBuf]);
+              const head = Buffer.allocUnsafe(HEADER_SIZE);
+              head[0] = MAGIC_BYTES[0];
+              head[1] = MAGIC_BYTES[1];
+              head.writeUInt32BE(0, 2);
+              head.writeUInt32BE(vBuf.length, 6);
+              head[10] = kBuf.length;
 
-            await tmpHandle.write(block);
+              const block = Buffer.concat([head, kBuf, vBuf]);
 
-            meta.offset = newPos + 11 + kBuf.length;
-            newPos += block.length;
-          }
+              await tmpHandle.write(block);
+
+              meta.offset = newPos + HEADER_SIZE + kBuf.length;
+              newPos += block.length;
+            }
 
-          await tmpHandle.close();
-          await this.fileHandles[i].close();
+            await tmpHandle.close();
+            tmpHandle = null;
 
-          await fs.promises.rename(tmpPath, oldPath);
+            await this.fileHandles[i].close();
 
-          this.fileHandles[i] = await fs.promises.open(oldPath, "a+");
+            await fs.promises.rename(tmpPath, oldPath);
+
+            this.fileHandles[i] = await fs.promises.open(oldPath, "a+");
+          } catch (err) {
+            console.error(`Vacuum error for shard ${i}:`, err);
+
+            if (tmpHandle) {
+              try {
+                await tmpHandle.close();
+              } catch (e) {}
+            }
+
+            if (fs.existsSync(tmpPath)) {
+              try {
+                await fs.promises.unlink(tmpPath);
+              } catch (e) {}
+            }
+
+            throw err;
+          }
         }),
       );
     }
+
     await Promise.all(tasks);
     this.cache.clear();
   }
 
   async close() {
     await Promise.all(this.queues);
-    for (const handle of this.fileHandles) {
-      if (handle) await handle.close();
-    }
+
+    const closePromises = this.fileHandles.map(async (handle) => {
+      if (handle) {
+        try {
+          await handle.close();
+        } catch (err) {
+          console.error("Error closing file handle:", err);
+        }
+      }
+    });
+
+    await Promise.all(closePromises);
+
+    this.fileHandles.fill(null);
     this.isLoaded.fill(false);
+    this.cache.clear();
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lba",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "description": "A lightweight, log-structured binary key-value store.",
   "main": "index.js",
   "scripts": {