@liorandb/core 1.0.19 → 1.1.1

package/src/core/wal.ts

@@ -16,12 +16,11 @@ type StoredRecord = WALRecord & { crc: number };
    CONSTANTS
 ========================= */
 
-const MAX_WAL_SIZE = 16 * 1024 * 1024; // 16 MB
+const MAX_WAL_SIZE = 16 * 1024 * 1024; // 16MB
 const WAL_DIR = "__wal";
 
 /* =========================
-   CRC32 IMPLEMENTATION
-   (no dependencies)
+   CRC32 (no deps)
 ========================= */
 
 const CRC32_TABLE = (() => {
@@ -37,12 +36,11 @@ const CRC32_TABLE = (() => {
 })();
 
 function crc32(input: string): number {
-  let crc = 0xFFFFFFFF;
+  let crc = 0xffffffff;
   for (let i = 0; i < input.length; i++) {
-    const byte = input.charCodeAt(i);
-    crc = CRC32_TABLE[(crc ^ byte) & 0xFF] ^ (crc >>> 8);
+    crc = CRC32_TABLE[(crc ^ input.charCodeAt(i)) & 0xff] ^ (crc >>> 8);
   }
-  return (crc ^ 0xFFFFFFFF) >>> 0;
+  return (crc ^ 0xffffffff) >>> 0;
 }
 
 /* =========================
@@ -54,11 +52,20 @@ export class WALManager {
   private currentGen = 1;
   private lsn = 0;
   private fd: fs.promises.FileHandle | null = null;
+  private readonlyMode: boolean;
 
-  constructor(baseDir: string) {
+  constructor(baseDir: string, options?: { readonly?: boolean }) {
     this.walDir = path.join(baseDir, WAL_DIR);
-    fs.mkdirSync(this.walDir, { recursive: true });
-    this.currentGen = this.detectLastGeneration();
+    this.readonlyMode = options?.readonly ?? false;
+
+    if (!this.readonlyMode) {
+      fs.mkdirSync(this.walDir, { recursive: true });
+    }
+
+    if (fs.existsSync(this.walDir)) {
+      this.currentGen = this.detectLastGeneration();
+      this.recoverLSNFromExistingLogs();
+    }
   }
 
   /* -------------------------
@@ -80,20 +87,69 @@ export class WALManager {
 
     for (const f of files) {
       const m = f.match(/^wal-(\d+)\.log$/);
-      if (m) max = Math.max(max, Number(m[1]));
+      if (m) {
+        const gen = Number(m[1]);
+        if (!Number.isNaN(gen)) {
+          max = Math.max(max, gen);
+        }
+      }
     }
 
     return max || 1;
   }
 
+  private recoverLSNFromExistingLogs() {
+    const files = this.getSortedWalFiles();
+
+    for (const file of files) {
+      const filePath = path.join(this.walDir, file);
+      const lines = fs.readFileSync(filePath, "utf8").split("\n");
+
+      for (const line of lines) {
+        if (!line.trim()) continue;
+
+        try {
+          const parsed: StoredRecord = JSON.parse(line);
+          const { crc, ...record } = parsed;
+
+          if (crc32(JSON.stringify(record)) !== crc) break;
+
+          this.lsn = Math.max(this.lsn, record.lsn);
+        } catch {
+          break;
+        }
+      }
+    }
+  }
+
+  private getSortedWalFiles(): string[] {
+    if (!fs.existsSync(this.walDir)) return [];
+
+    return fs
+      .readdirSync(this.walDir)
+      .filter(f => /^wal-\d+\.log$/.test(f))
+      .sort((a, b) => {
+        const ga = Number(a.match(/^wal-(\d+)\.log$/)![1]);
+        const gb = Number(b.match(/^wal-(\d+)\.log$/)![1]);
+        return ga - gb;
+      });
+  }
+
   private async open() {
+    if (this.readonlyMode) {
+      throw new Error("WAL is in readonly replica mode");
+    }
+
     if (!this.fd) {
       this.fd = await fs.promises.open(this.walPath(), "a");
     }
   }
 
   private async rotate() {
+    if (this.readonlyMode) return;
+
     if (this.fd) {
+      await this.fd.sync();
       await this.fd.close();
       this.fd = null;
     }
@@ -101,10 +157,14 @@ export class WALManager {
   }
 
   /* -------------------------
-     APPEND
+     APPEND (Primary only)
   ------------------------- */
 
   async append(record: Omit<WALRecord, "lsn">): Promise<number> {
+    if (this.readonlyMode) {
+      throw new Error("Cannot append WAL in readonly replica mode");
+    }
+
     await this.open();
 
     const full: WALRecord = {
@@ -113,12 +173,15 @@ export class WALManager {
     };
 
     const body = JSON.stringify(full);
+
     const stored: StoredRecord = {
       ...full,
       crc: crc32(body)
     };
 
-    await this.fd!.write(JSON.stringify(stored) + "\n");
+    const line = JSON.stringify(stored) + "\n";
+
+    await this.fd!.write(line);
     await this.fd!.sync();
 
     const stat = await this.fd!.stat();
@@ -130,7 +193,7 @@ export class WALManager {
   }
 
   /* -------------------------
-     REPLAY
+     REPLAY (Replica allowed)
   ------------------------- */
 
   async replay(
@@ -139,55 +202,72 @@ export class WALManager {
   ): Promise<void> {
     if (!fs.existsSync(this.walDir)) return;
 
-    const files = fs
-      .readdirSync(this.walDir)
-      .filter(f => f.startsWith("wal-"))
-      .sort();
+    const files = this.getSortedWalFiles();
 
     for (const file of files) {
       const filePath = path.join(this.walDir, file);
-      const data = fs.readFileSync(filePath, "utf8");
-      const lines = data.split("\n");
+
+      const fd = fs.openSync(
+        filePath,
+        this.readonlyMode ? "r" : "r+"
+      );
+
+      const content = fs.readFileSync(filePath, "utf8");
+      const lines = content.split("\n");
+
+      let validOffset = 0;
 
       for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        if (!line.trim()) continue;
+        if (!line.trim()) {
+          validOffset += line.length + 1;
+          continue;
+        }
 
         let parsed: StoredRecord;
+
         try {
           parsed = JSON.parse(line);
         } catch {
-          console.error("WAL parse error, stopping replay");
-          return;
+          break;
         }
 
         const { crc, ...record } = parsed;
         const expected = crc32(JSON.stringify(record));
 
         if (expected !== crc) {
-          console.error(
-            "WAL checksum mismatch, stopping replay",
-            { file, line: i + 1 }
-          );
-          return;
+          break;
         }
 
+        validOffset += line.length + 1;
+
         if (record.lsn <= fromLSN) continue;
 
         this.lsn = Math.max(this.lsn, record.lsn);
         await apply(record);
       }
+
+      if (!this.readonlyMode) {
+        const stat = fs.fstatSync(fd);
+        if (validOffset < stat.size) {
+          fs.ftruncateSync(fd, validOffset);
+        }
+      }
+
+      fs.closeSync(fd);
     }
   }
 
   /* -------------------------
-     CLEANUP
+     CLEANUP (Primary only)
   ------------------------- */
 
   async cleanup(beforeGen: number) {
+    if (this.readonlyMode) return;
     if (!fs.existsSync(this.walDir)) return;
 
     const files = fs.readdirSync(this.walDir);
+
     for (const f of files) {
       const m = f.match(/^wal-(\d+)\.log$/);
       if (!m) continue;
@@ -210,4 +290,8 @@ export class WALManager {
   getCurrentGen() {
     return this.currentGen;
   }
+
+  isReadonly() {
+    return this.readonlyMode;
+  }
 }

package/src/ipc/index.ts

@@ -8,6 +8,8 @@ class CollectionProxy {
     private collectionName: string
   ) {}
 
+  /* ------------------------------ INTERNAL CALLERS ------------------------------ */
+
   private call(method: string, params: any[]): Promise<any> {
     return dbQueue.exec("op", {
       db: this.dbName,
@@ -26,38 +28,56 @@ class CollectionProxy {
     });
   }
 
-  private callCompact(): Promise<any> {
-    return dbQueue.exec("compact:collection", {
-      db: this.dbName,
-      col: this.collectionName
-    });
-  }
-
   /* ------------------------------ CRUD ------------------------------ */
 
-  insertOne = (doc: any) => this.call("insertOne", [doc]);
-  insertMany = (docs: any[]) => this.call("insertMany", [docs]);
-  find = (query?: any) => this.call("find", [query]);
-  findOne = (query?: any) => this.call("findOne", [query]);
+  insertOne = (doc: any) =>
+    this.call("insertOne", [doc]);
+
+  insertMany = (docs: any[]) =>
+    this.call("insertMany", [docs]);
+
+  find = (query?: any) =>
+    this.call("find", [query]);
+
+  findOne = (query?: any) =>
+    this.call("findOne", [query]);
+
   updateOne = (filter: any, update: any, options?: any) =>
     this.call("updateOne", [filter, update, options]);
+
   updateMany = (filter: any, update: any) =>
     this.call("updateMany", [filter, update]);
-  deleteOne = (filter: any) => this.call("deleteOne", [filter]);
-  deleteMany = (filter: any) => this.call("deleteMany", [filter]);
+
+  deleteOne = (filter: any) =>
+    this.call("deleteOne", [filter]);
+
+  deleteMany = (filter: any) =>
+    this.call("deleteMany", [filter]);
+
   countDocuments = (filter?: any) =>
     this.call("countDocuments", [filter]);
 
   /* ------------------------------ INDEX ----------------------------- */
 
-  createIndex = (def: any) => this.callIndex("createIndex", [def]);
-  dropIndex = (field: string) => this.callIndex("dropIndex", [field]);
-  listIndexes = () => this.callIndex("listIndexes", []);
-  rebuildIndexes = () => this.callIndex("rebuildIndexes", []);
+  createIndex = (def: any) =>
+    this.callIndex("createIndex", [def]);
+
+  dropIndex = (field: string) =>
+    this.callIndex("dropIndex", [field]);
+
+  listIndexes = () =>
+    this.callIndex("listIndexes", []);
+
+  rebuildIndexes = () =>
+    this.callIndex("rebuildIndexes", []);
 
   /* --------------------------- COMPACTION --------------------------- */
 
-  compact = () => this.callCompact();
+  compact = () =>
+    dbQueue.exec("compact:collection", {
+      db: this.dbName,
+      col: this.collectionName
+    });
 }
 
 /* -------------------------------- DATABASE PROXY -------------------------------- */
@@ -103,9 +123,11 @@ class LioranManagerIPC {
   }
 
   shutdown() {
-    return dbQueue.shutdown();
+    return dbQueue.exec("shutdown", {});
   }
 }
 
+/* -------------------------------- EXPORTS -------------------------------- */
+
 export const manager = new LioranManagerIPC();
 export type { CollectionProxy, DBProxy };

package/src/ipc/pool.ts

@@ -0,0 +1,136 @@
+import { Worker } from "worker_threads";
+import os from "os";
+import path from "path";
+import { fileURLToPath } from "url";
+
+/**
+ * Worker Thread Pool
+ *
+ * - Spawns multiple worker threads (based on CPU cores)
+ * - Auto-restarts crashed workers
+ * - Supports graceful shutdown
+ * - Round-robin task scheduling
+ */
+
+export class IPCWorkerPool {
+  private workers: Worker[] = [];
+  private workerCount: number;
+  private shuttingDown = false;
+  private rrIndex = 0;
+
+  constructor() {
+    // Minimum 2 workers, scale with CPU cores
+    this.workerCount = Math.max(2, os.cpus().length);
+  }
+
+  /* -------------------------------------------------- */
+  /* START POOL                                         */
+  /* -------------------------------------------------- */
+
+  start() {
+    for (let i = 0; i < this.workerCount; i++) {
+      this.spawnWorker();
+    }
+
+    console.log(
+      `[WorkerPool] Started ${this.workerCount} worker threads`
+    );
+  }
+
+  /* -------------------------------------------------- */
+  /* SPAWN WORKER                                       */
+  /* -------------------------------------------------- */
+
+  private spawnWorker() {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+
+    // Worker compiled output must exist in dist
+    const workerFile = path.join(__dirname, "worker.js");
+
+    const worker = new Worker(workerFile);
+
+    worker.on("exit", code => {
+      if (this.shuttingDown) return;
+
+      console.error(
+        `[WorkerPool] Worker exited (code=${code}). Restarting...`
+      );
+
+      // Remove dead worker
+      this.workers = this.workers.filter(w => w !== worker);
+
+      // Restart after short delay
+      setTimeout(() => {
+        this.spawnWorker();
+      }, 500);
+    });
+
+    worker.on("error", err => {
+      console.error("[WorkerPool] Worker error:", err);
+    });
+
+    this.workers.push(worker);
+  }
+
+  /* -------------------------------------------------- */
+  /* EXECUTE TASK                                       */
+  /* -------------------------------------------------- */
+
+  exec(task: any): Promise<any> {
+    if (this.workers.length === 0) {
+      throw new Error("No workers available");
+    }
+
+    const worker = this.workers[this.rrIndex];
+    this.rrIndex = (this.rrIndex + 1) % this.workers.length;
+
+    return new Promise((resolve, reject) => {
+      const id = Date.now() + Math.random();
+
+      const messageHandler = (msg: any) => {
+        if (msg.id !== id) return;
+
+        worker.off("message", messageHandler);
+
+        if (msg.ok) resolve(msg.result);
+        else reject(new Error(msg.error));
+      };
+
+      worker.on("message", messageHandler);
+
+      worker.postMessage({
+        id,
+        task
+      });
+    });
+  }
+
+  /* -------------------------------------------------- */
+  /* SHUTDOWN                                           */
+  /* -------------------------------------------------- */
+
+  async shutdown() {
+    this.shuttingDown = true;
+
+    console.log("[WorkerPool] Shutting down worker threads...");
+
+    for (const worker of this.workers) {
+      try {
+        await worker.terminate();
+      } catch (err) {
+        console.error("[WorkerPool] Worker terminate error:", err);
+      }
+    }
+
+    this.workers = [];
+  }
+
+  /* -------------------------------------------------- */
+  /* INFO                                               */
+  /* -------------------------------------------------- */
+
+  get size(): number {
+    return this.workerCount;
+  }
+}

package/src/ipc/queue.ts

@@ -1,5 +1,6 @@
-import { IPCClient } from "./client.js";
+import { LioranManager } from "../LioranManager.js";
 import { getDefaultRootPath } from "../utils/rootpath.js";
+import { IPCWorkerPool } from "./pool.js";
 
 /* -------------------------------- ACTION TYPES -------------------------------- */
 
@@ -11,57 +12,110 @@ export type IPCAction =
   | "compact:db"
   | "compact:all"
   | "shutdown"
-  | "open"
-  | "close"
-  | "minimize"
-  | "maximize"
   | "restore"
   | "snapshot";
 
 /* -------------------------------- DB QUEUE -------------------------------- */
 
 export class DBQueue {
-  private client: IPCClient;
+  private manager: LioranManager;
+  private pool: IPCWorkerPool;
+  private destroyed = false;
 
-  constructor(rootPath = getDefaultRootPath()) {
-    this.client = new IPCClient(rootPath);
-  }
+  constructor(private rootPath = getDefaultRootPath()) {
+    // Single shared DB instance
+    this.manager = new LioranManager({ rootPath });
 
-  exec(action: IPCAction, args: any) {
-    return this.client.exec(action, args);
+    // Worker threads (for future compute-heavy tasks)
+    this.pool = new IPCWorkerPool();
+    this.pool.start();
   }
 
-  /* ----------------------------- COMPACTION API ----------------------------- */
+  /* -------------------------------- EXEC -------------------------------- */
 
-  compactCollection(db: string, col: string) {
-    return this.exec("compact:collection", { db, col });
-  }
+  async exec(action: IPCAction, args: any) {
+    if (this.destroyed) {
+      throw new Error("DBQueue already shutdown");
+    }
 
-  compactDB(db: string) {
-    return this.exec("compact:db", { db });
-  }
+    switch (action) {
+      /* ---------------- DB ---------------- */
 
-  compactAll() {
-    return this.exec("compact:all", {});
-  }
+      case "db":
+        await this.manager.db(args.db);
+        return true;
 
-  /* ----------------------------- SNAPSHOT API ----------------------------- */
+      /* ---------------- CRUD OPS ---------------- */
 
-  snapshot(path: string) {
-    return this.exec("snapshot", { path });
-  }
+      case "op": {
+        const { db, col, method, params } = args;
+        const collection = (await this.manager.db(db)).collection(col);
+        return await (collection as any)[method](...params);
+      }
+
+      /* ---------------- INDEX OPS ---------------- */
+
+      case "index": {
+        const { db, col, method, params } = args;
+        const collection = (await this.manager.db(db)).collection(col);
+        return await (collection as any)[method](...params);
+      }
+
+      /* ---------------- COMPACTION ---------------- */
+
+      case "compact:collection": {
+        const { db, col } = args;
+        const collection = (await this.manager.db(db)).collection(col);
+        await collection.compact();
+        return true;
+      }
 
-  restore(path: string) {
-    return this.exec("restore", { path });
+      case "compact:db": {
+        const { db } = args;
+        const database = await this.manager.db(db);
+        await database.compactAll();
+        return true;
+      }
+
+      case "compact:all": {
+        for (const db of this.manager.openDBs.values()) {
+          await db.compactAll();
+        }
+        return true;
+      }
+
+      /* ---------------- SNAPSHOT ---------------- */
+
+      case "snapshot":
+        await this.manager.snapshot(args.path);
+        return true;
+
+      case "restore":
+        await this.manager.restore(args.path);
+        return true;
+
+      /* ---------------- CONTROL ---------------- */
+
+      case "shutdown":
+        await this.shutdown();
+        return true;
+
+      default:
+        throw new Error(`Unknown action: ${action}`);
+    }
   }
 
   /* ------------------------------ SHUTDOWN ------------------------------ */
 
   async shutdown() {
-    try {
-      await this.exec("shutdown", {});
-    } catch {}
-    this.client.close();
+    if (this.destroyed) return;
+    this.destroyed = true;
+
+    // Close DBs
+    await this.manager.closeAll();
+
+    // Shutdown worker threads
+    await this.pool.shutdown();
   }
 }