@liorandb/core 1.0.18 → 1.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@liorandb/core",
-  "version": "1.0.18",
+  "version": "1.0.19",
   "description": "LioranDB Core Module – Lightweight, local-first, peer-to-peer database management for Node.js.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/core/checkpoint.ts

@@ -0,0 +1,111 @@
+import fs from "fs";
+import path from "path";
+
+/* =========================
+   TYPES
+========================= */
+
+export interface CheckpointData {
+  lsn: number;        // Last durable LSN
+  walGen: number;     // WAL generation at checkpoint
+  time: number;       // Timestamp (ms)
+  version: number;    // For future format upgrades
+}
+
+/* =========================
+   CONSTANTS
+========================= */
+
+const CHECKPOINT_FILE = "__checkpoint.json";
+const TMP_SUFFIX = ".tmp";
+const FORMAT_VERSION = 1;
+
+/* =========================
+   CHECKPOINT MANAGER
+========================= */
+
+export class CheckpointManager {
+  private filePath: string;
+  private data: CheckpointData;
+
+  constructor(baseDir: string) {
+    this.filePath = path.join(baseDir, CHECKPOINT_FILE);
+    this.data = {
+      lsn: 0,
+      walGen: 1,
+      time: 0,
+      version: FORMAT_VERSION
+    };
+
+    this.load();
+  }
+
+  /* -------------------------
+     LOAD (Crash-safe)
+  ------------------------- */
+
+  private load() {
+    if (!fs.existsSync(this.filePath)) {
+      return;
+    }
+
+    try {
+      const raw = fs.readFileSync(this.filePath, "utf8");
+      const parsed = JSON.parse(raw) as CheckpointData;
+
+      if (
+        typeof parsed.lsn === "number" &&
+        typeof parsed.walGen === "number"
+      ) {
+        this.data = parsed;
+      }
+    } catch {
+      console.error("Checkpoint corrupted, starting from zero");
+      this.data = {
+        lsn: 0,
+        walGen: 1,
+        time: 0,
+        version: FORMAT_VERSION
+      };
+    }
+  }
+
+  /* -------------------------
+     SAVE (Atomic Write)
+  ------------------------- */
+
+  save(lsn: number, walGen: number) {
+    const newData: CheckpointData = {
+      lsn,
+      walGen,
+      time: Date.now(),
+      version: FORMAT_VERSION
+    };
+
+    const tmpPath = this.filePath + TMP_SUFFIX;
+
+    try {
+      // Write to temp file first
+      fs.writeFileSync(
+        tmpPath,
+        JSON.stringify(newData, null, 2),
+        { encoding: "utf8" }
+      );
+
+      // Atomic rename
+      fs.renameSync(tmpPath, this.filePath);
+
+      this.data = newData;
+    } catch (err) {
+      console.error("Failed to write checkpoint:", err);
+    }
+  }
+
+  /* -------------------------
+     GET CURRENT
+  ------------------------- */
+
+  get(): CheckpointData {
+    return this.data;
+  }
+}

package/src/core/compaction.ts

@@ -5,117 +5,164 @@ import { Collection } from "./collection.js";
 import { Index } from "./index.js";
 import { decryptData } from "../utils/encryption.js";
 
+/* ---------------------------------------------------------
+   CONSTANTS
+--------------------------------------------------------- */
+
 const TMP_SUFFIX = "__compact_tmp";
-const OLD_SUFFIX = "__old";
+const OLD_SUFFIX = "__compact_old";
+const INDEX_DIR = "__indexes";
+
+/* ---------------------------------------------------------
+   PUBLIC ENTRY
+--------------------------------------------------------- */
 
 /**
- * Entry point: safe compaction wrapper
+ * Full safe compaction pipeline:
+ * 1. Crash recovery
+ * 2. Snapshot rebuild
+ * 3. Atomic directory swap
+ * 4. Index rebuild
  */
 export async function compactCollectionEngine(col: Collection) {
-  await crashRecovery(col.dir);
-
   const baseDir = col.dir;
   const tmpDir = baseDir + TMP_SUFFIX;
   const oldDir = baseDir + OLD_SUFFIX;
 
-  // Cleanup stale dirs
+  // Recover from any previous crash mid-compaction
+  await crashRecovery(baseDir);
+
+  // Clean leftovers (paranoia safety)
   safeRemove(tmpDir);
   safeRemove(oldDir);
 
-  // Snapshot rebuild
+  // Step 1: rebuild snapshot
   await snapshotRebuild(col, tmpDir);
 
-  // Atomic swap
+  // Step 2: atomic swap
   atomicSwap(baseDir, tmpDir, oldDir);
 
-  // Cleanup old data
+  // Cleanup
   safeRemove(oldDir);
 }
 
+/* ---------------------------------------------------------
+   SNAPSHOT REBUILD
+--------------------------------------------------------- */
+
 /**
- * Copies only live keys into fresh DB
+ * Rebuilds DB by copying only live keys
+ * WAL is assumed already checkpointed
  */
 async function snapshotRebuild(col: Collection, tmpDir: string) {
   fs.mkdirSync(tmpDir, { recursive: true });
 
-  const tmpDB = new ClassicLevel(tmpDir, { valueEncoding: "utf8" });
+  const tmpDB = new ClassicLevel(tmpDir, {
+    valueEncoding: "utf8"
+  });
 
   for await (const [key, val] of col.db.iterator()) {
-    await tmpDB.put(key, val);
+    if (val !== undefined) {
+      await tmpDB.put(key, val);
+    }
   }
 
   await tmpDB.close();
   await col.db.close();
 }
 
+/* ---------------------------------------------------------
+   ATOMIC SWAP
+--------------------------------------------------------- */
+
 /**
- * Atomic directory replace
+ * Atomic directory replacement (POSIX safe)
  */
 function atomicSwap(base: string, tmp: string, old: string) {
   fs.renameSync(base, old);
   fs.renameSync(tmp, base);
 }
 
+/* ---------------------------------------------------------
+   CRASH RECOVERY
+--------------------------------------------------------- */
+
 /**
- * Crash recovery handler
+ * Handles all partial-compaction states
  */
 export async function crashRecovery(baseDir: string) {
   const tmp = baseDir + TMP_SUFFIX;
   const old = baseDir + OLD_SUFFIX;
 
-  // If both exist → compaction mid-swap
-  if (fs.existsSync(tmp) && fs.existsSync(old)) {
+  const baseExists = fs.existsSync(baseDir);
+  const tmpExists = fs.existsSync(tmp);
+  const oldExists = fs.existsSync(old);
+
+  // Case 1: swap interrupted → tmp is valid snapshot
+  if (tmpExists && oldExists) {
     safeRemove(baseDir);
     fs.renameSync(tmp, baseDir);
     safeRemove(old);
+    return;
   }
 
-  // If only old exists → swap incomplete
-  if (fs.existsSync(old) && !fs.existsSync(baseDir)) {
+  // Case 2: rename(base → old) happened, but tmp missing
+  if (!baseExists && oldExists) {
     fs.renameSync(old, baseDir);
+    return;
   }
 
-  // If only tmp exists → rebuild incomplete
-  if (fs.existsSync(tmp) && !fs.existsSync(old)) {
+  // Case 3: rebuild interrupted
+  if (tmpExists && !oldExists) {
     safeRemove(tmp);
   }
 }
 
+/* ---------------------------------------------------------
+   INDEX REBUILD
+--------------------------------------------------------- */
+
 /**
- * Index rebuild engine
+ * Rebuilds all indexes from compacted DB
+ * Guarantees index consistency
  */
 export async function rebuildIndexes(col: Collection) {
-  const indexRoot = path.join(col.dir, "__indexes");
-
-  // Destroy existing indexes
-  safeRemove(indexRoot);
-  fs.mkdirSync(indexRoot, { recursive: true });
+  const indexRoot = path.join(col.dir, INDEX_DIR);
 
+  // Close existing index handles
   for (const idx of col["indexes"].values()) {
-    try { await idx.close(); } catch {}
+    try {
+      await idx.close();
+    } catch {}
   }
 
+  // Destroy index directory
+  safeRemove(indexRoot);
+  fs.mkdirSync(indexRoot, { recursive: true });
+
   const newIndexes = new Map<string, Index>();
 
   for (const idx of col["indexes"].values()) {
-    const fresh = new Index(col.dir, idx.field, {
+    const rebuilt = new Index(col.dir, idx.field, {
       unique: idx.unique
     });
 
     for await (const [, enc] of col.db.iterator()) {
+      if (!enc) continue;
       const doc = decryptData(enc);
-      await fresh.insert(doc);
+      await rebuilt.insert(doc);
     }
 
-    newIndexes.set(idx.field, fresh);
+    newIndexes.set(idx.field, rebuilt);
   }
 
   col["indexes"] = newIndexes;
 }
 
-/**
- * Safe recursive remove
- */
+/* ---------------------------------------------------------
+   UTIL
+--------------------------------------------------------- */
+
 function safeRemove(p: string) {
   if (fs.existsSync(p)) {
     fs.rmSync(p, { recursive: true, force: true });

package/src/core/database.ts

@@ -1,7 +1,5 @@
 import path from "path";
 import fs from "fs";
-import { execFile } from "child_process";
-import { promisify } from "util";
 import { Collection } from "./collection.js";
 import { Index, IndexOptions } from "./index.js";
 import { MigrationEngine } from "./migration.js";
@@ -9,7 +7,8 @@ import type { LioranManager } from "../LioranManager.js";
 import type { ZodSchema } from "zod";
 import { decryptData } from "../utils/encryption.js";
 
-const exec = promisify(execFile);
+import { WALManager } from "./wal.js";
+import { CheckpointManager } from "./checkpoint.js";
 
 /* ----------------------------- TYPES ----------------------------- */
 
@@ -26,11 +25,11 @@ type IndexMeta = {
 type DBMeta = {
   version: number;
   indexes: Record<string, IndexMeta[]>;
-  schemaVersion: string; // DB-level schema (not collection schema)
+  schemaVersion: string;
 };
 
 const META_FILE = "__db_meta.json";
-const META_VERSION = 1;
+const META_VERSION = 2;
 const DEFAULT_SCHEMA_VERSION = "v1";
 
 /* ---------------------- TRANSACTION CONTEXT ---------------------- */
@@ -59,11 +58,30 @@ class DBTransactionContext {
   }
 
   async commit() {
-    await this.db.writeWAL(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, commit: true }]);
+    for (const op of this.ops) {
+      const recordOp: any = {
+        tx: this.txId,
+        type: "op",
+        payload: op
+      };
+      await this.db.wal.append(recordOp);
+    }
+
+    const commitRecord: any = {
+      tx: this.txId,
+      type: "commit"
+    };
+    await this.db.wal.append(commitRecord);
+
     await this.db.applyTransaction(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, applied: true }]);
-    await this.db.clearWAL();
+
+    const appliedRecord: any = {
+      tx: this.txId,
+      type: "applied"
+    };
+    await this.db.wal.append(appliedRecord);
+
+    await this.db.postCommitMaintenance();
   }
 }
 
@@ -75,28 +93,68 @@ export class LioranDB {
   manager: LioranManager;
   collections: Map<string, Collection>;
 
-  private walPath: string;
   private metaPath: string;
   private meta!: DBMeta;
 
   private migrator: MigrationEngine;
   private static TX_SEQ = 0;
 
+  public wal: WALManager;
+  private checkpoint: CheckpointManager;
+
   constructor(basePath: string, dbName: string, manager: LioranManager) {
     this.basePath = basePath;
     this.dbName = dbName;
     this.manager = manager;
     this.collections = new Map();
 
-    this.walPath = path.join(basePath, "__tx_wal.log");
     this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
     this.loadMeta();
+
+    this.wal = new WALManager(basePath);
+    this.checkpoint = new CheckpointManager(basePath);
+
     this.migrator = new MigrationEngine(this);
 
-    this.recoverFromWAL().catch(console.error);
+    this.initialize().catch(console.error);
+  }
+
+  /* ------------------------- INIT & RECOVERY ------------------------- */
+
+  private async initialize() {
+    await this.recoverFromWAL();
+  }
+
+  private async recoverFromWAL() {
+    const checkpointData = this.checkpoint.get();
+    const fromLSN = checkpointData.lsn;
+
+    const committed = new Set<number>();
+    const applied = new Set<number>();
+    const ops = new Map<number, TXOp[]>();
+
+    await this.wal.replay(fromLSN, async (record) => {
+      if (record.type === "commit") {
+        committed.add(record.tx);
+      } else if (record.type === "applied") {
+        applied.add(record.tx);
+      } else if (record.type === "op") {
+        if (!ops.has(record.tx)) ops.set(record.tx, []);
+        ops.get(record.tx)!.push(record.payload);
+      }
+    });
+
+    for (const tx of committed) {
+      if (applied.has(tx)) continue;
+
+      const txOps = ops.get(tx);
+      if (txOps) {
+        await this.applyTransaction(txOps);
+      }
+    }
   }
 
   /* ------------------------- META ------------------------- */
@@ -146,50 +204,7 @@ export class LioranDB {
     await this.migrator.upgradeToLatest();
   }
 
-  /* ------------------------- WAL ------------------------- */
-
-  async writeWAL(entries: WALEntry[]) {
-    const fd = await fs.promises.open(this.walPath, "a");
-    for (const e of entries) {
-      await fd.write(JSON.stringify(e) + "\n");
-    }
-    await fd.sync();
-    await fd.close();
-  }
-
-  async clearWAL() {
-    try { await fs.promises.unlink(this.walPath); } catch {}
-  }
-
-  private async recoverFromWAL() {
-    if (!fs.existsSync(this.walPath)) return;
-
-    const raw = await fs.promises.readFile(this.walPath, "utf8");
-
-    const committed = new Set<number>();
-    const applied = new Set<number>();
-    const ops = new Map<number, TXOp[]>();
-
-    for (const line of raw.split("\n")) {
-      if (!line.trim()) continue;
-      const entry: WALEntry = JSON.parse(line);
-
-      if ("commit" in entry) committed.add(entry.tx);
-      else if ("applied" in entry) applied.add(entry.tx);
-      else {
-        if (!ops.has(entry.tx)) ops.set(entry.tx, []);
-        ops.get(entry.tx)!.push(entry);
-      }
-    }
-
-    for (const tx of committed) {
-      if (applied.has(tx)) continue;
-      const txOps = ops.get(tx);
-      if (txOps) await this.applyTransaction(txOps);
-    }
-
-    await this.clearWAL();
-  }
+  /* ------------------------- TX APPLY ------------------------- */
 
   async applyTransaction(ops: TXOp[]) {
     for (const { col, op, args } of ops) {
@@ -269,13 +284,11 @@ export class LioranDB {
   /* ------------------------- COMPACTION ------------------------- */
 
   async compactCollection(name: string) {
-    await this.clearWAL();
     const col = this.collection(name);
     await col.compact();
   }
 
   async compactAll() {
-    await this.clearWAL();
     for (const name of this.collections.keys()) {
       await this.compactCollection(name);
     }
@@ -291,12 +304,19 @@ export class LioranDB {
     return result;
   }
 
+  /* ------------------------- POST COMMIT ------------------------- */
+
+  public async postCommitMaintenance() {
+    // Custom maintenance can be added here
+  }
+
   /* ------------------------- SHUTDOWN ------------------------- */
 
   async close(): Promise<void> {
     for (const col of this.collections.values()) {
       try { await col.close(); } catch {}
     }
+
     this.collections.clear();
   }
 }