@liorandb/core 1.0.18 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@liorandb/core",
-  "version": "1.0.18",
+  "version": "1.1.0",
   "description": "LioranDB Core Module – Lightweight, local-first, peer-to-peer database management for Node.js.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/core/checkpoint.ts

@@ -0,0 +1,163 @@
+import fs from "fs";
+import path from "path";
+
+/* =========================
+   TYPES
+========================= */
+
+export interface CheckpointData {
+  lsn: number;        // Last durable LSN
+  walGen: number;     // WAL generation at checkpoint
+  time: number;       // Timestamp (ms)
+  version: number;    // Format version
+}
+
+interface StoredCheckpoint {
+  data: CheckpointData;
+  crc: number;
+}
+
+/* =========================
+   CONSTANTS
+========================= */
+
+const CHECKPOINT_A = "__checkpoint_A.json";
+const CHECKPOINT_B = "__checkpoint_B.json";
+const FORMAT_VERSION = 1;
+
+/* =========================
+   CRC32 (no deps)
+========================= */
+
+const CRC32_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let k = 0; k < 8; k++) {
+      c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+    }
+    table[i] = c >>> 0;
+  }
+  return table;
+})();
+
+function crc32(input: string): number {
+  let crc = 0xFFFFFFFF;
+  for (let i = 0; i < input.length; i++) {
+    crc = CRC32_TABLE[(crc ^ input.charCodeAt(i)) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+/* =========================
+   CHECKPOINT MANAGER
+========================= */
+
+export class CheckpointManager {
+  private baseDir: string;
+  private data: CheckpointData;
+
+  constructor(baseDir: string) {
+    this.baseDir = baseDir;
+    this.data = {
+      lsn: 0,
+      walGen: 1,
+      time: 0,
+      version: FORMAT_VERSION
+    };
+
+    this.load();
+  }
+
+  /* -------------------------
+     LOAD (CRC + FALLBACK)
+  ------------------------- */
+
+  private load() {
+    const a = this.readCheckpoint(CHECKPOINT_A);
+    const b = this.readCheckpoint(CHECKPOINT_B);
+
+    if (a && b) {
+      // pick newest valid checkpoint
+      this.data = a.data.lsn >= b.data.lsn ? a.data : b.data;
+      return;
+    }
+
+    if (a) {
+      this.data = a.data;
+      return;
+    }
+
+    if (b) {
+      this.data = b.data;
+      return;
+    }
+
+    console.warn("No valid checkpoint found, starting from zero");
+  }
+
+  private readCheckpoint(file: string): StoredCheckpoint | null {
+    const filePath = path.join(this.baseDir, file);
+    if (!fs.existsSync(filePath)) return null;
+
+    try {
+      const raw = fs.readFileSync(filePath, "utf8");
+      const parsed = JSON.parse(raw) as StoredCheckpoint;
+
+      if (!parsed?.data || typeof parsed.crc !== "number") {
+        return null;
+      }
+
+      const expected = crc32(JSON.stringify(parsed.data));
+      if (expected !== parsed.crc) {
+        console.error(`Checkpoint CRC mismatch: ${file}`);
+        return null;
+      }
+
+      return parsed;
+    } catch {
+      return null;
+    }
+  }
+
+  /* -------------------------
+     SAVE (DUAL WRITE)
+  ------------------------- */
+
+  save(lsn: number, walGen: number) {
+    const data: CheckpointData = {
+      lsn,
+      walGen,
+      time: Date.now(),
+      version: FORMAT_VERSION
+    };
+
+    const stored: StoredCheckpoint = {
+      data,
+      crc: crc32(JSON.stringify(data))
+    };
+
+    // alternate between A/B for crash safety
+    const target =
+      lsn % 2 === 0 ? CHECKPOINT_A : CHECKPOINT_B;
+
+    try {
+      fs.writeFileSync(
+        path.join(this.baseDir, target),
+        JSON.stringify(stored, null, 2),
+        "utf8"
+      );
+      this.data = data;
+    } catch (err) {
+      console.error("Failed to write checkpoint:", err);
+    }
+  }
+
+  /* -------------------------
+     GET CURRENT
+  ------------------------- */
+
+  get(): CheckpointData {
+    return this.data;
+  }
+}

package/src/core/compaction.ts

@@ -5,117 +5,179 @@ import { Collection } from "./collection.js";
 import { Index } from "./index.js";
 import { decryptData } from "../utils/encryption.js";
 
+/* ---------------------------------------------------------
+   CONSTANTS
+--------------------------------------------------------- */
+
 const TMP_SUFFIX = "__compact_tmp";
-const OLD_SUFFIX = "__old";
+const OLD_SUFFIX = "__compact_old";
+const INDEX_DIR = "__indexes";
+
+/* ---------------------------------------------------------
+   PUBLIC ENTRY
+--------------------------------------------------------- */
 
 /**
- * Entry point: safe compaction wrapper
+ * Full production-safe compaction:
+ * 1. Crash recovery
+ * 2. Snapshot rebuild
+ * 3. Atomic swap
+ * 4. Reopen DB
+ * 5. Rebuild indexes
  */
 export async function compactCollectionEngine(col: Collection) {
-  await crashRecovery(col.dir);
-
   const baseDir = col.dir;
   const tmpDir = baseDir + TMP_SUFFIX;
   const oldDir = baseDir + OLD_SUFFIX;
 
-  // Cleanup stale dirs
+  await crashRecovery(baseDir);
+
   safeRemove(tmpDir);
   safeRemove(oldDir);
 
-  // Snapshot rebuild
   await snapshotRebuild(col, tmpDir);
 
-  // Atomic swap
-  atomicSwap(baseDir, tmpDir, oldDir);
+  await atomicSwap(baseDir, tmpDir, oldDir);
 
-  // Cleanup old data
   safeRemove(oldDir);
+
+  // Reopen DB after swap
+  await reopenCollectionDB(col);
+
+  // Rebuild indexes after compaction
+  await rebuildIndexes(col);
 }
 
-/**
- * Copies only live keys into fresh DB
- */
+/* ---------------------------------------------------------
+   SNAPSHOT REBUILD
+--------------------------------------------------------- */
+
 async function snapshotRebuild(col: Collection, tmpDir: string) {
   fs.mkdirSync(tmpDir, { recursive: true });
 
-  const tmpDB = new ClassicLevel(tmpDir, { valueEncoding: "utf8" });
+  const tmpDB = new ClassicLevel(tmpDir, {
+    valueEncoding: "utf8"
+  });
 
   for await (const [key, val] of col.db.iterator()) {
-    await tmpDB.put(key, val);
+    if (val !== undefined) {
+      await tmpDB.put(key, val);
+    }
   }
 
   await tmpDB.close();
-  await col.db.close();
+  await col.db.close(); // important: close before swap
 }
 
-/**
- * Atomic directory replace
- */
-function atomicSwap(base: string, tmp: string, old: string) {
+/* ---------------------------------------------------------
+   ATOMIC SWAP (HARDENED)
+--------------------------------------------------------- */
+
+async function atomicSwap(base: string, tmp: string, old: string) {
+  // Phase 1: rename base → old
   fs.renameSync(base, old);
-  fs.renameSync(tmp, base);
+
+  try {
+    // Phase 2: rename tmp → base
+    fs.renameSync(tmp, base);
+  } catch (err) {
+    // Rollback if tmp rename fails
+    if (fs.existsSync(old)) {
+      fs.renameSync(old, base);
+    }
+    throw err;
+  }
 }
 
-/**
- * Crash recovery handler
- */
+/* ---------------------------------------------------------
+   CRASH RECOVERY
+--------------------------------------------------------- */
+
 export async function crashRecovery(baseDir: string) {
   const tmp = baseDir + TMP_SUFFIX;
   const old = baseDir + OLD_SUFFIX;
 
-  // If both exist → compaction mid-swap
-  if (fs.existsSync(tmp) && fs.existsSync(old)) {
+  const baseExists = fs.existsSync(baseDir);
+  const tmpExists = fs.existsSync(tmp);
+  const oldExists = fs.existsSync(old);
+
+  // Case 1: swap interrupted → tmp is valid snapshot
+  if (tmpExists && oldExists) {
     safeRemove(baseDir);
     fs.renameSync(tmp, baseDir);
     safeRemove(old);
+    return;
   }
 
-  // If only old exists → swap incomplete
-  if (fs.existsSync(old) && !fs.existsSync(baseDir)) {
+  // Case 2: base→old happened but tmp missing
+  if (!baseExists && oldExists) {
     fs.renameSync(old, baseDir);
+    return;
   }
 
-  // If only tmp exists → rebuild incomplete
-  if (fs.existsSync(tmp) && !fs.existsSync(old)) {
+  // Case 3: rebuild interrupted
+  if (tmpExists && !oldExists) {
     safeRemove(tmp);
   }
 }
 
-/**
- * Index rebuild engine
- */
+/* ---------------------------------------------------------
+   REOPEN DB
+--------------------------------------------------------- */
+
+async function reopenCollectionDB(col: Collection) {
+  col.db = new ClassicLevel(col.dir, {
+    valueEncoding: "utf8"
+  });
+}
+
+/* ---------------------------------------------------------
+   INDEX REBUILD (SAFE)
+--------------------------------------------------------- */
+
 export async function rebuildIndexes(col: Collection) {
-  const indexRoot = path.join(col.dir, "__indexes");
+  const indexRoot = path.join(col.dir, INDEX_DIR);
 
-  // Destroy existing indexes
-  safeRemove(indexRoot);
-  fs.mkdirSync(indexRoot, { recursive: true });
+  const oldIndexes = new Map(col["indexes"]);
 
-  for (const idx of col["indexes"].values()) {
-    try { await idx.close(); } catch {}
+  // Close old index handles
+  for (const idx of oldIndexes.values()) {
+    try {
+      await idx.close();
+    } catch {}
   }
 
-  const newIndexes = new Map<string, Index>();
+  safeRemove(indexRoot);
+  fs.mkdirSync(indexRoot, { recursive: true });
+
+  const rebuiltIndexes = new Map<string, Index>();
 
-  for (const idx of col["indexes"].values()) {
-    const fresh = new Index(col.dir, idx.field, {
+  for (const idx of oldIndexes.values()) {
+    const rebuilt = new Index(col.dir, idx.field, {
       unique: idx.unique
     });
 
     for await (const [, enc] of col.db.iterator()) {
-      const doc = decryptData(enc);
-      await fresh.insert(doc);
+      if (!enc) continue;
+
+      try {
+        const doc = decryptData(enc);
+        await rebuilt.insert(doc);
+      } catch {
+        // Skip corrupted doc safely
+      }
     }
 
-    newIndexes.set(idx.field, fresh);
+    rebuiltIndexes.set(idx.field, rebuilt);
   }
 
-  col["indexes"] = newIndexes;
+  col["indexes"] = rebuiltIndexes;
 }
 
-/**
- * Safe recursive remove
- */
+/* ---------------------------------------------------------
+   UTIL
+--------------------------------------------------------- */
+
 function safeRemove(p: string) {
   if (fs.existsSync(p)) {
     fs.rmSync(p, { recursive: true, force: true });

package/src/core/database.ts

@@ -1,7 +1,5 @@
 import path from "path";
 import fs from "fs";
-import { execFile } from "child_process";
-import { promisify } from "util";
 import { Collection } from "./collection.js";
 import { Index, IndexOptions } from "./index.js";
 import { MigrationEngine } from "./migration.js";
@@ -9,14 +7,12 @@ import type { LioranManager } from "../LioranManager.js";
 import type { ZodSchema } from "zod";
 import { decryptData } from "../utils/encryption.js";
 
-const exec = promisify(execFile);
+import { WALManager } from "./wal.js";
+import { CheckpointManager } from "./checkpoint.js";
 
 /* ----------------------------- TYPES ----------------------------- */
 
 type TXOp = { tx: number; col: string; op: string; args: any[] };
-type TXCommit = { tx: number; commit: true };
-type TXApplied = { tx: number; applied: true };
-type WALEntry = TXOp | TXCommit | TXApplied;
 
 type IndexMeta = {
   field: string;
@@ -26,11 +22,11 @@ type IndexMeta = {
 type DBMeta = {
   version: number;
   indexes: Record<string, IndexMeta[]>;
-  schemaVersion: string; // DB-level schema (not collection schema)
+  schemaVersion: string;
 };
 
 const META_FILE = "__db_meta.json";
-const META_VERSION = 1;
+const META_VERSION = 2;
 const DEFAULT_SCHEMA_VERSION = "v1";
 
 /* ---------------------- TRANSACTION CONTEXT ---------------------- */
@@ -59,11 +55,34 @@ class DBTransactionContext {
   }
 
   async commit() {
-    await this.db.writeWAL(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, commit: true }]);
+    // 1️⃣ Write all operations
+    for (const op of this.ops) {
+      await this.db.wal.append({
+        tx: this.txId,
+        type: "op",
+        payload: op
+      } as any);
+    }
+
+    // 2️⃣ Commit marker
+    const commitLSN = await this.db.wal.append({
+      tx: this.txId,
+      type: "commit"
+    } as any);
+
+    // 3️⃣ Apply to storage
     await this.db.applyTransaction(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, applied: true }]);
-    await this.db.clearWAL();
+
+    // 4️⃣ Applied marker
+    const appliedLSN = await this.db.wal.append({
+      tx: this.txId,
+      type: "applied"
+    } as any);
+
+    // 5️⃣ Advance checkpoint to durable applied LSN
+    this.db.advanceCheckpoint(appliedLSN);
+
+    await this.db.postCommitMaintenance();
   }
 }
 
@@ -75,28 +94,87 @@ export class LioranDB {
   manager: LioranManager;
   collections: Map<string, Collection>;
 
-  private walPath: string;
   private metaPath: string;
   private meta!: DBMeta;
 
   private migrator: MigrationEngine;
   private static TX_SEQ = 0;
 
+  public wal: WALManager;
+  private checkpoint: CheckpointManager;
+
   constructor(basePath: string, dbName: string, manager: LioranManager) {
     this.basePath = basePath;
     this.dbName = dbName;
     this.manager = manager;
     this.collections = new Map();
 
-    this.walPath = path.join(basePath, "__tx_wal.log");
     this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
     this.loadMeta();
+
+    this.wal = new WALManager(basePath);
+    this.checkpoint = new CheckpointManager(basePath);
+
     this.migrator = new MigrationEngine(this);
 
-    this.recoverFromWAL().catch(console.error);
+    this.initialize().catch(console.error);
+  }
+
+  /* ------------------------- INIT & RECOVERY ------------------------- */
+
+  private async initialize() {
+    await this.recoverFromWAL();
+  }
+
+  private async recoverFromWAL() {
+    const checkpointData = this.checkpoint.get();
+    const fromLSN = checkpointData.lsn;
+
+    const committed = new Set<number>();
+    const applied = new Set<number>();
+    const ops = new Map<number, TXOp[]>();
+
+    await this.wal.replay(fromLSN, async (record) => {
+      if (record.type === "commit") {
+        committed.add(record.tx);
+      } else if (record.type === "applied") {
+        applied.add(record.tx);
+      } else if (record.type === "op") {
+        if (!ops.has(record.tx)) ops.set(record.tx, []);
+        ops.get(record.tx)!.push(record.payload as TXOp);
+      }
+    });
+
+    let highestAppliedLSN = fromLSN;
+
+    for (const tx of committed) {
+      if (applied.has(tx)) continue;
+
+      const txOps = ops.get(tx);
+      if (txOps) {
+        await this.applyTransaction(txOps);
+        highestAppliedLSN = this.wal.getCurrentLSN();
+      }
+    }
+
+    // Advance checkpoint after recovery
+    this.advanceCheckpoint(highestAppliedLSN);
+  }
+
+  /* ------------------------- CHECKPOINT ADVANCE ------------------------- */
+
+  public advanceCheckpoint(lsn: number) {
+    const current = this.checkpoint.get();
+
+    if (lsn > current.lsn) {
+      this.checkpoint.save(lsn, this.wal.getCurrentGen());
+
+      // Optional WAL cleanup (safe because checkpoint advanced)
+      this.wal.cleanup(this.wal.getCurrentGen() - 1).catch(() => {});
+    }
   }
 
   /* ------------------------- META ------------------------- */
@@ -146,50 +224,7 @@ export class LioranDB {
     await this.migrator.upgradeToLatest();
   }
 
-  /* ------------------------- WAL ------------------------- */
-
-  async writeWAL(entries: WALEntry[]) {
-    const fd = await fs.promises.open(this.walPath, "a");
-    for (const e of entries) {
-      await fd.write(JSON.stringify(e) + "\n");
-    }
-    await fd.sync();
-    await fd.close();
-  }
-
-  async clearWAL() {
-    try { await fs.promises.unlink(this.walPath); } catch {}
-  }
-
-  private async recoverFromWAL() {
-    if (!fs.existsSync(this.walPath)) return;
-
-    const raw = await fs.promises.readFile(this.walPath, "utf8");
-
-    const committed = new Set<number>();
-    const applied = new Set<number>();
-    const ops = new Map<number, TXOp[]>();
-
-    for (const line of raw.split("\n")) {
-      if (!line.trim()) continue;
-      const entry: WALEntry = JSON.parse(line);
-
-      if ("commit" in entry) committed.add(entry.tx);
-      else if ("applied" in entry) applied.add(entry.tx);
-      else {
-        if (!ops.has(entry.tx)) ops.set(entry.tx, []);
-        ops.get(entry.tx)!.push(entry);
-      }
-    }
-
-    for (const tx of committed) {
-      if (applied.has(tx)) continue;
-      const txOps = ops.get(tx);
-      if (txOps) await this.applyTransaction(txOps);
-    }
-
-    await this.clearWAL();
-  }
+  /* ------------------------- TX APPLY ------------------------- */
 
   async applyTransaction(ops: TXOp[]) {
     for (const { col, op, args } of ops) {
@@ -269,13 +304,11 @@ export class LioranDB {
   /* ------------------------- COMPACTION ------------------------- */
 
   async compactCollection(name: string) {
-    await this.clearWAL();
     const col = this.collection(name);
     await col.compact();
   }
 
   async compactAll() {
-    await this.clearWAL();
     for (const name of this.collections.keys()) {
       await this.compactCollection(name);
     }
@@ -291,12 +324,19 @@ export class LioranDB {
     return result;
   }
 
+  /* ------------------------- POST COMMIT ------------------------- */
+
+  public async postCommitMaintenance() {
+    // Hook for background compaction, stats, etc.
+  }
+
   /* ------------------------- SHUTDOWN ------------------------- */
 
   async close(): Promise<void> {
     for (const col of this.collections.values()) {
       try { await col.close(); } catch {}
     }
+
     this.collections.clear();
   }
 }