@liorandb/core 1.0.17 → 1.0.19

package/src/core/compaction.ts

@@ -5,117 +5,164 @@ import { Collection } from "./collection.js";
 import { Index } from "./index.js";
 import { decryptData } from "../utils/encryption.js";
 
+/* ---------------------------------------------------------
+   CONSTANTS
+--------------------------------------------------------- */
+
 const TMP_SUFFIX = "__compact_tmp";
-const OLD_SUFFIX = "__old";
+const OLD_SUFFIX = "__compact_old";
+const INDEX_DIR = "__indexes";
+
+/* ---------------------------------------------------------
+   PUBLIC ENTRY
+--------------------------------------------------------- */
 
 /**
- * Entry point: safe compaction wrapper
+ * Full safe compaction pipeline:
+ * 1. Crash recovery
+ * 2. Snapshot rebuild
+ * 3. Atomic directory swap
+ * 4. Index rebuild
  */
 export async function compactCollectionEngine(col: Collection) {
-  await crashRecovery(col.dir);
-
   const baseDir = col.dir;
   const tmpDir = baseDir + TMP_SUFFIX;
   const oldDir = baseDir + OLD_SUFFIX;
 
-  // Cleanup stale dirs
+  // Recover from any previous crash mid-compaction
+  await crashRecovery(baseDir);
+
+  // Clean leftovers (paranoia safety)
   safeRemove(tmpDir);
   safeRemove(oldDir);
 
-  // Snapshot rebuild
+  // Step 1: rebuild snapshot
   await snapshotRebuild(col, tmpDir);
 
-  // Atomic swap
+  // Step 2: atomic swap
   atomicSwap(baseDir, tmpDir, oldDir);
 
-  // Cleanup old data
+  // Cleanup
   safeRemove(oldDir);
 }
 
+/* ---------------------------------------------------------
+   SNAPSHOT REBUILD
+--------------------------------------------------------- */
+
 /**
- * Copies only live keys into fresh DB
+ * Rebuilds DB by copying only live keys
+ * WAL is assumed already checkpointed
  */
 async function snapshotRebuild(col: Collection, tmpDir: string) {
   fs.mkdirSync(tmpDir, { recursive: true });
 
-  const tmpDB = new ClassicLevel(tmpDir, { valueEncoding: "utf8" });
+  const tmpDB = new ClassicLevel(tmpDir, {
+    valueEncoding: "utf8"
+  });
 
   for await (const [key, val] of col.db.iterator()) {
-    await tmpDB.put(key, val);
+    if (val !== undefined) {
+      await tmpDB.put(key, val);
+    }
   }
 
   await tmpDB.close();
   await col.db.close();
 }
 
+/* ---------------------------------------------------------
+   ATOMIC SWAP
+--------------------------------------------------------- */
+
 /**
- * Atomic directory replace
+ * Atomic directory replacement (POSIX safe)
  */
 function atomicSwap(base: string, tmp: string, old: string) {
   fs.renameSync(base, old);
   fs.renameSync(tmp, base);
 }
 
+/* ---------------------------------------------------------
+   CRASH RECOVERY
+--------------------------------------------------------- */
+
 /**
- * Crash recovery handler
+ * Handles all partial-compaction states
  */
 export async function crashRecovery(baseDir: string) {
   const tmp = baseDir + TMP_SUFFIX;
   const old = baseDir + OLD_SUFFIX;
 
-  // If both exist → compaction mid-swap
-  if (fs.existsSync(tmp) && fs.existsSync(old)) {
+  const baseExists = fs.existsSync(baseDir);
+  const tmpExists = fs.existsSync(tmp);
+  const oldExists = fs.existsSync(old);
+
+  // Case 1: swap interrupted → tmp is valid snapshot
+  if (tmpExists && oldExists) {
     safeRemove(baseDir);
     fs.renameSync(tmp, baseDir);
     safeRemove(old);
+    return;
   }
 
-  // If only old exists → swap incomplete
-  if (fs.existsSync(old) && !fs.existsSync(baseDir)) {
+  // Case 2: rename(base → old) happened, but tmp missing
+  if (!baseExists && oldExists) {
     fs.renameSync(old, baseDir);
+    return;
   }
 
-  // If only tmp exists → rebuild incomplete
-  if (fs.existsSync(tmp) && !fs.existsSync(old)) {
+  // Case 3: rebuild interrupted
+  if (tmpExists && !oldExists) {
     safeRemove(tmp);
   }
 }
 
+/* ---------------------------------------------------------
+   INDEX REBUILD
+--------------------------------------------------------- */
+
 /**
- * Index rebuild engine
+ * Rebuilds all indexes from compacted DB
+ * Guarantees index consistency
  */
 export async function rebuildIndexes(col: Collection) {
-  const indexRoot = path.join(col.dir, "__indexes");
-
-  // Destroy existing indexes
-  safeRemove(indexRoot);
-  fs.mkdirSync(indexRoot, { recursive: true });
+  const indexRoot = path.join(col.dir, INDEX_DIR);
 
+  // Close existing index handles
   for (const idx of col["indexes"].values()) {
-    try { await idx.close(); } catch {}
+    try {
+      await idx.close();
+    } catch {}
   }
 
+  // Destroy index directory
+  safeRemove(indexRoot);
+  fs.mkdirSync(indexRoot, { recursive: true });
+
   const newIndexes = new Map<string, Index>();
 
   for (const idx of col["indexes"].values()) {
-    const fresh = new Index(col.dir, idx.field, {
+    const rebuilt = new Index(col.dir, idx.field, {
       unique: idx.unique
     });
 
     for await (const [, enc] of col.db.iterator()) {
+      if (!enc) continue;
       const doc = decryptData(enc);
-      await fresh.insert(doc);
+      await rebuilt.insert(doc);
     }
 
-    newIndexes.set(idx.field, fresh);
+    newIndexes.set(idx.field, rebuilt);
   }
 
   col["indexes"] = newIndexes;
 }
 
-/**
- * Safe recursive remove
- */
+/* ---------------------------------------------------------
+   UTIL
+--------------------------------------------------------- */
+
 function safeRemove(p: string) {
   if (fs.existsSync(p)) {
     fs.rmSync(p, { recursive: true, force: true });

package/src/core/database.ts

@@ -1,14 +1,14 @@
 import path from "path";
 import fs from "fs";
-import { execFile } from "child_process";
-import { promisify } from "util";
 import { Collection } from "./collection.js";
 import { Index, IndexOptions } from "./index.js";
 import { MigrationEngine } from "./migration.js";
 import type { LioranManager } from "../LioranManager.js";
 import type { ZodSchema } from "zod";
+import { decryptData } from "../utils/encryption.js";
 
-const exec = promisify(execFile);
+import { WALManager } from "./wal.js";
+import { CheckpointManager } from "./checkpoint.js";
 
 /* ----------------------------- TYPES ----------------------------- */
 
@@ -29,7 +29,7 @@ type DBMeta = {
 };
 
 const META_FILE = "__db_meta.json";
-const META_VERSION = 1;
+const META_VERSION = 2;
 const DEFAULT_SCHEMA_VERSION = "v1";
 
 /* ---------------------- TRANSACTION CONTEXT ---------------------- */
@@ -40,7 +40,7 @@ class DBTransactionContext {
   constructor(
     private db: LioranDB,
     public readonly txId: number
-  ) { }
+  ) {}
 
   collection(name: string) {
     return new Proxy({}, {
@@ -58,11 +58,30 @@ class DBTransactionContext {
   }
 
   async commit() {
-    await this.db.writeWAL(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, commit: true }]);
+    for (const op of this.ops) {
+      const recordOp: any = {
+        tx: this.txId,
+        type: "op",
+        payload: op
+      };
+      await this.db.wal.append(recordOp);
+    }
+
+    const commitRecord: any = {
+      tx: this.txId,
+      type: "commit"
+    };
+    await this.db.wal.append(commitRecord);
+
     await this.db.applyTransaction(this.ops);
-    await this.db.writeWAL([{ tx: this.txId, applied: true }]);
-    await this.db.clearWAL();
+
+    const appliedRecord: any = {
+      tx: this.txId,
+      type: "applied"
+    };
+    await this.db.wal.append(appliedRecord);
+
+    await this.db.postCommitMaintenance();
   }
 }
 
@@ -74,29 +93,68 @@ export class LioranDB {
   manager: LioranManager;
   collections: Map<string, Collection>;
 
-  private walPath: string;
   private metaPath: string;
   private meta!: DBMeta;
 
   private migrator: MigrationEngine;
-
   private static TX_SEQ = 0;
 
+  public wal: WALManager;
+  private checkpoint: CheckpointManager;
+
   constructor(basePath: string, dbName: string, manager: LioranManager) {
     this.basePath = basePath;
     this.dbName = dbName;
     this.manager = manager;
     this.collections = new Map();
 
-    this.walPath = path.join(basePath, "__tx_wal.log");
     this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
     this.loadMeta();
+
+    this.wal = new WALManager(basePath);
+    this.checkpoint = new CheckpointManager(basePath);
+
     this.migrator = new MigrationEngine(this);
 
-    this.recoverFromWAL().catch(console.error);
+    this.initialize().catch(console.error);
+  }
+
+  /* ------------------------- INIT & RECOVERY ------------------------- */
+
+  private async initialize() {
+    await this.recoverFromWAL();
+  }
+
+  private async recoverFromWAL() {
+    const checkpointData = this.checkpoint.get();
+    const fromLSN = checkpointData.lsn;
+
+    const committed = new Set<number>();
+    const applied = new Set<number>();
+    const ops = new Map<number, TXOp[]>();
+
+    await this.wal.replay(fromLSN, async (record) => {
+      if (record.type === "commit") {
+        committed.add(record.tx);
+      } else if (record.type === "applied") {
+        applied.add(record.tx);
+      } else if (record.type === "op") {
+        if (!ops.has(record.tx)) ops.set(record.tx, []);
+        ops.get(record.tx)!.push(record.payload);
+      }
+    });
+
+    for (const tx of committed) {
+      if (applied.has(tx)) continue;
+
+      const txOps = ops.get(tx);
+      if (txOps) {
+        await this.applyTransaction(txOps);
+      }
+    }
   }
 
   /* ------------------------- META ------------------------- */
@@ -133,7 +191,7 @@ export class LioranDB {
     this.saveMeta();
   }
 
-  /* ------------------------- MIGRATION API ------------------------- */
+  /* ------------------------- DB MIGRATIONS ------------------------- */
 
   migrate(from: string, to: string, fn: (db: LioranDB) => Promise<void>) {
     this.migrator.register(from, to, async db => {
@@ -146,51 +204,7 @@ export class LioranDB {
     await this.migrator.upgradeToLatest();
   }
 
-  /* ------------------------- WAL ------------------------- */
-
-  async writeWAL(entries: WALEntry[]) {
-    const fd = await fs.promises.open(this.walPath, "a");
-    for (const e of entries) {
-      await fd.write(JSON.stringify(e) + "\n");
-    }
-    await fd.sync();
-    await fd.close();
-  }
-
-  async clearWAL() {
-    try { await fs.promises.unlink(this.walPath); } catch { }
-  }
-
-  private async recoverFromWAL() {
-    if (!fs.existsSync(this.walPath)) return;
-
-    const raw = await fs.promises.readFile(this.walPath, "utf8");
-
-    const committed = new Set<number>();
-    const applied = new Set<number>();
-    const ops = new Map<number, TXOp[]>();
-
-    for (const line of raw.split("\n")) {
-      if (!line.trim()) continue;
-
-      const entry: WALEntry = JSON.parse(line);
-
-      if ("commit" in entry) committed.add(entry.tx);
-      else if ("applied" in entry) applied.add(entry.tx);
-      else {
-        if (!ops.has(entry.tx)) ops.set(entry.tx, []);
-        ops.get(entry.tx)!.push(entry);
-      }
-    }
-
-    for (const tx of committed) {
-      if (applied.has(tx)) continue;
-      const txOps = ops.get(tx);
-      if (txOps) await this.applyTransaction(txOps);
-    }
-
-    await this.clearWAL();
-  }
+  /* ------------------------- TX APPLY ------------------------- */
 
   async applyTransaction(ops: TXOp[]) {
     for (const { col, op, args } of ops) {
@@ -201,17 +215,27 @@ export class LioranDB {
 
   /* ------------------------- COLLECTION ------------------------- */
 
-  collection<T = any>(name: string, schema?: ZodSchema<T>): Collection<T> {
+  collection<T = any>(
+    name: string,
+    schema?: ZodSchema<T>,
+    schemaVersion?: number
+  ): Collection<T> {
     if (this.collections.has(name)) {
       const col = this.collections.get(name)!;
-      if (schema) col.setSchema(schema);
+      if (schema && schemaVersion !== undefined) {
+        col.setSchema(schema, schemaVersion);
+      }
       return col as Collection<T>;
     }
 
     const colPath = path.join(this.basePath, name);
     fs.mkdirSync(colPath, { recursive: true });
 
-    const col = new Collection<T>(colPath, schema);
+    const col = new Collection<T>(
+      colPath,
+      schema,
+      schemaVersion ?? 1
+    );
 
     const metas = this.meta.indexes[name] ?? [];
     for (const m of metas) {
@@ -236,25 +260,14 @@ export class LioranDB {
 
     const index = new Index(col.dir, field, options);
 
-    // for await (const [, enc] of col.db.iterator()) {
-    //   // const doc = JSON.parse(
-    //   //   Buffer.from(enc, "base64").subarray(32).toString("utf8")
-    //   // );
-    //   const payload = Buffer.from(enc, "utf8").subarray(32);
-    //   const doc = JSON.parse(payload.toString("utf8"));
-    //   await index.insert(doc);
-    // }
-
     for await (const [key, enc] of col.db.iterator()) {
       if (!enc) continue;
-
       try {
-        const doc = decryptData(enc);           // ← this does base64 → AES-GCM → JSON
+        const doc = decryptData(enc);
         await index.insert(doc);
       } catch (err) {
-        const errorMessage = err instanceof Error ? err.message : String(err);
-        console.warn(`Could not decrypt document ${key} during index build: ${errorMessage}`);
-        // You can continue, or collect bad keys for later inspection
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`Index build skipped doc ${key}: ${msg}`);
       }
     }
 
@@ -271,13 +284,11 @@ export class LioranDB {
   /* ------------------------- COMPACTION ------------------------- */
 
   async compactCollection(name: string) {
-    await this.clearWAL();
     const col = this.collection(name);
     await col.compact();
   }
 
   async compactAll() {
-    await this.clearWAL();
     for (const name of this.collections.keys()) {
       await this.compactCollection(name);
     }
@@ -293,16 +304,19 @@ export class LioranDB {
     return result;
   }
 
+  /* ------------------------- POST COMMIT ------------------------- */
+
+  public async postCommitMaintenance() {
+    // Custom maintenance can be added here
+  }
+
   /* ------------------------- SHUTDOWN ------------------------- */
 
   async close(): Promise<void> {
     for (const col of this.collections.values()) {
-      try { await col.close(); } catch { }
+      try { await col.close(); } catch {}
     }
+
     this.collections.clear();
   }
-}
-
-function decryptData(enc: string) {
-  throw new Error("Function not implemented.");
-}
+}

package/src/core/wal.ts

@@ -0,0 +1,213 @@
+import fs from "fs";
+import path from "path";
+
+/* =========================
+   WAL RECORD TYPES
+========================= */
+
+export type WALRecord =
+  | { lsn: number; tx: number; type: "op"; payload: any }
+  | { lsn: number; tx: number; type: "commit" }
+  | { lsn: number; tx: number; type: "applied" };
+
+type StoredRecord = WALRecord & { crc: number };
+
+/* =========================
+   CONSTANTS
+========================= */
+
+const MAX_WAL_SIZE = 16 * 1024 * 1024; // 16 MB
+const WAL_DIR = "__wal";
+
+/* =========================
+   CRC32 IMPLEMENTATION
+   (no dependencies)
+========================= */
+
+const CRC32_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let k = 0; k < 8; k++) {
+      c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+    }
+    table[i] = c >>> 0;
+  }
+  return table;
+})();
+
+function crc32(input: string): number {
+  let crc = 0xFFFFFFFF;
+  for (let i = 0; i < input.length; i++) {
+    const byte = input.charCodeAt(i);
+    crc = CRC32_TABLE[(crc ^ byte) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+/* =========================
+   WAL MANAGER
+========================= */
+
+export class WALManager {
+  private walDir: string;
+  private currentGen = 1;
+  private lsn = 0;
+  private fd: fs.promises.FileHandle | null = null;
+
+  constructor(baseDir: string) {
+    this.walDir = path.join(baseDir, WAL_DIR);
+    fs.mkdirSync(this.walDir, { recursive: true });
+    this.currentGen = this.detectLastGeneration();
+  }
+
+  /* -------------------------
+     INTERNAL HELPERS
+  ------------------------- */
+
+  private walPath(gen = this.currentGen) {
+    return path.join(
+      this.walDir,
+      `wal-${String(gen).padStart(6, "0")}.log`
+    );
+  }
+
+  private detectLastGeneration(): number {
+    if (!fs.existsSync(this.walDir)) return 1;
+
+    const files = fs.readdirSync(this.walDir);
+    let max = 0;
+
+    for (const f of files) {
+      const m = f.match(/^wal-(\d+)\.log$/);
+      if (m) max = Math.max(max, Number(m[1]));
+    }
+
+    return max || 1;
+  }
+
+  private async open() {
+    if (!this.fd) {
+      this.fd = await fs.promises.open(this.walPath(), "a");
+    }
+  }
+
+  private async rotate() {
+    if (this.fd) {
+      await this.fd.close();
+      this.fd = null;
+    }
+    this.currentGen++;
+  }
+
+  /* -------------------------
+     APPEND
+  ------------------------- */
+
+  async append(record: Omit<WALRecord, "lsn">): Promise<number> {
+    await this.open();
+
+    const full: WALRecord = {
+      ...(record as any),
+      lsn: ++this.lsn
+    };
+
+    const body = JSON.stringify(full);
+    const stored: StoredRecord = {
+      ...full,
+      crc: crc32(body)
+    };
+
+    await this.fd!.write(JSON.stringify(stored) + "\n");
+    await this.fd!.sync();
+
+    const stat = await this.fd!.stat();
+    if (stat.size >= MAX_WAL_SIZE) {
+      await this.rotate();
+    }
+
+    return full.lsn;
+  }
+
+  /* -------------------------
+     REPLAY
+  ------------------------- */
+
+  async replay(
+    fromLSN: number,
+    apply: (r: WALRecord) => Promise<void>
+  ): Promise<void> {
+    if (!fs.existsSync(this.walDir)) return;
+
+    const files = fs
+      .readdirSync(this.walDir)
+      .filter(f => f.startsWith("wal-"))
+      .sort();
+
+    for (const file of files) {
+      const filePath = path.join(this.walDir, file);
+      const data = fs.readFileSync(filePath, "utf8");
+      const lines = data.split("\n");
+
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (!line.trim()) continue;
+
+        let parsed: StoredRecord;
+        try {
+          parsed = JSON.parse(line);
+        } catch {
+          console.error("WAL parse error, stopping replay");
+          return;
+        }
+
+        const { crc, ...record } = parsed;
+        const expected = crc32(JSON.stringify(record));
+
+        if (expected !== crc) {
+          console.error(
+            "WAL checksum mismatch, stopping replay",
+            { file, line: i + 1 }
+          );
+          return;
+        }
+
+        if (record.lsn <= fromLSN) continue;
+
+        this.lsn = Math.max(this.lsn, record.lsn);
+        await apply(record);
+      }
+    }
+  }
+
+  /* -------------------------
+     CLEANUP
+  ------------------------- */
+
+  async cleanup(beforeGen: number) {
+    if (!fs.existsSync(this.walDir)) return;
+
+    const files = fs.readdirSync(this.walDir);
+    for (const f of files) {
+      const m = f.match(/^wal-(\d+)\.log$/);
+      if (!m) continue;
+
+      const gen = Number(m[1]);
+      if (gen < beforeGen) {
+        fs.unlinkSync(path.join(this.walDir, f));
+      }
+    }
+  }
+
+  /* -------------------------
+     GETTERS
+  ------------------------- */
+
+  getCurrentLSN() {
+    return this.lsn;
+  }
+
+  getCurrentGen() {
+    return this.currentGen;
+  }
+}