@liorandb/core 1.0.15 → 1.0.17

package/src/LioranManager.ts

@@ -51,6 +51,8 @@ export class LioranManager {
     }
   }
 
+  /* ---------------- LOCK MANAGEMENT ---------------- */
+
   private isProcessAlive(pid: number): boolean {
     try {
       process.kill(pid, 0);
@@ -68,7 +70,6 @@ export class LioranManager {
       fs.writeSync(this.lockFd, String(process.pid));
       return true;
     } catch {
-      // Possible stale lock → validate PID
       try {
         const pid = Number(fs.readFileSync(lockPath, "utf8"));
         if (!this.isProcessAlive(pid)) {
@@ -77,18 +78,18 @@ export class LioranManager {
           fs.writeSync(this.lockFd, String(process.pid));
           return true;
         }
-      } catch {}
-
+      } catch { }
       return false;
     }
   }
 
+  /* ---------------- DB OPEN ---------------- */
+
   async db(name: string): Promise<LioranDB> {
     if (this.mode === ProcessMode.CLIENT) {
       await dbQueue.exec("db", { db: name });
       return new IPCDatabase(name) as any;
     }
-
     return this.openDatabase(name);
   }
 
@@ -107,6 +108,64 @@ export class LioranManager {
     return db;
   }
 
+  /* ---------------- SNAPSHOT ORCHESTRATION ---------------- */
+
+  /**
+   * Create TAR snapshot of full DB directory
+   */
+  async snapshot(snapshotPath: string) {
+    if (this.mode === ProcessMode.CLIENT) {
+      return dbQueue.exec("snapshot", { path: snapshotPath });
+    }
+
+    // Flush all DBs safely
+    for (const db of this.openDBs.values()) {
+      for (const col of db.collections.values()) {
+        try { await col.db.close(); } catch { }
+      }
+    }
+
+    // Ensure backup directory exists
+    fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
+
+    const tar = await import("tar");
+
+    await tar.c({
+      gzip: true,
+      file: snapshotPath,
+      cwd: this.rootPath,
+      portable: true
+    }, ["./"]);
+
+    return true;
+  }
+
+  /**
+   * Restore TAR snapshot safely
+   */
+  async restore(snapshotPath: string) {
+    if (this.mode === ProcessMode.CLIENT) {
+      return dbQueue.exec("restore", { path: snapshotPath });
+    }
+
+    await this.closeAll();
+
+    fs.rmSync(this.rootPath, { recursive: true, force: true });
+    fs.mkdirSync(this.rootPath, { recursive: true });
+
+    const tar = await import("tar");
+
+    await tar.x({
+      file: snapshotPath,
+      cwd: this.rootPath
+    });
+
+    console.log("Restore completed. Restart required.");
+    process.exit(0);
+  }
+
+  /* ---------------- SHUTDOWN ---------------- */
+
   async closeAll(): Promise<void> {
     if (this.closed) return;
     this.closed = true;
@@ -117,7 +176,7 @@ export class LioranManager {
     }
 
     for (const db of this.openDBs.values()) {
-      try { await db.close(); } catch {}
+      try { await db.close(); } catch { }
     }
 
     this.openDBs.clear();
@@ -125,7 +184,7 @@ export class LioranManager {
     try {
       if (this.lockFd) fs.closeSync(this.lockFd);
       fs.unlinkSync(path.join(this.rootPath, ".lioran.lock"));
-    } catch {}
+    } catch { }
 
     await this.ipcServer?.close();
   }
@@ -154,7 +213,7 @@ export class LioranManager {
 /* ---------------- IPC PROXY DB ---------------- */
 
 class IPCDatabase {
-  constructor(private name: string) {}
+  constructor(private name: string) { }
 
   collection(name: string) {
     return new IPCCollection(this.name, name);
@@ -165,7 +224,7 @@ class IPCCollection {
   constructor(
     private db: string,
     private col: string
-  ) {}
+  ) { }
 
   private call(method: string, params: any[]) {
     return dbQueue.exec("op", {

package/src/core/database.ts

@@ -1,10 +1,15 @@
 import path from "path";
 import fs from "fs";
+import { execFile } from "child_process";
+import { promisify } from "util";
 import { Collection } from "./collection.js";
 import { Index, IndexOptions } from "./index.js";
+import { MigrationEngine } from "./migration.js";
 import type { LioranManager } from "../LioranManager.js";
 import type { ZodSchema } from "zod";
 
+const exec = promisify(execFile);
+
 /* ----------------------------- TYPES ----------------------------- */
 
 type TXOp = { tx: number; col: string; op: string; args: any[] };
@@ -20,10 +25,12 @@ type IndexMeta = {
 type DBMeta = {
   version: number;
   indexes: Record<string, IndexMeta[]>;
+  schemaVersion: string;
 };
 
 const META_FILE = "__db_meta.json";
 const META_VERSION = 1;
+const DEFAULT_SCHEMA_VERSION = "v1";
 
 /* ---------------------- TRANSACTION CONTEXT ---------------------- */
 
@@ -33,7 +40,7 @@ class DBTransactionContext {
   constructor(
     private db: LioranDB,
     public readonly txId: number
-  ) {}
+  ) { }
 
   collection(name: string) {
     return new Proxy({}, {
@@ -66,10 +73,13 @@ export class LioranDB {
   dbName: string;
   manager: LioranManager;
   collections: Map<string, Collection>;
+
   private walPath: string;
   private metaPath: string;
   private meta!: DBMeta;
 
+  private migrator: MigrationEngine;
+
   private static TX_SEQ = 0;
 
   constructor(basePath: string, dbName: string, manager: LioranManager) {
@@ -77,12 +87,15 @@ export class LioranDB {
     this.dbName = dbName;
     this.manager = manager;
     this.collections = new Map();
+
     this.walPath = path.join(basePath, "__tx_wal.log");
     this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
     this.loadMeta();
+    this.migrator = new MigrationEngine(this);
+
     this.recoverFromWAL().catch(console.error);
   }
 
@@ -90,15 +103,20 @@ export class LioranDB {
 
   private loadMeta() {
     if (!fs.existsSync(this.metaPath)) {
-      this.meta = { version: META_VERSION, indexes: {} };
+      this.meta = {
+        version: META_VERSION,
+        indexes: {},
+        schemaVersion: DEFAULT_SCHEMA_VERSION
+      };
       this.saveMeta();
       return;
     }
 
-    try {
-      this.meta = JSON.parse(fs.readFileSync(this.metaPath, "utf8"));
-    } catch {
-      throw new Error("Database metadata corrupted");
+    this.meta = JSON.parse(fs.readFileSync(this.metaPath, "utf8"));
+
+    if (!this.meta.schemaVersion) {
+      this.meta.schemaVersion = DEFAULT_SCHEMA_VERSION;
+      this.saveMeta();
     }
   }
 
@@ -106,6 +124,28 @@ export class LioranDB {
     fs.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
   }
 
+  getSchemaVersion(): string {
+    return this.meta.schemaVersion;
+  }
+
+  setSchemaVersion(v: string) {
+    this.meta.schemaVersion = v;
+    this.saveMeta();
+  }
+
+  /* ------------------------- MIGRATION API ------------------------- */
+
+  migrate(from: string, to: string, fn: (db: LioranDB) => Promise<void>) {
+    this.migrator.register(from, to, async db => {
+      await fn(db);
+      db.setSchemaVersion(to);
+    });
+  }
+
+  async applyMigrations(targetVersion: string) {
+    await this.migrator.upgradeToLatest();
+  }
+
   /* ------------------------- WAL ------------------------- */
 
   async writeWAL(entries: WALEntry[]) {
@@ -118,7 +158,7 @@ export class LioranDB {
   }
 
   async clearWAL() {
-    try { await fs.promises.unlink(this.walPath); } catch {}
+    try { await fs.promises.unlink(this.walPath); } catch { }
   }
 
   private async recoverFromWAL() {
@@ -133,17 +173,13 @@ export class LioranDB {
     for (const line of raw.split("\n")) {
       if (!line.trim()) continue;
 
-      try {
-        const entry: WALEntry = JSON.parse(line);
-
-        if ("commit" in entry) committed.add(entry.tx);
-        else if ("applied" in entry) applied.add(entry.tx);
-        else {
-          if (!ops.has(entry.tx)) ops.set(entry.tx, []);
-          ops.get(entry.tx)!.push(entry);
-        }
-      } catch {
-        break;
+      const entry: WALEntry = JSON.parse(line);
+
+      if ("commit" in entry) committed.add(entry.tx);
+      else if ("applied" in entry) applied.add(entry.tx);
+      else {
+        if (!ops.has(entry.tx)) ops.set(entry.tx, []);
+        ops.get(entry.tx)!.push(entry);
       }
     }
 
@@ -177,7 +213,6 @@ export class LioranDB {
 
     const col = new Collection<T>(colPath, schema);
 
-    // Auto-load indexes
     const metas = this.meta.indexes[name] ?? [];
     for (const m of metas) {
       col.registerIndex(new Index(colPath, m.field, m.options));
@@ -201,9 +236,26 @@ export class LioranDB {
 
     const index = new Index(col.dir, field, options);
 
-    for await (const [, enc] of col.db.iterator()) {
-      const doc = JSON.parse(Buffer.from(enc, "base64").subarray(32).toString("utf8"));
-      await index.insert(doc);
+    // for await (const [, enc] of col.db.iterator()) {
+    //   // const doc = JSON.parse(
+    //   //   Buffer.from(enc, "base64").subarray(32).toString("utf8")
+    //   // );
+    //   const payload = Buffer.from(enc, "utf8").subarray(32);
+    //   const doc = JSON.parse(payload.toString("utf8"));
+    //   await index.insert(doc);
+    // }
+
+    for await (const [key, enc] of col.db.iterator()) {
+      if (!enc) continue;
+
+      try {
+        const doc = decryptData(enc);           // ← this does base64 → AES-GCM → JSON
+        await index.insert(doc);
+      } catch (err) {
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        console.warn(`Could not decrypt document ${key} during index build: ${errorMessage}`);
+        // You can continue, or collect bad keys for later inspection
+      }
     }
 
     col.registerIndex(index);
@@ -216,27 +268,16 @@ export class LioranDB {
     this.saveMeta();
   }
 
-  /* ---------------------- COMPACTION ORCHESTRATOR ---------------------- */
+  /* ------------------------- COMPACTION ------------------------- */
 
-  /**
-   * Compact single collection safely (WAL + TX safe)
-   */
   async compactCollection(name: string) {
-    const col = this.collection(name);
-
-    // Ensure WAL is fully flushed
     await this.clearWAL();
-
-    // Perform compaction inside write barrier
+    const col = this.collection(name);
     await col.compact();
   }
 
-  /**
-   * Compact entire database safely
-   */
   async compactAll() {
     await this.clearWAL();
-
     for (const name of this.collections.keys()) {
       await this.compactCollection(name);
     }
@@ -247,10 +288,8 @@ export class LioranDB {
   async transaction<T>(fn: (tx: DBTransactionContext) => Promise<T>): Promise<T> {
     const txId = ++LioranDB.TX_SEQ;
     const tx = new DBTransactionContext(this, txId);
-
     const result = await fn(tx);
     await tx.commit();
-
     return result;
   }
 
@@ -258,8 +297,12 @@ export class LioranDB {
 
   async close(): Promise<void> {
     for (const col of this.collections.values()) {
-      try { await col.close(); } catch {}
+      try { await col.close(); } catch { }
     }
     this.collections.clear();
   }
-}
+}
+
+function decryptData(enc: string) {
+  throw new Error("Function not implemented.");
+}

package/src/core/migration.store.ts

@@ -0,0 +1,40 @@
+import { MigrationMeta } from "./migration.types";
+import type { LioranDB } from "./database.js";
+
+const MIGRATION_COLLECTION = "__migrations__";
+const MIGRATION_KEY = "__migration_meta__";
+
+export class MigrationStore {
+  constructor(private db: LioranDB) {}
+
+  async get(): Promise<MigrationMeta> {
+    const col = this.db.collection<MigrationMeta>(MIGRATION_COLLECTION);
+    const meta = await col.findOne((d: MigrationMeta) => d.id === MIGRATION_KEY);
+    return (
+      meta ?? {
+        currentVersion: "v1",
+        history: []
+      }
+    );
+  }
+
+  async set(meta: MigrationMeta): Promise<void> {
+    const col = this.db.collection<MigrationMeta>(MIGRATION_COLLECTION);
+    const existing = await col.findOne((d: MigrationMeta) => d.id === MIGRATION_KEY);
+    if (existing) {
+      await col.updateOne(existing._id, { ...meta, id: MIGRATION_KEY });
+    } else {
+      await col.insertOne({ ...meta, id: MIGRATION_KEY });
+    }
+  }
+
+  async updateVersion(version: string): Promise<void> {
+    const meta = await this.get();
+    meta.currentVersion = version;
+    meta.history.push({
+      version,
+      appliedAt: Date.now()
+    });
+    await this.set(meta);
+  }
+}

package/src/core/migration.ts

@@ -0,0 +1,162 @@
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import type { LioranDB } from "./database.js";
+
+export type MigrationFn = (db: LioranDB) => Promise<void>;
+
+type MigrationRecord = {
+  from: string;
+  to: string;
+  checksum: string;
+  appliedAt: number;
+};
+
+const LOCK_FILE = "__migration.lock";
+const HISTORY_FILE = "__migration_history.json";
+
+export class MigrationEngine {
+  private migrations = new Map<string, MigrationFn>();
+
+  constructor(private db: LioranDB) {}
+
+  /* ------------------------------------------------------------ */
+  /* Public API */
+  /* ------------------------------------------------------------ */
+
+  register(from: string, to: string, fn: MigrationFn) {
+    const key = `${from}→${to}`;
+
+    if (this.migrations.has(key)) {
+      throw new Error(`Duplicate migration: ${key}`);
+    }
+
+    this.migrations.set(key, fn);
+  }
+
+  async migrate(from: string, to: string, fn: MigrationFn) {
+    this.register(from, to, fn);
+    await this.execute();
+  }
+
+  async upgradeToLatest() {
+    await this.execute();
+  }
+
+  /* ------------------------------------------------------------ */
+  /* Core Execution Logic */
+  /* ------------------------------------------------------------ */
+
+  private async execute() {
+    let current = this.db.getSchemaVersion();
+
+    while (true) {
+      const next = this.findNext(current);
+      if (!next) break;
+
+      const fn = this.migrations.get(`${current}→${next}`)!;
+      await this.runMigration(current, next, fn);
+      current = next;
+    }
+  }
+
+  private findNext(current: string): string | null {
+    for (const key of this.migrations.keys()) {
+      const [from, to] = key.split("→");
+      if (from === current) return to;
+    }
+    return null;
+  }
+
+  /* ------------------------------------------------------------ */
+  /* Atomic Migration Execution */
+  /* ------------------------------------------------------------ */
+
+  private async runMigration(from: string, to: string, fn: MigrationFn) {
+    const current = this.db.getSchemaVersion();
+    if (current !== from) {
+      throw new Error(
+        `Schema mismatch: DB=${current}, expected=${from}`
+      );
+    }
+
+    const lockPath = path.join(this.db.basePath, LOCK_FILE);
+
+    if (fs.existsSync(lockPath)) {
+      throw new Error(
+        "Previous migration interrupted. Resolve manually before continuing."
+      );
+    }
+
+    this.acquireLock(lockPath);
+
+    try {
+      await this.db.transaction(async () => {
+        await fn(this.db);
+        this.writeHistory(from, to, fn);
+        this.db.setSchemaVersion(to);
+      });
+    } finally {
+      this.releaseLock(lockPath);
+    }
+  }
+
+  /* ------------------------------------------------------------ */
+  /* Locking */
+  /* ------------------------------------------------------------ */
+
+  private acquireLock(file: string) {
+    const token = crypto.randomBytes(16).toString("hex");
+
+    fs.writeFileSync(
+      file,
+      JSON.stringify({
+        pid: process.pid,
+        token,
+        time: Date.now(),
+      })
+    );
+  }
+
+  private releaseLock(file: string) {
+    if (fs.existsSync(file)) fs.unlinkSync(file);
+  }
+
+  /* ------------------------------------------------------------ */
+  /* Migration History */
+  /* ------------------------------------------------------------ */
+
+  private historyPath() {
+    return path.join(this.db.basePath, HISTORY_FILE);
+  }
+
+  private readHistory(): MigrationRecord[] {
+    if (!fs.existsSync(this.historyPath())) return [];
+    return JSON.parse(fs.readFileSync(this.historyPath(), "utf8"));
+  }
+
+  private writeHistory(from: string, to: string, fn: MigrationFn) {
+    const history = this.readHistory();
+
+    history.push({
+      from,
+      to,
+      checksum: this.hash(fn.toString()),
+      appliedAt: Date.now(),
+    });
+
+    fs.writeFileSync(this.historyPath(), JSON.stringify(history, null, 2));
+  }
+
+  private hash(data: string) {
+    return crypto.createHash("sha256").update(data).digest("hex");
+  }
+
+  /* ------------------------------------------------------------ */
+  /* Diagnostics */
+  /* ------------------------------------------------------------ */
+
+  getHistory(): MigrationRecord[] {
+    return this.readHistory();
+  }
+}

package/src/core/migration.types.ts

@@ -0,0 +1,22 @@
+import type { LioranDB } from "./database.js";
+
+export type MigrationVersion = string;
+
+export type MigrationFn = (db: LioranDB) => Promise<void>;
+
+export interface Migration {
+  from: MigrationVersion;
+  to: MigrationVersion;
+  run: MigrationFn;
+}
+
+export interface MigrationRecord {
+  version: MigrationVersion;
+  appliedAt: number;
+}
+
+export interface MigrationMeta {
+  id?: string;
+  currentVersion: MigrationVersion;
+  history: MigrationRecord[];
+}

package/src/ipc/index.ts

@@ -72,6 +72,14 @@ class DBProxy {
   compact() {
     return dbQueue.exec("compact:db", { db: this.dbName });
   }
+
+  snapshot(path: string) {
+    return dbQueue.exec("snapshot", { path });
+  }
+
+  restore(path: string) {
+    return dbQueue.exec("restore", { path });
+  }
 }
 
 /* -------------------------------- MANAGER PROXY -------------------------------- */
@@ -85,6 +93,18 @@ class LioranManagerIPC {
   compactAll() {
     return dbQueue.exec("compact:all", {});
   }
+
+  snapshot(path: string) {
+    return dbQueue.exec("snapshot", { path });
+  }
+
+  restore(path: string) {
+    return dbQueue.exec("restore", { path });
+  }
+
+  shutdown() {
+    return dbQueue.shutdown();
+  }
 }
 
 export const manager = new LioranManagerIPC();

package/src/ipc/queue.ts

@@ -10,7 +10,13 @@ export type IPCAction =
   | "compact:collection"
   | "compact:db"
   | "compact:all"
-  | "shutdown";
+  | "shutdown"
+  | "open"
+  | "close"
+  | "minimize"
+  | "maximize"
+  | "restore"
+  | "snapshot";
 
 /* -------------------------------- DB QUEUE -------------------------------- */
 
@@ -39,6 +45,16 @@ export class DBQueue {
     return this.exec("compact:all", {});
   }
 
+  /* ----------------------------- SNAPSHOT API ----------------------------- */
+
+  snapshot(path: string) {
+    return this.exec("snapshot", { path });
+  }
+
+  restore(path: string) {
+    return this.exec("restore", { path });
+  }
+
   /* ------------------------------ SHUTDOWN ------------------------------ */
 
   async shutdown() {