@liorandb/core 1.0.19 → 1.1.1

package/src/core/collection.ts

@@ -23,6 +23,10 @@ export interface UpdateOptions {
   upsert?: boolean;
 }
 
+export interface CollectionOptions {
+  readonly?: boolean;
+}
+
 export class Collection<T = any> {
   dir: string;
   db: ClassicLevel<string, string>;
@@ -33,16 +37,31 @@ export class Collection<T = any> {
   private migrations: Migration<T>[] = [];
 
   private indexes = new Map<string, Index>();
+  private readonlyMode: boolean;
 
   constructor(
     dir: string,
     schema?: ZodSchema<T>,
-    schemaVersion: number = 1
+    schemaVersion: number = 1,
+    options?: CollectionOptions
   ) {
     this.dir = dir;
-    this.db = new ClassicLevel(dir, { valueEncoding: "utf8" });
     this.schema = schema;
     this.schemaVersion = schemaVersion;
+    this.readonlyMode = options?.readonly ?? false;
+
+    this.db = new ClassicLevel(dir, {
+      valueEncoding: "utf8",
+      readOnly: this.readonlyMode
+    } as any);
+  }
+
+  /* ===================== INTERNAL ===================== */
+
+  private assertWritable() {
+    if (this.readonlyMode) {
+      throw new Error("Collection is in readonly replica mode");
+    }
   }
 
   /* ===================== SCHEMA ===================== */
@@ -106,6 +125,7 @@ export class Collection<T = any> {
   }
 
   private async _updateIndexes(oldDoc: any, newDoc: any) {
+    if (this.readonlyMode) return;
     for (const index of this.indexes.values()) {
       await index.update(oldDoc, newDoc);
     }
@@ -114,6 +134,8 @@ export class Collection<T = any> {
   /* ===================== COMPACTION ===================== */
 
   async compact(): Promise<void> {
+    this.assertWritable();
+
     return this._enqueue(async () => {
       try { await this.db.close(); } catch {}
 
@@ -145,6 +167,8 @@ export class Collection<T = any> {
   /* ===================== STORAGE ===================== */
 
   private async _insertOne(doc: any) {
+    this.assertWritable();
+
     const _id = doc._id ?? uuid();
     const final = this.validate({
       _id,
@@ -159,6 +183,8 @@ export class Collection<T = any> {
   }
 
   private async _insertMany(docs: any[]) {
+    this.assertWritable();
+
     const batch: any[] = [];
     const out = [];
 
@@ -214,14 +240,13 @@ export class Collection<T = any> {
   }
 
   private async _readAndMigrate(id: string) {
-    const enc = await this.db.get(id);
+    const enc = await this.db.get(id).catch(() => null);
     if (!enc) return null;
 
     const raw = decryptData(enc);
     const migrated = this.migrateIfNeeded(raw);
 
-    // Lazy write-back if migrated
-    if (raw.__v !== this.schemaVersion) {
+    if (!this.readonlyMode && raw.__v !== this.schemaVersion) {
       await this.db.put(id, encryptData(migrated));
       await this._updateIndexes(raw, migrated);
     }
@@ -247,9 +272,7 @@ export class Collection<T = any> {
 
   private async _findOne(query: any) {
     if (query?._id) {
-      try {
-        return await this._readAndMigrate(String(query._id));
-      } catch { return null; }
+      return this._readAndMigrate(String(query._id));
     }
 
     const ids = await this._getCandidateIds(query);
@@ -285,6 +308,8 @@ export class Collection<T = any> {
   /* ===================== UPDATE ===================== */
 
   private async _updateOne(filter: any, update: any, options: UpdateOptions) {
+    this.assertWritable();
+
     const ids = await this._getCandidateIds(filter);
 
     for (const id of ids) {
@@ -313,6 +338,8 @@ export class Collection<T = any> {
   }
 
   private async _updateMany(filter: any, update: any) {
+    this.assertWritable();
+
     const ids = await this._getCandidateIds(filter);
     const out = [];
 
@@ -340,6 +367,8 @@ export class Collection<T = any> {
   /* ===================== DELETE ===================== */
 
   private async _deleteOne(filter: any) {
+    this.assertWritable();
+
     const ids = await this._getCandidateIds(filter);
 
     for (const id of ids) {
@@ -357,6 +386,8 @@ export class Collection<T = any> {
   }
 
   private async _deleteMany(filter: any) {
+    this.assertWritable();
+
     const ids = await this._getCandidateIds(filter);
     let count = 0;
 

package/src/core/compaction.ts

@@ -18,42 +18,40 @@ const INDEX_DIR = "__indexes";
 --------------------------------------------------------- */
 
 /**
- * Full safe compaction pipeline:
+ * Full production-safe compaction:
  * 1. Crash recovery
  * 2. Snapshot rebuild
- * 3. Atomic directory swap
- * 4. Index rebuild
+ * 3. Atomic swap
+ * 4. Reopen DB
+ * 5. Rebuild indexes
  */
 export async function compactCollectionEngine(col: Collection) {
   const baseDir = col.dir;
   const tmpDir = baseDir + TMP_SUFFIX;
   const oldDir = baseDir + OLD_SUFFIX;
 
-  // Recover from any previous crash mid-compaction
   await crashRecovery(baseDir);
 
-  // Clean leftovers (paranoia safety)
   safeRemove(tmpDir);
   safeRemove(oldDir);
 
-  // Step 1: rebuild snapshot
   await snapshotRebuild(col, tmpDir);
 
-  // Step 2: atomic swap
-  atomicSwap(baseDir, tmpDir, oldDir);
+  await atomicSwap(baseDir, tmpDir, oldDir);
 
-  // Cleanup
   safeRemove(oldDir);
+
+  // Reopen DB after swap
+  await reopenCollectionDB(col);
+
+  // Rebuild indexes after compaction
+  await rebuildIndexes(col);
 }
 
 /* ---------------------------------------------------------
    SNAPSHOT REBUILD
 --------------------------------------------------------- */
 
-/**
- * Rebuilds DB by copying only live keys
- * WAL is assumed already checkpointed
- */
 async function snapshotRebuild(col: Collection, tmpDir: string) {
   fs.mkdirSync(tmpDir, { recursive: true });
 
@@ -68,28 +66,33 @@ async function snapshotRebuild(col: Collection, tmpDir: string) {
   }
 
   await tmpDB.close();
-  await col.db.close();
+  await col.db.close(); // important: close before swap
 }
 
 /* ---------------------------------------------------------
-   ATOMIC SWAP
+   ATOMIC SWAP (HARDENED)
 --------------------------------------------------------- */
 
-/**
- * Atomic directory replacement (POSIX safe)
- */
-function atomicSwap(base: string, tmp: string, old: string) {
+async function atomicSwap(base: string, tmp: string, old: string) {
+  // Phase 1: rename base → old
   fs.renameSync(base, old);
-  fs.renameSync(tmp, base);
+
+  try {
+    // Phase 2: rename tmp → base
+    fs.renameSync(tmp, base);
+  } catch (err) {
+    // Rollback if tmp rename fails
+    if (fs.existsSync(old)) {
+      fs.renameSync(old, base);
+    }
+    throw err;
+  }
 }
 
 /* ---------------------------------------------------------
    CRASH RECOVERY
 --------------------------------------------------------- */
 
-/**
- * Handles all partial-compaction states
- */
 export async function crashRecovery(baseDir: string) {
   const tmp = baseDir + TMP_SUFFIX;
   const old = baseDir + OLD_SUFFIX;
@@ -106,7 +109,7 @@ export async function crashRecovery(baseDir: string) {
     return;
   }
 
-  // Case 2: rename(base → old) happened, but tmp missing
+  // Case 2: base→old happened but tmp missing
   if (!baseExists && oldExists) {
     fs.renameSync(old, baseDir);
     return;
@@ -119,44 +122,56 @@ export async function crashRecovery(baseDir: string) {
 }
 
 /* ---------------------------------------------------------
-   INDEX REBUILD
+   REOPEN DB
+--------------------------------------------------------- */
+
+async function reopenCollectionDB(col: Collection) {
+  col.db = new ClassicLevel(col.dir, {
+    valueEncoding: "utf8"
+  });
+}
+
+/* ---------------------------------------------------------
+   INDEX REBUILD (SAFE)
 --------------------------------------------------------- */
 
-/**
- * Rebuilds all indexes from compacted DB
- * Guarantees index consistency
- */
 export async function rebuildIndexes(col: Collection) {
   const indexRoot = path.join(col.dir, INDEX_DIR);
 
-  // Close existing index handles
-  for (const idx of col["indexes"].values()) {
+  const oldIndexes = new Map(col["indexes"]);
+
+  // Close old index handles
+  for (const idx of oldIndexes.values()) {
     try {
       await idx.close();
     } catch {}
   }
 
-  // Destroy index directory
   safeRemove(indexRoot);
   fs.mkdirSync(indexRoot, { recursive: true });
 
-  const newIndexes = new Map<string, Index>();
+  const rebuiltIndexes = new Map<string, Index>();
 
-  for (const idx of col["indexes"].values()) {
+  for (const idx of oldIndexes.values()) {
     const rebuilt = new Index(col.dir, idx.field, {
       unique: idx.unique
     });
 
     for await (const [, enc] of col.db.iterator()) {
       if (!enc) continue;
-      const doc = decryptData(enc);
-      await rebuilt.insert(doc);
+
+      try {
+        const doc = decryptData(enc);
+        await rebuilt.insert(doc);
+      } catch {
+        // Skip corrupted doc safely
+      }
     }
 
-    newIndexes.set(idx.field, rebuilt);
+    rebuiltIndexes.set(idx.field, rebuilt);
   }
 
-  col["indexes"] = newIndexes;
+  col["indexes"] = rebuiltIndexes;
 }
 
 /* ---------------------------------------------------------

package/src/core/database.ts

@@ -13,9 +13,6 @@ import { CheckpointManager } from "./checkpoint.js";
 /* ----------------------------- TYPES ----------------------------- */
 
 type TXOp = { tx: number; col: string; op: string; args: any[] };
-type TXCommit = { tx: number; commit: true };
-type TXApplied = { tx: number; applied: true };
-type WALEntry = TXOp | TXCommit | TXApplied;
 
 type IndexMeta = {
   field: string;
@@ -43,43 +40,49 @@ class DBTransactionContext {
   ) {}
 
   collection(name: string) {
-    return new Proxy({}, {
-      get: (_, prop: string) => {
-        return (...args: any[]) => {
-          this.ops.push({
-            tx: this.txId,
-            col: name,
-            op: prop,
-            args
-          });
-        };
+    return new Proxy(
+      {},
+      {
+        get: (_, prop: string) => {
+          return (...args: any[]) => {
+            this.ops.push({
+              tx: this.txId,
+              col: name,
+              op: prop,
+              args
+            });
+          };
+        }
       }
-    });
+    );
   }
 
   async commit() {
+    if (this.db.isReadonly()) {
+      throw new Error("Cannot commit transaction in readonly mode");
+    }
+
     for (const op of this.ops) {
-      const recordOp: any = {
+      await this.db.wal.append({
         tx: this.txId,
         type: "op",
         payload: op
-      };
-      await this.db.wal.append(recordOp);
+      } as any);
     }
 
-    const commitRecord: any = {
+    const commitLSN = await this.db.wal.append({
       tx: this.txId,
       type: "commit"
-    };
-    await this.db.wal.append(commitRecord);
+    } as any);
 
     await this.db.applyTransaction(this.ops);
 
-    const appliedRecord: any = {
+    const appliedLSN = await this.db.wal.append({
       tx: this.txId,
       type: "applied"
-    };
-    await this.db.wal.append(appliedRecord);
+    } as any);
+
+    this.db.advanceCheckpoint(appliedLSN);
 
     await this.db.postCommitMaintenance();
   }
@@ -99,8 +102,10 @@ export class LioranDB {
   private migrator: MigrationEngine;
   private static TX_SEQ = 0;
 
-  public wal: WALManager;
-  private checkpoint: CheckpointManager;
+  public wal!: WALManager;
+  private checkpoint!: CheckpointManager;
+
+  private readonly readonlyMode: boolean;
 
   constructor(basePath: string, dbName: string, manager: LioranManager) {
     this.basePath = basePath;
@@ -108,24 +113,42 @@ export class LioranDB {
     this.manager = manager;
     this.collections = new Map();
 
+    this.readonlyMode = (manager as any)?.isReadonly?.() ?? false;
+
     this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
     this.loadMeta();
 
-    this.wal = new WALManager(basePath);
-    this.checkpoint = new CheckpointManager(basePath);
+    if (!this.readonlyMode) {
+      this.wal = new WALManager(basePath);
+      this.checkpoint = new CheckpointManager(basePath);
+    }
 
     this.migrator = new MigrationEngine(this);
 
     this.initialize().catch(console.error);
   }
 
+  /* ------------------------- MODE ------------------------- */
+
+  public isReadonly(): boolean {
+    return this.readonlyMode;
+  }
+
+  private assertWritable() {
+    if (this.readonlyMode) {
+      throw new Error("Database is in readonly replica mode");
+    }
+  }
+
   /* ------------------------- INIT & RECOVERY ------------------------- */
 
   private async initialize() {
-    await this.recoverFromWAL();
+    if (!this.readonlyMode) {
+      await this.recoverFromWAL();
+    }
   }
 
   private async recoverFromWAL() {
@@ -143,18 +166,36 @@ export class LioranDB {
         applied.add(record.tx);
       } else if (record.type === "op") {
         if (!ops.has(record.tx)) ops.set(record.tx, []);
-        ops.get(record.tx)!.push(record.payload);
+        ops.get(record.tx)!.push(record.payload as TXOp);
       }
     });
 
+    let highestAppliedLSN = fromLSN;
+
     for (const tx of committed) {
       if (applied.has(tx)) continue;
 
       const txOps = ops.get(tx);
       if (txOps) {
         await this.applyTransaction(txOps);
+        highestAppliedLSN = this.wal.getCurrentLSN();
       }
     }
+
+    this.advanceCheckpoint(highestAppliedLSN);
+  }
+
+  /* ------------------------- CHECKPOINT ADVANCE ------------------------- */
+
+  public advanceCheckpoint(lsn: number) {
+    if (this.readonlyMode) return;
+
+    const current = this.checkpoint.get();
+
+    if (lsn > current.lsn) {
+      this.checkpoint.save(lsn, this.wal.getCurrentGen());
+      this.wal.cleanup(this.wal.getCurrentGen() - 1).catch(() => {});
+    }
   }
 
   /* ------------------------- META ------------------------- */
@@ -179,6 +220,7 @@ export class LioranDB {
   }
 
   private saveMeta() {
+    if (this.readonlyMode) return;
     fs.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
   }
 
@@ -187,6 +229,7 @@ export class LioranDB {
   }
 
   setSchemaVersion(v: string) {
+    this.assertWritable();
     this.meta.schemaVersion = v;
     this.saveMeta();
   }
@@ -194,6 +237,7 @@ export class LioranDB {
   /* ------------------------- DB MIGRATIONS ------------------------- */
 
   migrate(from: string, to: string, fn: (db: LioranDB) => Promise<void>) {
+    this.assertWritable();
     this.migrator.register(from, to, async db => {
       await fn(db);
       db.setSchemaVersion(to);
@@ -201,6 +245,7 @@ export class LioranDB {
   }
 
   async applyMigrations(targetVersion: string) {
+    this.assertWritable();
     await this.migrator.upgradeToLatest();
   }
 
@@ -234,7 +279,8 @@ export class LioranDB {
     const col = new Collection<T>(
       colPath,
       schema,
-      schemaVersion ?? 1
+      schemaVersion ?? 1,
+      { readonly: this.readonlyMode }
     );
 
     const metas = this.meta.indexes[name] ?? [];
@@ -253,6 +299,8 @@ export class LioranDB {
     field: string,
     options: IndexOptions = {}
   ) {
+    this.assertWritable();
+
     const col = this.collection(collection);
 
     const existing = this.meta.indexes[collection]?.find(i => i.field === field);
@@ -265,10 +313,7 @@ export class LioranDB {
       try {
         const doc = decryptData(enc);
         await index.insert(doc);
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        console.warn(`Index build skipped doc ${key}: ${msg}`);
-      }
+      } catch {}
     }
 
     col.registerIndex(index);
@@ -284,11 +329,13 @@ export class LioranDB {
   /* ------------------------- COMPACTION ------------------------- */
 
   async compactCollection(name: string) {
+    this.assertWritable();
     const col = this.collection(name);
     await col.compact();
   }
 
   async compactAll() {
+    this.assertWritable();
     for (const name of this.collections.keys()) {
       await this.compactCollection(name);
     }
@@ -297,6 +344,7 @@ export class LioranDB {
   /* ------------------------- TX API ------------------------- */
 
   async transaction<T>(fn: (tx: DBTransactionContext) => Promise<T>): Promise<T> {
+    this.assertWritable();
     const txId = ++LioranDB.TX_SEQ;
     const tx = new DBTransactionContext(this, txId);
     const result = await fn(tx);
@@ -306,9 +354,7 @@ export class LioranDB {
 
   /* ------------------------- POST COMMIT ------------------------- */
 
-  public async postCommitMaintenance() {
-    // Custom maintenance can be added here
-  }
+  public async postCommitMaintenance() {}
 
   /* ------------------------- SHUTDOWN ------------------------- */
 
@@ -316,7 +362,6 @@ export class LioranDB {
     for (const col of this.collections.values()) {
       try { await col.close(); } catch {}
     }
-
     this.collections.clear();
   }
 }