@liorandb/core 1.0.13 → 1.0.14

package/src/core/collection.ts

@@ -1,9 +1,10 @@
 import { ClassicLevel } from "classic-level";
-import { matchDocument, applyUpdate } from "./query.js";
+import { matchDocument, applyUpdate, extractIndexQuery } from "./query.js";
 import { v4 as uuid } from "uuid";
 import { encryptData, decryptData } from "../utils/encryption.js";
 import type { ZodSchema } from "zod";
 import { validateSchema } from "../utils/schema.js";
+import { Index } from "./index.js";
 
 export interface UpdateOptions {
   upsert?: boolean;
@@ -14,6 +15,7 @@ export class Collection<T = any> {
   db: ClassicLevel<string, string>;
   private queue: Promise<any> = Promise.resolve();
   private schema?: ZodSchema<T>;
+  private indexes = new Map<string, Index>();
 
   constructor(dir: string, schema?: ZodSchema<T>) {
     this.dir = dir;
@@ -21,6 +23,18 @@ export class Collection<T = any> {
     this.schema = schema;
   }
 
+  /* ---------------------- INDEX MANAGEMENT ---------------------- */
+
+  registerIndex(index: Index) {
+    this.indexes.set(index.field, index);
+  }
+
+  getIndex(field: string) {
+    return this.indexes.get(field);
+  }
+
+  /* -------------------------- CORE -------------------------- */
+
   setSchema(schema: ZodSchema<T>) {
     this.schema = schema;
   }
@@ -35,6 +49,9 @@ export class Collection<T = any> {
   }
 
   async close(): Promise<void> {
+    for (const idx of this.indexes.values()) {
+      try { await idx.close(); } catch {}
+    }
     try { await this.db.close(); } catch {}
   }
 
@@ -53,6 +70,8 @@ export class Collection<T = any> {
     }
   }
 
+  /* --------------------- PUBLIC API --------------------- */
+
   insertOne(doc: T & { _id?: string }) {
     return this._enqueue(() => this._exec("insertOne", [doc]));
   }
@@ -99,12 +118,23 @@ export class Collection<T = any> {
     );
   }
 
+  /* ------------------ INDEX HOOK ------------------ */
+
+  private async _updateIndexes(oldDoc: any, newDoc: any) {
+    for (const index of this.indexes.values()) {
+      await index.update(oldDoc, newDoc);
+    }
+  }
+
   /* ---------------- Storage ---------------- */
 
   private async _insertOne(doc: any) {
     const _id = doc._id ?? uuid();
     const final = this.validate({ _id, ...doc });
+
     await this.db.put(String(_id), encryptData(final));
+    await this._updateIndexes(null, final);
+
     return final;
   }
 
@@ -115,11 +145,22 @@ export class Collection<T = any> {
     for (const d of docs) {
       const _id = d._id ?? uuid();
       const final = this.validate({ _id, ...d });
-      batch.push({ type: "put", key: String(_id), value: encryptData(final) });
+
+      batch.push({
+        type: "put",
+        key: String(_id),
+        value: encryptData(final)
+      });
+
       out.push(final);
     }
 
     await this.db.batch(batch);
+
+    for (const doc of out) {
+      await this._updateIndexes(null, doc);
+    }
+
     return out;
   }
 
@@ -131,6 +172,19 @@ export class Collection<T = any> {
       } catch { return null; }
     }
 
+    const iq = extractIndexQuery(query);
+    if (iq && this.indexes.has(iq.field)) {
+      const ids = await this.indexes.get(iq.field)!.find(iq.value);
+      if (!ids.length) return null;
+
+      try {
+        const enc = await this.db.get(ids[0]);
+        return enc ? decryptData(enc) : null;
+      } catch {
+        return null;
+      }
+    }
+
     for await (const [, enc] of this.db.iterator()) {
       const v = decryptData(enc);
       if (matchDocument(v, query)) return v;
@@ -142,17 +196,27 @@ export class Collection<T = any> {
   private async _updateOne(filter: any, update: any, options: UpdateOptions) {
     for await (const [key, enc] of this.db.iterator()) {
       const value = decryptData(enc);
+
       if (matchDocument(value, filter)) {
         const updated = this.validate(applyUpdate(value, update)) as any;
         updated._id = value._id;
+
         await this.db.put(key, encryptData(updated));
+        await this._updateIndexes(value, updated);
+
         return updated;
       }
     }
 
     if (options?.upsert) {
-      const doc = this.validate({ _id: uuid(), ...applyUpdate({}, update) }) as any;
+      const doc = this.validate({
+        _id: uuid(),
+        ...applyUpdate({}, update)
+      }) as any;
+
       await this.db.put(String(doc._id), encryptData(doc));
+      await this._updateIndexes(null, doc);
+
       return doc;
     }
 
@@ -164,10 +228,14 @@ export class Collection<T = any> {
 
     for await (const [key, enc] of this.db.iterator()) {
       const value = decryptData(enc);
+
       if (matchDocument(value, filter)) {
         const updated = this.validate(applyUpdate(value, update)) as any;
         updated._id = value._id;
+
         await this.db.put(key, encryptData(updated));
+        await this._updateIndexes(value, updated);
+
         out.push(updated);
       }
     }
@@ -176,18 +244,38 @@ export class Collection<T = any> {
   }
 
   private async _find(query: any) {
+    const iq = extractIndexQuery(query);
+
+    if (iq && this.indexes.has(iq.field)) {
+      const ids = await this.indexes.get(iq.field)!.find(iq.value);
+      const out = [];
+
+      for (const id of ids) {
+        try {
+          const enc = await this.db.get(id);
+          if (enc) out.push(decryptData(enc));
+        } catch {}
+      }
+
+      return out;
+    }
+
     const out = [];
     for await (const [, enc] of this.db.iterator()) {
       const v = decryptData(enc);
       if (matchDocument(v, query)) out.push(v);
     }
+
     return out;
   }
 
   private async _deleteOne(filter: any) {
     for await (const [key, enc] of this.db.iterator()) {
-      if (matchDocument(decryptData(enc), filter)) {
+      const value = decryptData(enc);
+
+      if (matchDocument(value, filter)) {
         await this.db.del(key);
+        await this._updateIndexes(value, null);
         return true;
       }
     }
@@ -196,20 +284,32 @@ export class Collection<T = any> {
 
   private async _deleteMany(filter: any) {
     let count = 0;
+
     for await (const [key, enc] of this.db.iterator()) {
-      if (matchDocument(decryptData(enc), filter)) {
+      const value = decryptData(enc);
+
+      if (matchDocument(value, filter)) {
         await this.db.del(key);
+        await this._updateIndexes(value, null);
         count++;
       }
     }
+
     return count;
   }
 
   private async _countDocuments(filter: any) {
     let c = 0;
+
+    const iq = extractIndexQuery(filter);
+    if (iq && this.indexes.has(iq.field)) {
+      return (await this.indexes.get(iq.field)!.find(iq.value)).length;
+    }
+
     for await (const [, enc] of this.db.iterator()) {
       if (matchDocument(decryptData(enc), filter)) c++;
     }
+
     return c;
   }
 }

package/src/core/database.ts

@@ -1,14 +1,32 @@
 import path from "path";
 import fs from "fs";
 import { Collection } from "./collection.js";
+import { Index, IndexOptions } from "./index.js";
 import type { LioranManager } from "../LioranManager.js";
 import type { ZodSchema } from "zod";
 
+/* ----------------------------- TYPES ----------------------------- */
+
 type TXOp = { tx: number; col: string; op: string; args: any[] };
 type TXCommit = { tx: number; commit: true };
 type TXApplied = { tx: number; applied: true };
 type WALEntry = TXOp | TXCommit | TXApplied;
 
+type IndexMeta = {
+  field: string;
+  options: IndexOptions;
+};
+
+type DBMeta = {
+  version: number;
+  indexes: Record<string, IndexMeta[]>;
+};
+
+const META_FILE = "__db_meta.json";
+const META_VERSION = 1;
+
+/* ---------------------- TRANSACTION CONTEXT ---------------------- */
+
 class DBTransactionContext {
   private ops: TXOp[] = [];
 
@@ -41,12 +59,16 @@ class DBTransactionContext {
   }
 }
 
+/* ----------------------------- DATABASE ----------------------------- */
+
 export class LioranDB {
   basePath: string;
   dbName: string;
   manager: LioranManager;
   collections: Map<string, Collection>;
   private walPath: string;
+  private metaPath: string;
+  private meta!: DBMeta;
 
   private static TX_SEQ = 0;
 
@@ -56,12 +78,36 @@ export class LioranDB {
     this.manager = manager;
     this.collections = new Map();
     this.walPath = path.join(basePath, "__tx_wal.log");
+    this.metaPath = path.join(basePath, META_FILE);
 
     fs.mkdirSync(basePath, { recursive: true });
 
+    this.loadMeta();
     this.recoverFromWAL().catch(console.error);
   }
 
+  /* ------------------------- META ------------------------- */
+
+  private loadMeta() {
+    if (!fs.existsSync(this.metaPath)) {
+      this.meta = { version: META_VERSION, indexes: {} };
+      this.saveMeta();
+      return;
+    }
+
+    try {
+      this.meta = JSON.parse(fs.readFileSync(this.metaPath, "utf8"));
+    } catch {
+      throw new Error("Database metadata corrupted");
+    }
+  }
+
+  private saveMeta() {
+    fs.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
+  }
+
+  /* ------------------------- WAL ------------------------- */
+
   async writeWAL(entries: WALEntry[]) {
     const fd = await fs.promises.open(this.walPath, "a");
     for (const e of entries) {
@@ -97,14 +143,12 @@ export class LioranDB {
           ops.get(entry.tx)!.push(entry);
         }
       } catch {
-        // Ignore corrupted WAL tail
         break;
       }
     }
 
     for (const tx of committed) {
       if (applied.has(tx)) continue;
-
       const txOps = ops.get(tx);
       if (txOps) await this.applyTransaction(txOps);
     }
@@ -119,6 +163,8 @@ export class LioranDB {
     }
   }
 
+  /* ------------------------- COLLECTION ------------------------- */
+
   collection<T = any>(name: string, schema?: ZodSchema<T>): Collection<T> {
     if (this.collections.has(name)) {
       const col = this.collections.get(name)!;
@@ -130,10 +176,49 @@ export class LioranDB {
     fs.mkdirSync(colPath, { recursive: true });
 
     const col = new Collection<T>(colPath, schema);
+
+    // 🔥 Auto-load indexes for this collection
+    const metas = this.meta.indexes[name] ?? [];
+    for (const m of metas) {
+      col.registerIndex(new Index(colPath, m.field, m.options));
+    }
+
     this.collections.set(name, col);
     return col;
   }
 
+  /* ------------------------- INDEX API ------------------------- */
+
+  async createIndex(
+    collection: string,
+    field: string,
+    options: IndexOptions = {}
+  ) {
+    const col = this.collection(collection);
+
+    const existing = this.meta.indexes[collection]?.find(i => i.field === field);
+    if (existing) return;
+
+    const index = new Index(col.dir, field, options);
+
+    // 🔁 Build index from existing documents
+    for await (const [, enc] of col.db.iterator()) {
+      const doc = JSON.parse(Buffer.from(enc, "base64").subarray(32).toString("utf8"));
+      await index.insert(doc);
+    }
+
+    col.registerIndex(index);
+
+    if (!this.meta.indexes[collection]) {
+      this.meta.indexes[collection] = [];
+    }
+
+    this.meta.indexes[collection].push({ field, options });
+    this.saveMeta();
+  }
+
+  /* ------------------------- TX API ------------------------- */
+
   async transaction<T>(fn: (tx: DBTransactionContext) => Promise<T>): Promise<T> {
     const txId = ++LioranDB.TX_SEQ;
     const tx = new DBTransactionContext(this, txId);
@@ -144,6 +229,8 @@ export class LioranDB {
     return result;
   }
 
+  /* ------------------------- SHUTDOWN ------------------------- */
+
   async close(): Promise<void> {
     for (const col of this.collections.values()) {
       try { await col.close(); } catch {}

package/src/core/index.ts

@@ -0,0 +1,140 @@
+import path from "path";
+import fs from "fs";
+import { ClassicLevel } from "classic-level";
+
+/* ----------------------------- TYPES ----------------------------- */
+
+export interface IndexOptions {
+  unique?: boolean;
+}
+
+type IndexValue = string | string[];
+
+/* ----------------------------- INDEX ----------------------------- */
+
+export class Index {
+  readonly field: string;
+  readonly unique: boolean;
+  readonly dir: string;
+  readonly db: ClassicLevel<string, string>;
+
+  constructor(baseDir: string, field: string, options: IndexOptions = {}) {
+    this.field = field;
+    this.unique = !!options.unique;
+
+    this.dir = path.join(baseDir, "__indexes", field + ".idx");
+    fs.mkdirSync(this.dir, { recursive: true });
+
+    this.db = new ClassicLevel(this.dir, { valueEncoding: "utf8" });
+  }
+
+  /* ------------------------- INTERNAL ------------------------- */
+
+  private normalizeKey(value: any): string {
+    if (value === null || value === undefined) return "__null__";
+
+    if (typeof value === "object") {
+      return JSON.stringify(value);
+    }
+
+    return String(value);
+  }
+
+  private async getRaw(key: string): Promise<IndexValue | null> {
+    try {
+      const v = await this.db.get(key);
+      if (v === undefined) return null;
+      return JSON.parse(v);
+    } catch {
+      return null;
+    }
+  }
+
+  private async setRaw(key: string, value: IndexValue) {
+    await this.db.put(key, JSON.stringify(value));
+  }
+
+  private async delRaw(key: string) {
+    try { await this.db.del(key); } catch {}
+  }
+
+  /* --------------------------- API --------------------------- */
+
+  async insert(doc: any) {
+    const val = doc[this.field];
+    if (val === undefined) return;
+
+    const key = this.normalizeKey(val);
+
+    if (this.unique) {
+      const existing = await this.getRaw(key);
+      if (existing) {
+        throw new Error(
+          `Unique index violation on "${this.field}" = ${val}`
+        );
+      }
+
+      await this.setRaw(key, doc._id);
+      return;
+    }
+
+    const arr = (await this.getRaw(key)) as string[] | null;
+
+    if (!arr) {
+      await this.setRaw(key, [doc._id]);
+    } else {
+      if (!arr.includes(doc._id)) {
+        arr.push(doc._id);
+        await this.setRaw(key, arr);
+      }
+    }
+  }
+
+  async delete(doc: any) {
+    const val = doc[this.field];
+    if (val === undefined) return;
+
+    const key = this.normalizeKey(val);
+
+    if (this.unique) {
+      await this.delRaw(key);
+      return;
+    }
+
+    const arr = (await this.getRaw(key)) as string[] | null;
+    if (!arr) return;
+
+    const next = arr.filter(id => id !== doc._id);
+
+    if (next.length === 0) {
+      await this.delRaw(key);
+    } else {
+      await this.setRaw(key, next);
+    }
+  }
+
+  async update(oldDoc: any, newDoc: any) {
+    const oldVal = oldDoc?.[this.field];
+    const newVal = newDoc?.[this.field];
+
+    if (oldVal === newVal) return;
+
+    if (oldDoc) await this.delete(oldDoc);
+    if (newDoc) await this.insert(newDoc);
+  }
+
+  async find(value: any): Promise<string[]> {
+    const key = this.normalizeKey(value);
+
+    const raw = await this.getRaw(key);
+    if (!raw) return [];
+
+    if (this.unique) return [raw as string];
+
+    return raw as string[];
+  }
+
+  async close() {
+    try { await this.db.close(); } catch {}
+  }
+}

package/src/core/query.ts

@@ -2,6 +2,8 @@ function getByPath(obj: any, path: string): any {
   return path.split(".").reduce((o, p) => (o ? o[p] : undefined), obj);
 }
 
+/* ----------------------------- MATCH ENGINE ----------------------------- */
+
 export function matchDocument(doc: any, query: any): boolean {
   for (const key of Object.keys(query)) {
     const cond = query[key];
@@ -16,8 +18,7 @@ export function matchDocument(doc: any, query: any): boolean {
         if (op === "$lte" && !(val <= v)) return false;
         if (op === "$ne" && val === v) return false;
         if (op === "$eq" && val !== v) return false;
-        if (op === "$in" && (!Array.isArray(v) || !v.includes(val)))
-          return false;
+        if (op === "$in" && (!Array.isArray(v) || !v.includes(val))) return false;
       }
     } else {
       if (val !== cond) return false;
@@ -26,6 +27,8 @@ export function matchDocument(doc: any, query: any): boolean {
   return true;
 }
 
+/* ------------------------------ UPDATE ENGINE ------------------------------ */
+
 export function applyUpdate(oldDoc: any, update: any): any {
   const doc = structuredClone(oldDoc);
 
@@ -61,3 +64,103 @@ export function applyUpdate(oldDoc: any, update: any): any {
 
   return doc;
 }
+
+/* ------------------------------ INDEX ROUTER ------------------------------ */
+
+export function extractIndexQuery(query: any): { field: string; value: any } | null {
+  for (const key of Object.keys(query)) {
+    const cond = query[key];
+
+    // Skip _id as it's handled separately
+    if (key === "_id") continue;
+
+    // Simple equality: { field: value }
+    if (!cond || typeof cond !== "object" || Array.isArray(cond)) {
+      return { field: key, value: cond };
+    }
+
+    // $eq operator: { field: { $eq: value } }
+    if ("$eq" in cond) {
+      return { field: key, value: cond.$eq };
+    }
+  }
+
+  return null;
+}
+
+export interface IndexProvider {
+  findByIndex(
+    field: string,
+    value: any
+  ): Promise<Set<string> | null>;
+
+  rangeByIndex?(
+    field: string,
+    cond: any
+  ): Promise<Set<string> | null>;
+}
+
+/**
+ * Selects best possible index from query.
+ */
+export function selectIndex(query: any, indexes: Set<string>) {
+  for (const key of Object.keys(query)) {
+    if (!indexes.has(key)) continue;
+
+    const cond = query[key];
+
+    if (cond && typeof cond === "object" && !Array.isArray(cond)) {
+      return { field: key, cond };
+    }
+
+    return { field: key, cond: { $eq: cond } };
+  }
+
+  return null;
+}
+
+/**
+ * Executes indexed query or fallback full scan.
+ */
+export async function runIndexedQuery(
+  query: any,
+  indexProvider: IndexProvider,
+  allDocIds: () => Promise<string[]>
+): Promise<Set<string>> {
+  const indexes = (indexProvider as any).indexes as Set<string>;
+
+  if (!indexes?.size) {
+    return new Set(await allDocIds());
+  }
+
+  const sel = selectIndex(query, indexes);
+  if (!sel) {
+    return new Set(await allDocIds());
+  }
+
+  const { field, cond } = sel;
+
+  if ("$eq" in cond) {
+    return (await indexProvider.findByIndex(field, cond.$eq)) ??
+      new Set(await allDocIds());
+  }
+
+  if ("$in" in cond) {
+    const out = new Set<string>();
+    for (const v of cond.$in) {
+      const r = await indexProvider.findByIndex(field, v);
+      if (r) for (const id of r) out.add(id);
+    }
+    return out;
+  }
+
+  if (
+    indexProvider.rangeByIndex &&
+    ("$gt" in cond || "$gte" in cond || "$lt" in cond || "$lte" in cond)
+  ) {
+    return (await indexProvider.rangeByIndex(field, cond)) ??
+      new Set(await allDocIds());
+  }
+
+  return new Set(await allDocIds());
+}