@bod.ee/db 0.7.0

package/src/server/BodDB.ts

@@ -0,0 +1,319 @@
+import { StorageEngine } from './StorageEngine.ts';
+import { SubscriptionEngine, type SubscriptionCallback, type ChildCallback } from './SubscriptionEngine.ts';
+import { QueryEngine } from './QueryEngine.ts';
+import { RulesEngine, type PathRule } from './RulesEngine.ts';
+import { Transport, type TransportOptions } from './Transport.ts';
+import { FTSEngine, type FTSEngineOptions } from './FTSEngine.ts';
+import { VectorEngine, type VectorEngineOptions } from './VectorEngine.ts';
+import { StreamEngine, type CompactOptions } from './StreamEngine.ts';
+import { MQEngine, type MQEngineOptions } from './MQEngine.ts';
+import { validatePath } from '../shared/pathUtils.ts';
+
+export interface TransactionProxy {
+  get(path: string): unknown;
+  set(path: string, value: unknown): void;
+  update(updates: Record<string, unknown>): void;
+  delete(path: string): void;
+}
+
+export class BodDBOptions {
+  path: string = ':memory:';
+  /** Inline rules OR path to .json/.ts config file */
+  rules?: Record<string, PathRule> | string;
+  /** Index definitions: { "basePath": ["field1", "field2"] } */
+  indexes?: Record<string, string[]>;
+  /** TTL sweep interval in ms (0 = disabled, default 60000) */
+  sweepInterval: number = 60000;
+  /** FTS5 full-text search config */
+  fts?: Partial<FTSEngineOptions>;
+  /** Vector search config */
+  vectors?: Partial<VectorEngineOptions>;
+  /** Auto-compact stream topics on sweep: { "events/orders": { maxAge: 86400 } } */
+  compact?: Record<string, CompactOptions>;
+  /** Message queue config */
+  mq?: Partial<MQEngineOptions>;
+  port?: number;
+  auth?: TransportOptions['auth'];
+  transport?: Partial<TransportOptions>;
+}
+
+export class BodDB {
+  readonly storage: StorageEngine;
+  readonly subs: SubscriptionEngine;
+  readonly rules: RulesEngine;
+  readonly stream: StreamEngine;
+  readonly mq: MQEngine;
+  readonly fts: FTSEngine | null = null;
+  readonly vectors: VectorEngine | null = null;
+  readonly options: BodDBOptions;
+  private _transport: Transport | null = null;
+  get transport(): Transport | null { return this._transport; }
+  private sweepTimer: ReturnType<typeof setInterval> | null = null;
+
+  constructor(options?: Partial<BodDBOptions>) {
+    this.options = { ...new BodDBOptions(), ...options };
+    this.storage = new StorageEngine({ path: this.options.path });
+    this.subs = new SubscriptionEngine();
+    this.stream = new StreamEngine(this.storage, this.subs, { compact: this.options.compact });
+    this.mq = new MQEngine(this.storage, {
+      ...this.options.mq,
+      notify: (paths) => {
+        if (this.subs.hasSubscriptions) {
+          this.subs.notify(paths, (p) => this.storage.getRaw(p));
+        }
+      },
+    });
+
+    const rulesConfig = this.options.rules;
+    const resolvedRules = typeof rulesConfig === 'string'
+      ? BodDB.loadRulesFileSync(rulesConfig)
+      : (rulesConfig ?? {});
+    this.rules = new RulesEngine({ rules: resolvedRules });
+
+    // Auto-create indexes from config
+    if (this.options.indexes) {
+      for (const [basePath, fields] of Object.entries(this.options.indexes)) {
+        for (const field of fields) {
+          this.storage.createIndex(basePath, field);
+        }
+      }
+    }
+
+    // Initialize FTS if configured
+    if (this.options.fts) {
+      this.fts = new FTSEngine(this.storage.db, this.options.fts);
+    }
+
+    // Initialize vectors if configured
+    if (this.options.vectors) {
+      this.vectors = new VectorEngine(this.storage.db, this.options.vectors);
+    }
+
+    // Start TTL sweep
+    if (this.options.sweepInterval > 0) {
+      this.sweepTimer = setInterval(() => this.sweep(), this.options.sweepInterval);
+    }
+  }
+
+  /** Load rules from a JSON or TS file synchronously */
+  private static loadRulesFileSync(filePath: string): Record<string, PathRule> {
+    if (filePath.endsWith('.json')) {
+      const file = Bun.file(filePath);
+      // Use spawnSync to read synchronously since Bun.file is async
+      const text = require('fs').readFileSync(filePath, 'utf-8');
+      const parsed = JSON.parse(text);
+      return parsed.rules ?? parsed;
+    }
+    throw new Error(`Unsupported rules file format: ${filePath}. Use .json or call BodDB.create() for .ts files.`);
+  }
+
+  /** Async factory — accepts a config file path (string) or inline options */
+  static async create(configOrOptions?: string | Partial<BodDBOptions>): Promise<BodDB> {
+    const options = typeof configOrOptions === 'string'
+      ? await BodDB.loadConfigFile(configOrOptions)
+      : configOrOptions;
+
+    if (typeof options?.rules === 'string' && (options.rules.endsWith('.ts') || options.rules.endsWith('.js'))) {
+      const mod = await import(options.rules);
+      const rules = mod.rules ?? mod.default;
+      return new BodDB({ ...options, rules });
+    }
+    return new BodDB(options);
+  }
+
+  /** Load a full config file (.ts/.js/.json) → BodDBOptions */
+  private static async loadConfigFile(filePath: string): Promise<Partial<BodDBOptions>> {
+    const { resolve } = require('path');
+    const absPath = resolve(filePath);
+    if (filePath.endsWith('.ts') || filePath.endsWith('.js')) {
+      const mod = await import(absPath);
+      return mod.default ?? mod.config ?? mod;
+    }
+    if (filePath.endsWith('.json')) {
+      const text = require('fs').readFileSync(absPath, 'utf-8');
+      const parsed = JSON.parse(text);
+      return parsed.config ?? parsed;
+    }
+    throw new Error(`Unsupported config file format: ${filePath}. Use .ts, .js, or .json.`);
+  }
+
+  get(path: string): unknown {
+    return this.storage.get(path);
+  }
+
+  getShallow(path?: string): Array<{ key: string; isLeaf: boolean; value?: unknown }> {
+    return this.storage.getShallow(path);
+  }
+
+  set(path: string, value: unknown, options?: { ttl?: number }): void {
+    const p = validatePath(path);
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    const changed = this.storage.set(path, value);
+    if (options?.ttl) {
+      this.storage.setExpiry(path, options.ttl);
+    }
+    if (this.subs.hasSubscriptions) {
+      this.subs.notify(changed, (pp) => this.storage.get(pp), existedBefore);
+    }
+  }
+
+  /** Manually trigger TTL sweep + stream auto-compact, returns expired paths */
+  sweep(): string[] {
+    const expired = this.storage.sweep();
+    if (expired.length > 0 && this.subs.hasSubscriptions) {
+      this.subs.notify(expired, (p) => this.storage.get(p));
+    }
+    this.stream.autoCompact();
+    this.mq.sweep();
+    return expired;
+  }
+
+  update(updates: Record<string, unknown>): void {
+    const existedBefore = new Set<string>();
+    if (this.subs.hasChildSubscriptions) {
+      for (const path of Object.keys(updates)) {
+        const p = validatePath(path);
+        for (const ep of this.snapshotExisting(p)) existedBefore.add(ep);
+      }
+    }
+    const changed = this.storage.update(updates);
+    if (this.subs.hasSubscriptions) {
+      this.subs.notify(changed, (p) => this.storage.get(p), existedBefore);
+    }
+  }
+
+  delete(path: string): void {
+    const p = validatePath(path);
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    this.storage.delete(path);
+    if (this.subs.hasSubscriptions) {
+      this.subs.notify([p], (pp) => this.storage.get(pp), existedBefore);
+    }
+  }
+
+  /** Push a value with auto-generated time-sortable key (stored as single JSON row, not flattened) */
+  push(path: string, value: unknown, opts?: { idempotencyKey?: string }): string {
+    const p = validatePath(path);
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    const { key, changedPaths, duplicate } = this.storage.push(path, value, opts);
+    if (!duplicate && this.subs.hasSubscriptions) {
+      this.subs.notify(changedPaths, (pp) => this.storage.get(pp), existedBefore);
+    }
+    return key;
+  }
+
+  query(path: string): QueryEngine {
+    return new QueryEngine(this.storage, path);
+  }
+
+  /** Full-text search (requires fts config) */
+  search(options: { text: string; path?: string; limit?: number }): Array<{ path: string; data: unknown; rank: number }> {
+    if (!this.fts) throw new Error('FTS not configured. Pass { fts: {} } in options.');
+    const results = this.fts.search({ text: options.text, prefix: options.path, limit: options.limit });
+    return results.map(r => ({ path: r.path, data: this.storage.get(r.path), rank: r.rank }));
+  }
+
+  /** Index a path for full-text search */
+  index(path: string, content: string): void;
+  index(path: string, fields: string[]): void;
+  index(path: string, contentOrFields: string | string[]): void {
+    if (!this.fts) throw new Error('FTS not configured. Pass { fts: {} } in options.');
+    if (typeof contentOrFields === 'string') {
+      this.fts.index(path, contentOrFields);
+    } else {
+      const data = this.storage.get(path);
+      if (data && typeof data === 'object') {
+        this.fts.indexFields(path, data as Record<string, unknown>, contentOrFields);
+      }
+    }
+  }
+
+  /** Vector similarity search (requires vectors config) */
+  vectorSearch(options: { query: number[]; path?: string; limit?: number; threshold?: number }): Array<{ path: string; data: unknown; score: number }> {
+    if (!this.vectors) throw new Error('Vectors not configured. Pass { vectors: {} } in options.');
+    const results = this.vectors.search({ query: options.query, prefix: options.path, limit: options.limit, threshold: options.threshold });
+    return results.map(r => ({ path: r.path, data: this.storage.get(r.path), score: r.score }));
+  }
+
+  on(path: string, cb: SubscriptionCallback): () => void {
+    return this.subs.onValue(path, cb);
+  }
+
+  onChild(path: string, cb: ChildCallback): () => void {
+    return this.subs.onChild(path, cb);
+  }
+
+  /** Run multiple operations in a single SQLite transaction. Notifications fire after commit. */
+  transaction<T>(fn: (tx: TransactionProxy) => T): T {
+    const allChanged: string[] = [];
+    const allExistedBefore = new Set<string>();
+    const proxy: TransactionProxy = {
+      get: (path: string) => this.storage.get(path),
+      set: (path: string, value: unknown) => {
+        const p = validatePath(path);
+        if (this.subs.hasChildSubscriptions) {
+          for (const ep of this.snapshotExisting(p)) allExistedBefore.add(ep);
+        }
+        allChanged.push(...this.storage.set(path, value));
+      },
+      update: (updates: Record<string, unknown>) => {
+        if (this.subs.hasChildSubscriptions) {
+          for (const path of Object.keys(updates)) {
+            for (const ep of this.snapshotExisting(validatePath(path))) allExistedBefore.add(ep);
+          }
+        }
+        allChanged.push(...this.storage.update(updates));
+      },
+      delete: (path: string) => {
+        const p = validatePath(path);
+        if (this.subs.hasChildSubscriptions) {
+          for (const ep of this.snapshotExisting(p)) allExistedBefore.add(ep);
+        }
+        this.storage.delete(path);
+        allChanged.push(p);
+      },
+    };
+
+    const sqliteTx = this.storage.db.transaction(() => fn(proxy));
+    const result = sqliteTx();
+
+    if (this.subs.hasSubscriptions && allChanged.length > 0) {
+      this.subs.notify(allChanged, (p) => this.storage.get(p), allExistedBefore);
+    }
+    return result;
+  }
+
+  /** Start WebSocket + REST server */
+  serve(options?: Partial<TransportOptions>) {
+    const port = options?.port ?? this.options.port;
+    const auth = options?.auth ?? this.options.auth;
+    this._transport = new Transport(this, this.rules, { ...this.options.transport, ...options, port, auth });
+    return this._transport.start();
+  }
+
+  stop() {
+    this._transport?.stop();
+    this._transport = null;
+  }
+
+  close(): void {
+    if (this.sweepTimer) {
+      clearInterval(this.sweepTimer);
+      this.sweepTimer = null;
+    }
+    this.stop();
+    this.subs.clear();
+    this.storage.close();
+  }
+
+  private snapshotExisting(path: string): Set<string> {
+    const existing = new Set<string>();
+    if (this.storage.exists(path)) existing.add(path);
+    const parts = path.split('/');
+    for (let i = 1; i < parts.length; i++) {
+      const childPath = parts.slice(0, i + 1).join('/');
+      if (this.storage.exists(childPath)) existing.add(childPath);
+    }
+    return existing;
+  }
+}

package/src/server/ExpressionRules.ts

@@ -0,0 +1,250 @@
+/**
+ * Expression-based rules engine (V2).
+ * Parses Firebase-style string expressions into safe AST — no eval.
+ * Supports: auth.uid, auth.role, $wildcards, data, newData, comparisons, logical ops.
+ */
+
+// AST node types
+type Expr =
+  | { type: 'literal'; value: unknown }
+  | { type: 'path'; segments: string[] }  // auth.uid, data.name, $uid
+  | { type: 'binary'; op: string; left: Expr; right: Expr }
+  | { type: 'unary'; op: '!'; expr: Expr }
+  | { type: 'logical'; op: '&&' | '||'; left: Expr; right: Expr };
+
+// Tokenizer
+type Token =
+  | { type: 'ident'; value: string }
+  | { type: 'string'; value: string }
+  | { type: 'number'; value: number }
+  | { type: 'bool'; value: boolean }
+  | { type: 'null' }
+  | { type: 'op'; value: string }
+  | { type: 'dot' }
+  | { type: 'paren'; value: '(' | ')' }
+  | { type: 'not' };
+
+function tokenize(input: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+
+  while (i < input.length) {
+    const ch = input[i];
+
+    if (/\s/.test(ch)) { i++; continue; }
+
+    // Strings
+    if (ch === "'" || ch === '"') {
+      const quote = ch;
+      let str = '';
+      i++;
+      while (i < input.length && input[i] !== quote) {
+        if (input[i] === '\\') { i++; str += input[i] ?? ''; }
+        else str += input[i];
+        i++;
+      }
+      i++; // skip closing quote
+      tokens.push({ type: 'string', value: str });
+      continue;
+    }
+
+    // Numbers
+    if (/\d/.test(ch) || (ch === '-' && i + 1 < input.length && /\d/.test(input[i + 1]))) {
+      let num = '';
+      if (ch === '-') { num += ch; i++; }
+      while (i < input.length && /[\d.]/.test(input[i])) { num += input[i]; i++; }
+      tokens.push({ type: 'number', value: Number(num) });
+      continue;
+    }
+
+    // Operators (multi-char first)
+    const twoChar = input.slice(i, i + 2);
+    if (['===', '!=='].includes(input.slice(i, i + 3))) {
+      tokens.push({ type: 'op', value: input.slice(i, i + 3) });
+      i += 3;
+      continue;
+    }
+    if (['==', '!=', '<=', '>=', '&&', '||'].includes(twoChar)) {
+      tokens.push(twoChar === '&&' || twoChar === '||' ? { type: 'op', value: twoChar } : { type: 'op', value: twoChar });
+      i += 2;
+      continue;
+    }
+    if (['<', '>'].includes(ch)) {
+      tokens.push({ type: 'op', value: ch });
+      i++;
+      continue;
+    }
+
+    if (ch === '!') { tokens.push({ type: 'not' }); i++; continue; }
+    if (ch === '.') { tokens.push({ type: 'dot' }); i++; continue; }
+    if (ch === '(' || ch === ')') { tokens.push({ type: 'paren', value: ch }); i++; continue; }
+
+    // Identifiers (including $wildcards)
+    if (/[a-zA-Z_$]/.test(ch)) {
+      let ident = '';
+      while (i < input.length && /[a-zA-Z0-9_$]/.test(input[i])) { ident += input[i]; i++; }
+      if (ident === 'true') tokens.push({ type: 'bool', value: true });
+      else if (ident === 'false') tokens.push({ type: 'bool', value: false });
+      else if (ident === 'null') tokens.push({ type: 'null' });
+      else tokens.push({ type: 'ident', value: ident });
+      continue;
+    }
+
+    throw new Error(`Unexpected character '${ch}' in expression at position ${i}`);
+  }
+
+  return tokens;
+}
+
+// Parser (recursive descent)
+function parse(tokens: Token[]): Expr {
+  let pos = 0;
+
+  function peek(): Token | undefined { return tokens[pos]; }
+  function advance(): Token { return tokens[pos++]; }
+
+  function parseExpr(): Expr { return parseOr(); }
+
+  function parseOr(): Expr {
+    let left = parseAnd();
+    while (peek()?.type === 'op' && (peek() as any).value === '||') {
+      advance();
+      left = { type: 'logical', op: '||', left, right: parseAnd() };
+    }
+    return left;
+  }
+
+  function parseAnd(): Expr {
+    let left = parseComparison();
+    while (peek()?.type === 'op' && (peek() as any).value === '&&') {
+      advance();
+      left = { type: 'logical', op: '&&', left, right: parseComparison() };
+    }
+    return left;
+  }
+
+  function parseComparison(): Expr {
+    let left = parseUnary();
+    const t = peek();
+    if (t?.type === 'op' && ['==', '!=', '===', '!==', '<', '<=', '>', '>='].includes((t as any).value)) {
+      const op = (advance() as any).value;
+      return { type: 'binary', op, left, right: parseUnary() };
+    }
+    return left;
+  }
+
+  function parseUnary(): Expr {
+    if (peek()?.type === 'not') {
+      advance();
+      return { type: 'unary', op: '!', expr: parseUnary() };
+    }
+    return parsePrimary();
+  }
+
+  function parsePrimary(): Expr {
+    const t = peek();
+    if (!t) throw new Error('Unexpected end of expression');
+
+    if (t.type === 'paren' && t.value === '(') {
+      advance();
+      const expr = parseExpr();
+      if (peek()?.type !== 'paren' || (peek() as any).value !== ')') throw new Error('Expected )');
+      advance();
+      return expr;
+    }
+
+    if (t.type === 'string') { advance(); return { type: 'literal', value: t.value }; }
+    if (t.type === 'number') { advance(); return { type: 'literal', value: t.value }; }
+    if (t.type === 'bool') { advance(); return { type: 'literal', value: t.value }; }
+    if (t.type === 'null') { advance(); return { type: 'literal', value: null }; }
+
+    if (t.type === 'ident') {
+      const segments = [advance().value as string];
+      while (peek()?.type === 'dot' && tokens[pos + 1]?.type === 'ident') {
+        advance(); // dot
+        segments.push((advance() as any).value);
+      }
+      return { type: 'path', segments };
+    }
+
+    throw new Error(`Unexpected token: ${JSON.stringify(t)}`);
+  }
+
+  const result = parseExpr();
+  if (pos < tokens.length) throw new Error(`Unexpected token at position ${pos}: ${JSON.stringify(tokens[pos])}`);
+  return result;
+}
+
+// Evaluator
+interface EvalContext {
+  auth: Record<string, unknown> | null;
+  params: Record<string, string>;
+  data: unknown;
+  newData: unknown;
+}
+
+function evaluate(expr: Expr, ctx: EvalContext): unknown {
+  switch (expr.type) {
+    case 'literal':
+      return expr.value;
+
+    case 'path': {
+      const [root, ...rest] = expr.segments;
+      let val: unknown;
+      if (root.startsWith('$')) {
+        val = ctx.params[root.slice(1)] ?? null;
+      } else if (root === 'auth') {
+        val = ctx.auth;
+      } else if (root === 'data') {
+        val = ctx.data;
+      } else if (root === 'newData') {
+        val = ctx.newData;
+      } else {
+        return null;
+      }
+      for (const seg of rest) {
+        if (val == null || typeof val !== 'object') return null;
+        val = (val as Record<string, unknown>)[seg];
+      }
+      return val ?? null;
+    }
+
+    case 'binary': {
+      const l = evaluate(expr.left, ctx);
+      const r = evaluate(expr.right, ctx);
+      switch (expr.op) {
+        case '==': case '===': return l === r;
+        case '!=': case '!==': return l !== r;
+        case '<': return (l as number) < (r as number);
+        case '<=': return (l as number) <= (r as number);
+        case '>': return (l as number) > (r as number);
+        case '>=': return (l as number) >= (r as number);
+        default: return false;
+      }
+    }
+
+    case 'unary':
+      return !evaluate(expr.expr, ctx);
+
+    case 'logical': {
+      const lv = evaluate(expr.left, ctx);
+      if (expr.op === '&&') return lv ? evaluate(expr.right, ctx) : false;
+      return lv ? lv : evaluate(expr.right, ctx);
+    }
+  }
+}
+
+/** Compile an expression string into a reusable evaluator function */
+export function compileRule(expression: string | boolean): (ctx: EvalContext) => boolean {
+  if (typeof expression === 'boolean') return () => expression;
+  const ast = parse(tokenize(expression));
+  return (ctx) => {
+    try {
+      return !!evaluate(ast, ctx);
+    } catch {
+      return false;
+    }
+  };
+}
+
+export type { EvalContext };

package/src/server/FTSEngine.ts

@@ -0,0 +1,76 @@
+import type { Database } from 'bun:sqlite';
+
+export class FTSEngineOptions {
+  /** Auto-index writes for paths matching these prefixes */
+  indexPaths?: string[];
+  /** Fields to extract and index from written objects */
+  fields?: string[];
+}
+
+export class FTSEngine {
+  readonly options: FTSEngineOptions;
+  private db: Database;
+
+  constructor(db: Database, options?: Partial<FTSEngineOptions>) {
+    this.options = { ...new FTSEngineOptions(), ...options };
+    this.db = db;
+    this.init();
+  }
+
+  private init() {
+    this.db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(path, content)`);
+  }
+
+  /** Index a path with text content */
+  index(path: string, content: string): void {
+    // Upsert: delete then insert
+    this.db.run('DELETE FROM fts WHERE path = ?', [path]);
+    this.db.run('INSERT INTO fts (path, content) VALUES (?, ?)', [path, content]);
+  }
+
+  /** Index specific fields from an object */
+  indexFields(path: string, obj: Record<string, unknown>, fields?: string[]): void {
+    const fieldsToIndex = fields ?? this.options.fields;
+    if (!fieldsToIndex?.length) return;
+
+    const parts: string[] = [];
+    for (const field of fieldsToIndex) {
+      const val = obj[field];
+      if (typeof val === 'string') parts.push(val);
+    }
+    if (parts.length > 0) {
+      this.index(path, parts.join(' '));
+    }
+  }
+
+  /** Remove a path from the FTS index */
+  remove(path: string): void {
+    this.db.run('DELETE FROM fts WHERE path = ?', [path]);
+  }
+
+  /** Search for text, returns matching paths with rank */
+  search(options: { text: string; prefix?: string; limit?: number }): Array<{ path: string; rank: number }> {
+    let query = 'SELECT path, rank FROM fts WHERE fts MATCH ?';
+    const params: unknown[] = [options.text];
+
+    if (options.prefix) {
+      query += ' AND path >= ? AND path < ?';
+      params.push(options.prefix + '/', options.prefix + '/\uffff');
+    }
+
+    query += ' ORDER BY rank';
+
+    if (options.limit) {
+      query += ' LIMIT ?';
+      params.push(options.limit);
+    }
+
+    return this.db.prepare(query).all(...params) as Array<{ path: string; rank: number }>;
+  }
+
+  /** Check if a path should be auto-indexed based on config */
+  shouldAutoIndex(path: string): boolean {
+    if (!this.options.indexPaths?.length) return false;
+    return this.options.indexPaths.some(prefix => path.startsWith(prefix + '/') || path === prefix);
+  }
+}