@bod.ee/db 0.7.0

package/src/server/StorageEngine.ts

@@ -0,0 +1,464 @@
+import { Database } from 'bun:sqlite';
+import { validatePath, flatten, reconstruct, prefixEnd } from '../shared/pathUtils.ts';
+import type { QueryFilter, OrderClause } from '../shared/protocol.ts';
+import { isSentinel, resolveTransforms } from '../shared/transforms.ts';
+
+export class StorageEngineOptions {
+  path: string = ':memory:';
+}
+
+export class StorageEngine {
+  readonly db: Database;
+  readonly options: StorageEngineOptions;
+
+  private stmtGet!: ReturnType<Database['prepare']>;
+  private stmtGetRaw!: ReturnType<Database['prepare']>;
+  private stmtPrefix!: ReturnType<Database['prepare']>;
+  private stmtInsert!: ReturnType<Database['prepare']>;
+  private stmtDeleteExact!: ReturnType<Database['prepare']>;
+  private stmtDeletePrefix!: ReturnType<Database['prepare']>;
+  private stmtExistsPrefix!: ReturnType<Database['prepare']>;
+  private stmtSetExpiry!: ReturnType<Database['prepare']>;
+  private stmtSweep!: ReturnType<Database['prepare']>;
+  private stmtExpiredPaths!: ReturnType<Database['prepare']>;
+  private stmtPushIdempotent!: ReturnType<Database['prepare']>;
+  private stmtGetByIdempotencyKey!: ReturnType<Database['prepare']>;
+  private stmtQueryAfterKey!: ReturnType<Database['prepare']>;
+
+  constructor(options?: Partial<StorageEngineOptions>) {
+    this.options = { ...new StorageEngineOptions(), ...options };
+    this.db = new Database(this.options.path);
+    this.init();
+  }
+
+  private init() {
+    this.db.run('PRAGMA journal_mode = WAL');
+    this.db.run('PRAGMA synchronous = NORMAL');
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS nodes (
+        path TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        updated_at INTEGER NOT NULL
+      )
+    `);
+
+    // Migrations
+    const cols = this.db.prepare("PRAGMA table_info(nodes)").all() as Array<{ name: string }>;
+    if (!cols.some(c => c.name === 'expires_at')) {
+      this.db.run('ALTER TABLE nodes ADD COLUMN expires_at INTEGER');
+    }
+    if (!cols.some(c => c.name === 'idempotency_key')) {
+      this.db.run('ALTER TABLE nodes ADD COLUMN idempotency_key TEXT');
+    }
+    this.db.run('CREATE UNIQUE INDEX IF NOT EXISTS idx_idempotency ON nodes(idempotency_key) WHERE idempotency_key IS NOT NULL');
+
+    // MQ columns
+    if (!cols.some(c => c.name === 'mq_status'))          this.db.run('ALTER TABLE nodes ADD COLUMN mq_status TEXT');
+    if (!cols.some(c => c.name === 'mq_inflight_until'))   this.db.run('ALTER TABLE nodes ADD COLUMN mq_inflight_until INTEGER');
+    if (!cols.some(c => c.name === 'mq_delivery_count'))   this.db.run('ALTER TABLE nodes ADD COLUMN mq_delivery_count INTEGER DEFAULT 0');
+    this.db.run('CREATE INDEX IF NOT EXISTS idx_mq ON nodes(path, mq_status, mq_inflight_until) WHERE mq_status IS NOT NULL');
+
+    this.stmtGet = this.db.prepare('SELECT path, value FROM nodes WHERE path = ? AND mq_status IS NULL');
+    this.stmtGetRaw = this.db.prepare('SELECT path, value FROM nodes WHERE path = ?');
+    this.stmtPrefix = this.db.prepare('SELECT path, value FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL');
+    this.stmtInsert = this.db.prepare('INSERT OR REPLACE INTO nodes (path, value, updated_at) VALUES (?, ?, ?)');
+    this.stmtDeleteExact = this.db.prepare('DELETE FROM nodes WHERE path = ? AND mq_status IS NULL');
+    this.stmtDeletePrefix = this.db.prepare('DELETE FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL');
+    this.stmtExistsPrefix = this.db.prepare('SELECT 1 FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL LIMIT 1');
+    this.stmtSetExpiry = this.db.prepare('UPDATE nodes SET expires_at = ? WHERE path = ?');
+    this.stmtSweep = this.db.prepare('DELETE FROM nodes WHERE expires_at IS NOT NULL AND expires_at < ?');
+    this.stmtExpiredPaths = this.db.prepare('SELECT path FROM nodes WHERE expires_at IS NOT NULL AND expires_at < ?');
+    this.stmtPushIdempotent = this.db.prepare('INSERT OR IGNORE INTO nodes (path, value, updated_at, idempotency_key) VALUES (?, ?, ?, ?)');
+    this.stmtGetByIdempotencyKey = this.db.prepare('SELECT path FROM nodes WHERE idempotency_key = ?');
+    this.stmtQueryAfterKey = this.db.prepare('SELECT path, value FROM nodes WHERE path > ? AND path < ? AND mq_status IS NULL ORDER BY path ASC LIMIT ?');
+  }
+
+  /** Check if a path has any data (exact or subtree) */
+  exists(path: string): boolean {
+    path = validatePath(path);
+    const exact = this.stmtGet.get(path);
+    if (exact) return true;
+    const prefix = path + '/';
+    return !!this.stmtExistsPrefix.get(prefix, prefixEnd(prefix));
+  }
+
+  /** Get value at path (reconstructs subtree if needed) */
+  get(path: string, options?: { resolve?: boolean | string[] }): unknown {
+    path = validatePath(path);
+
+    // Try exact match first
+    const exact = this.stmtGet.get(path) as { path: string; value: string } | null;
+    let result: unknown;
+    if (exact) {
+      result = JSON.parse(exact.value);
+    } else {
+      // Try prefix (subtree)
+      const prefix = path + '/';
+      const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string; value: string }>;
+      if (rows.length === 0) return null;
+      result = reconstruct(path, rows);
+    }
+
+    if (options?.resolve && result !== null && typeof result === 'object') {
+      result = this.resolveRefs(result, options.resolve === true ? undefined : options.resolve);
+    }
+    return result;
+  }
+
+  /** Get value at path including MQ rows (for internal notification use) */
+  getRaw(path: string): unknown {
+    path = validatePath(path);
+    const exact = this.stmtGetRaw.get(path) as { path: string; value: string } | null;
+    if (exact) return JSON.parse(exact.value);
+    // Check subtree (including MQ rows)
+    const prefix = path + '/';
+    const row = this.db.prepare('SELECT 1 FROM nodes WHERE path >= ? AND path < ? LIMIT 1').get(prefix, prefixEnd(prefix));
+    if (row) return true; // signal existence without full reconstruction
+    return null;
+  }
+
+  /** Get immediate children of a path (one level deep). Returns { key, isLeaf, value? }[] */
+  getShallow(path?: string): Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number }> {
+    const prefix = path ? path + '/' : '';
+    const end = prefix + '\uffff';
+    const rows = (prefix
+      ? this.db.prepare('SELECT path, value, expires_at FROM nodes WHERE path >= ? AND path < ? ORDER BY path').all(prefix, end)
+      : this.db.prepare('SELECT path, value, expires_at FROM nodes ORDER BY path').all()
+    ) as Array<{ path: string; value: string; expires_at: number | null }>;
+
+    const children: Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number }> = [];
+    const seen = new Set<string>();
+    for (const row of rows) {
+      const rest = row.path.slice(prefix.length);
+      const slash = rest.indexOf('/');
+      const key = slash === -1 ? rest : rest.slice(0, slash);
+      if (!key || seen.has(key)) continue;
+      seen.add(key);
+      if (slash === -1) {
+        const entry: { key: string; isLeaf: boolean; value: unknown; ttl?: number } = { key, isLeaf: true, value: JSON.parse(row.value) };
+        if (row.expires_at) entry.ttl = Math.max(0, row.expires_at - Math.floor(Date.now() / 1000));
+        children.push(entry);
+      } else {
+        // Check if any child in this branch has TTL
+        const branchPrefix = prefix + key + '/';
+        const hasTTL = rows.some(r => r.path.startsWith(branchPrefix) && r.expires_at);
+        children.push(hasTTL ? { key, isLeaf: false, ttl: -1 } : { key, isLeaf: false });
+      }
+    }
+    return children;
+  }
+
+  /** Resolve _ref fields in a value tree */
+  private resolveRefs(value: unknown, fields?: string[]): unknown {
+    if (value === null || typeof value !== 'object' || Array.isArray(value)) return value;
+    const obj = value as Record<string, unknown>;
+    const result: Record<string, unknown> = {};
+    for (const [key, val] of Object.entries(obj)) {
+      if (val !== null && typeof val === 'object' && '_ref' in (val as Record<string, unknown>)) {
+        if (!fields || fields.includes(key)) {
+          const refPath = (val as { _ref: string })._ref;
+          result[key] = this.get(refPath);
+        } else {
+          result[key] = val;
+        }
+      } else if (typeof val === 'object' && val !== null && !Array.isArray(val)) {
+        result[key] = this.resolveRefs(val, fields);
+      } else {
+        result[key] = val;
+      }
+    }
+    return result;
+  }
+
+  /** Check if a value tree contains any sentinels */
+  private hasSentinels(value: unknown): boolean {
+    if (isSentinel(value)) return true;
+    if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+      return Object.values(value as Record<string, unknown>).some(v => this.hasSentinels(v));
+    }
+    return false;
+  }
+
+  /** Set value at path (flattens objects into leaf rows) */
+  set(path: string, value: unknown): string[] {
+    path = validatePath(path);
+    if (this.hasSentinels(value)) {
+      value = resolveTransforms(value, () => this.get(path));
+    }
+    const now = Date.now();
+    const leaves = flatten(path, value);
+
+    // Delete existing subtree + exact
+    this.stmtDeleteExact.run(path);
+    const prefix = path + '/';
+    this.stmtDeletePrefix.run(prefix, prefixEnd(prefix));
+
+    // Insert new leaves
+    const tx = this.db.transaction(() => {
+      for (const leaf of leaves) {
+        this.stmtInsert.run(leaf.path, leaf.value, now);
+      }
+    });
+    tx();
+
+    return leaves.map(l => l.path);
+  }
+
+  /** Delete value at path (exact + subtree) */
+  delete(path: string): void {
+    path = validatePath(path);
+    this.stmtDeleteExact.run(path);
+    const prefix = path + '/';
+    this.stmtDeletePrefix.run(prefix, prefixEnd(prefix));
+  }
+
+  /** Merge value at path — shallow merge for objects, replaces for primitives */
+  merge(path: string, value: unknown): string[] {
+    path = validatePath(path);
+    if (this.hasSentinels(value)) {
+      value = resolveTransforms(value, () => this.get(path));
+    }
+
+    // For non-objects, merge is the same as set
+    if (value === null || value === undefined || typeof value !== 'object' || Array.isArray(value)) {
+      return this.set(path, value);
+    }
+
+    // For objects: only delete+write the keys being provided, leave others intact
+    const now = Date.now();
+    const allPaths: string[] = [];
+    const obj = value as Record<string, unknown>;
+
+    const tx = this.db.transaction(() => {
+      for (const [key, val] of Object.entries(obj)) {
+        const childPath = `${path}/${key}`;
+        if (val === null || val === undefined) {
+          // null = delete this key
+          this.stmtDeleteExact.run(childPath);
+          const prefix = childPath + '/';
+          this.stmtDeletePrefix.run(prefix, prefixEnd(prefix));
+          allPaths.push(childPath);
+        } else {
+          // Replace only this key's subtree
+          this.stmtDeleteExact.run(childPath);
+          const prefix = childPath + '/';
+          this.stmtDeletePrefix.run(prefix, prefixEnd(prefix));
+          const leaves = flatten(childPath, val);
+          for (const leaf of leaves) {
+            this.stmtInsert.run(leaf.path, leaf.value, now);
+          }
+          allPaths.push(...leaves.map(l => l.path));
+        }
+      }
+      // Also clean up the exact parent if it was a leaf (now it has children)
+      this.stmtDeleteExact.run(path);
+    });
+    tx();
+
+    return allPaths;
+  }
+
+  /** Multi-path atomic update (merge semantics — shallow merge per path) */
+  update(updates: Record<string, unknown>): string[] {
+    const allPaths: string[] = [];
+    const tx = this.db.transaction(() => {
+      for (const [path, value] of Object.entries(updates)) {
+        if (value === null || value === undefined) {
+          this.delete(path);
+          allPaths.push(validatePath(path));
+        } else {
+          allPaths.push(...this.merge(path, value));
+        }
+      }
+    });
+    tx();
+    return allPaths;
+  }
+
+  /** Query children of a path with optional filters, ordering, limits */
+  query(
+    path: string,
+    filters?: QueryFilter[],
+    order?: OrderClause,
+    limit?: number,
+    offset?: number,
+  ): Array<{ _path: string; _key: string; [k: string]: unknown }> {
+    path = validatePath(path);
+    const prefix = path + '/';
+
+    // Get all rows under this path
+    const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string; value: string }>;
+    if (rows.length === 0) return [];
+
+    // Group rows by direct child key
+    const children = new Map<string, Array<{ path: string; value: string }>>();
+    for (const row of rows) {
+      const rel = row.path.slice(prefix.length);
+      const childKey = rel.split('/')[0];
+      if (!children.has(childKey)) children.set(childKey, []);
+      children.get(childKey)!.push(row);
+    }
+
+    // Reconstruct each child
+    let results: Array<{ _path: string; _key: string; [k: string]: unknown }> = [];
+    for (const [key, childRows] of children) {
+      const childPath = `${path}/${key}`;
+      const val = reconstruct(childPath, childRows);
+      if (val && typeof val === 'object' && !Array.isArray(val)) {
+        results.push({ _path: childPath, _key: key, ...(val as Record<string, unknown>) });
+      } else {
+        results.push({ _path: childPath, _key: key, _value: val });
+      }
+    }
+
+    // Apply filters
+    if (filters?.length) {
+      for (const f of filters) {
+        results = results.filter(item => {
+          const v = (item as Record<string, unknown>)[f.field];
+          switch (f.op) {
+            case '==': return v === f.value;
+            case '!=': return v !== f.value;
+            case '<': return (v as number) < (f.value as number);
+            case '<=': return (v as number) <= (f.value as number);
+            case '>': return (v as number) > (f.value as number);
+            case '>=': return (v as number) >= (f.value as number);
+            default: return true;
+          }
+        });
+      }
+    }
+
+    // Apply ordering
+    if (order) {
+      const dir = order.dir === 'desc' ? -1 : 1;
+      results.sort((a, b) => {
+        const av = (a as Record<string, unknown>)[order.field];
+        const bv = (b as Record<string, unknown>)[order.field];
+        if (av == null && bv == null) return 0;
+        if (av == null) return dir;
+        if (bv == null) return -dir;
+        return av < bv ? -dir : av > bv ? dir : 0;
+      });
+    }
+
+    // Apply offset + limit
+    if (offset) results = results.slice(offset);
+    if (limit) results = results.slice(0, limit);
+
+    return results;
+  }
+
+  /** Create expression index for a field under a path prefix. Field must be alphanumeric. */
+  createIndex(basePath: string, field: string) {
+    if (!/^[a-zA-Z_]\w*$/.test(field)) throw new Error(`Invalid index field: "${field}"`);
+    const p = validatePath(basePath);
+    const safeName = `idx_${p.replace(/[^a-zA-Z0-9]/g, '_')}_${field}`;
+    const prefix = p + '/';
+    const end = prefixEnd(prefix);
+    // SQLite requires literals in index expressions — field/path are validated above
+    this.db.run(
+      `CREATE INDEX IF NOT EXISTS "${safeName}" ON nodes(json_extract(value, '$.${field}')) WHERE path >= '${prefix}' AND path < '${end}'`,
+    );
+  }
+
+  /** Set expiry on all leaf paths under a given path */
+  setExpiry(path: string, ttlSeconds: number): void {
+    path = validatePath(path);
+    const expiresAt = Math.floor(Date.now() / 1000) + ttlSeconds;
+    // Set on exact match
+    this.stmtSetExpiry.run(expiresAt, path);
+    // Set on prefix matches
+    const prefix = path + '/';
+    const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string }>;
+    for (const row of rows) {
+      this.stmtSetExpiry.run(expiresAt, row.path);
+    }
+  }
+
+  /** Sweep expired entries, returns affected paths */
+  sweep(): string[] {
+    const now = Math.floor(Date.now() / 1000);
+    const expired = this.stmtExpiredPaths.all(now) as Array<{ path: string }>;
+    if (expired.length === 0) return [];
+    this.stmtSweep.run(now);
+    return expired.map(r => r.path);
+  }
+
+  /** Push a value as a single JSON row (not flattened) with auto-generated time-sortable key */
+  push(path: string, value: unknown, opts?: { idempotencyKey?: string }): { key: string; changedPaths: string[]; duplicate?: boolean } {
+    path = validatePath(path);
+    const key = generatePushId();
+    const fullPath = `${path}/${key}`;
+    const now = Date.now();
+
+    if (opts?.idempotencyKey) {
+      const existing = this.stmtGetByIdempotencyKey.get(opts.idempotencyKey) as { path: string } | null;
+      if (existing) {
+        const existingKey = existing.path.split('/').pop()!;
+        return { key: existingKey, changedPaths: [], duplicate: true };
+      }
+      this.stmtPushIdempotent.run(fullPath, JSON.stringify(value), now, opts.idempotencyKey);
+      return { key, changedPaths: [fullPath] };
+    }
+
+    this.stmtInsert.run(fullPath, JSON.stringify(value), now);
+    return { key, changedPaths: [fullPath] };
+  }
+
+  /** Query rows after a given key under a path prefix, ordered by path. For stream replay. */
+  queryAfterKey(path: string, afterKey: string | null, limit: number): Array<{ key: string; value: unknown }> {
+    path = validatePath(path);
+    const prefix = path + '/';
+    const startBound = afterKey ? `${path}/${afterKey}` : prefix;
+    const endBound = prefixEnd(prefix);
+    const rows = this.stmtQueryAfterKey.all(startBound, endBound, limit) as Array<{ path: string; value: string }>;
+    return rows.map(r => ({
+      key: r.path.slice(prefix.length),
+      value: JSON.parse(r.value),
+    }));
+  }
+
+  close() {
+    this.db.close();
+  }
+}
+
+// Firebase-style push ID: timestamp (base36) + random suffix
+const PUSH_CHARS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+let lastPushTime = 0;
+let lastRandChars: number[] = [];
+
+export function generatePushId(): string {
+  let now = Date.now();
+  const isDuplicate = now === lastPushTime;
+  lastPushTime = now;
+
+  const timeChars = new Array(8);
+  for (let i = 7; i >= 0; i--) {
+    timeChars[i] = PUSH_CHARS.charAt(now % 62);
+    now = Math.floor(now / 62);
+  }
+
+  let id = timeChars.join('');
+
+  if (!isDuplicate) {
+    lastRandChars = [];
+    for (let i = 0; i < 4; i++) {
+      lastRandChars.push(Math.floor(Math.random() * 62));
+    }
+  } else {
+    // Increment last random chars for guaranteed uniqueness
+    let i = lastRandChars.length - 1;
+    while (i >= 0 && lastRandChars[i] === 61) {
+      lastRandChars[i] = 0;
+      i--;
+    }
+    if (i >= 0) lastRandChars[i]++;
+  }
+
+  for (const c of lastRandChars) {
+    id += PUSH_CHARS.charAt(c);
+  }
+  return id;
+}