@bod.ee/db 0.7.0

package/src/server/MQEngine.ts

@@ -0,0 +1,286 @@
+import type { StorageEngine } from './StorageEngine.ts';
+import { generatePushId } from './StorageEngine.ts';
+import { validatePath, prefixEnd } from '../shared/pathUtils.ts';
+import type { Database } from 'bun:sqlite';
+
+export class MQEngineOptions {
+  visibilityTimeout: number = 30;   // seconds
+  maxDeliveries: number = 5;
+  dlqSuffix: string = '_dlq';
+  queues?: Record<string, Partial<MQEngineOptions>>;  // per-queue overrides, longest prefix match
+  /** Called after mutating operations with affected paths */
+  notify?: (paths: string[]) => void;
+}
+
+export interface MQMessage {
+  key: string;
+  path: string;
+  data: unknown;
+  deliveryCount: number;
+  status: 'pending' | 'inflight' | 'dead';
+}
+
+export class MQEngine {
+  readonly options: MQEngineOptions;
+  private db: Database;
+
+  private stmtPush: ReturnType<Database['prepare']>;
+  private stmtPushIdempotent: ReturnType<Database['prepare']>;
+  private stmtGetByIdempotencyKey: ReturnType<Database['prepare']>;
+  private stmtFetchCandidates: ReturnType<Database['prepare']>;
+  private stmtClaim: ReturnType<Database['prepare']>;
+  private stmtGetRow: ReturnType<Database['prepare']>;
+  private stmtAck: ReturnType<Database['prepare']>;
+  private stmtRelease: ReturnType<Database['prepare']>;
+  private stmtSweepFetch: ReturnType<Database['prepare']>;
+  private stmtPeek: ReturnType<Database['prepare']>;
+  private stmtDlqInsert: ReturnType<Database['prepare']>;
+  private stmtDlqDelete: ReturnType<Database['prepare']>;
+  private stmtPurge: ReturnType<Database['prepare']>;
+  private stmtPurgeAll: ReturnType<Database['prepare']>;
+
+  constructor(private storage: StorageEngine, options?: Partial<MQEngineOptions>) {
+    this.options = { ...new MQEngineOptions(), ...options };
+    this.db = storage.db;
+
+    this.stmtPush = this.db.prepare(
+      'INSERT INTO nodes (path, value, updated_at, mq_status, mq_delivery_count) VALUES (?, ?, ?, \'pending\', 0)'
+    );
+    this.stmtPushIdempotent = this.db.prepare(
+      'INSERT OR IGNORE INTO nodes (path, value, updated_at, mq_status, mq_delivery_count, idempotency_key) VALUES (?, ?, ?, \'pending\', 0, ?)'
+    );
+    this.stmtGetByIdempotencyKey = this.db.prepare(
+      'SELECT path FROM nodes WHERE idempotency_key = ?'
+    );
+    // Fetch pending OR expired inflight, ordered by path (chronological for push IDs)
+    this.stmtFetchCandidates = this.db.prepare(
+      `SELECT path, value, mq_delivery_count FROM nodes
+       WHERE path >= ? AND path < ?
+         AND mq_status IS NOT NULL
+         AND (mq_status = 'pending' OR (mq_status = 'inflight' AND mq_inflight_until < ?))
+       ORDER BY path ASC LIMIT ?`
+    );
+    // TOCTOU-safe claim: only update if still eligible
+    this.stmtClaim = this.db.prepare(
+      `UPDATE nodes SET mq_status = 'inflight', mq_inflight_until = ?, mq_delivery_count = mq_delivery_count + 1
+       WHERE path = ? AND mq_status IS NOT NULL
+         AND (mq_status = 'pending' OR (mq_status = 'inflight' AND mq_inflight_until < ?))`
+    );
+    this.stmtGetRow = this.db.prepare(
+      'SELECT path, value, mq_delivery_count FROM nodes WHERE path = ? AND mq_status IS NOT NULL'
+    );
+    this.stmtAck = this.db.prepare(
+      'DELETE FROM nodes WHERE path = ? AND mq_status IS NOT NULL'
+    );
+    this.stmtRelease = this.db.prepare(
+      `UPDATE nodes SET mq_status = 'pending', mq_inflight_until = NULL WHERE path = ? AND mq_status = 'inflight'`
+    );
+    this.stmtSweepFetch = this.db.prepare(
+      `SELECT path, value, mq_delivery_count FROM nodes
+       WHERE mq_status = 'inflight' AND mq_inflight_until < ?`
+    );
+    this.stmtPeek = this.db.prepare(
+      `SELECT path, value, mq_delivery_count, mq_status FROM nodes
+       WHERE path >= ? AND path < ? AND mq_status IS NOT NULL
+       ORDER BY path ASC LIMIT ?`
+    );
+    this.stmtDlqInsert = this.db.prepare(
+      'INSERT OR REPLACE INTO nodes (path, value, updated_at, mq_status) VALUES (?, ?, ?, \'dead\')'
+    );
+    this.stmtDlqDelete = this.db.prepare(
+      'DELETE FROM nodes WHERE path = ?'
+    );
+    this.stmtPurge = this.db.prepare(
+      `DELETE FROM nodes WHERE path >= ? AND path < ? AND mq_status = 'pending'`
+    );
+    this.stmtPurgeAll = this.db.prepare(
+      `DELETE FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NOT NULL`
+    );
+  }
+
+  /** Push a message onto the queue. Returns the generated key. */
+  push(queue: string, value: unknown, opts?: { idempotencyKey?: string }): string {
+    queue = validatePath(queue);
+    const key = generatePushId();
+    const fullPath = `${queue}/${key}`;
+    const now = Date.now();
+    const json = JSON.stringify(value);
+
+    if (opts?.idempotencyKey) {
+      const existing = this.stmtGetByIdempotencyKey.get(opts.idempotencyKey) as { path: string } | null;
+      if (existing) return existing.path.split('/').pop()!;
+      this.stmtPushIdempotent.run(fullPath, json, now, opts.idempotencyKey);
+    } else {
+      this.stmtPush.run(fullPath, json, now);
+    }
+    this.emit([fullPath, queue]);
+    return key;
+  }
+
+  /** Fetch and claim up to `count` messages. Atomic via transaction. */
+  fetch(queue: string, count: number = 1): MQMessage[] {
+    queue = validatePath(queue);
+    const prefix = queue + '/';
+    const end = prefixEnd(prefix);
+    const opts = this.optionsFor(queue);
+    const nowMs = Date.now();
+    const inflightUntil = nowMs + opts.visibilityTimeout * 1000;
+
+    const messages: MQMessage[] = [];
+    const tx = this.db.transaction(() => {
+      const candidates = this.stmtFetchCandidates.all(prefix, end, nowMs, count) as Array<{ path: string; value: string; mq_delivery_count: number }>;
+      for (const row of candidates) {
+        const result = this.stmtClaim.run(inflightUntil, row.path, nowMs);
+        if (result.changes > 0) {
+          // Re-read to get updated delivery count
+          const updated = this.stmtGetRow.get(row.path) as { path: string; value: string; mq_delivery_count: number } | null;
+          if (updated) {
+            const key = updated.path.slice(prefix.length);
+            messages.push({
+              key,
+              path: updated.path,
+              data: JSON.parse(updated.value),
+              deliveryCount: updated.mq_delivery_count,
+              status: 'inflight',
+            });
+          }
+        }
+      }
+    });
+    tx();
+    return messages;
+  }
+
+  /** Acknowledge (delete) a processed message. */
+  ack(queue: string, key: string): void {
+    queue = validatePath(queue);
+    const fullPath = `${queue}/${key}`;
+    this.stmtAck.run(fullPath);
+    this.emit([fullPath, queue]);
+  }
+
+  /** Release a message back to pending (negative ack). */
+  nack(queue: string, key: string): void {
+    queue = validatePath(queue);
+    const fullPath = `${queue}/${key}`;
+    this.stmtRelease.run(fullPath);
+    this.emit([fullPath, queue]);
+  }
+
+  /** Peek at messages without claiming them. */
+  peek(queue: string, count: number = 10): MQMessage[] {
+    queue = validatePath(queue);
+    const prefix = queue + '/';
+    const end = prefixEnd(prefix);
+    const dlqPrefix = queue + '/' + this.options.dlqSuffix;
+
+    const rows = this.stmtPeek.all(prefix, end, count) as Array<{ path: string; value: string; mq_delivery_count: number; mq_status: string }>;
+    return rows
+      .filter(r => !r.path.startsWith(dlqPrefix))
+      .map(r => ({
+        key: r.path.slice(prefix.length),
+        path: r.path,
+        data: JSON.parse(r.value),
+        deliveryCount: r.mq_delivery_count,
+        status: r.mq_status as 'pending' | 'inflight',
+      }));
+  }
+
+  /** Read dead letter queue messages. */
+  dlq(queue: string): MQMessage[] {
+    queue = validatePath(queue);
+    const dlqPath = `${queue}/${this.options.dlqSuffix}`;
+    const prefix = dlqPath + '/';
+    const end = prefixEnd(prefix);
+
+    const rows = this.db.prepare(
+      'SELECT path, value FROM nodes WHERE path >= ? AND path < ? AND mq_status = \'dead\' ORDER BY path ASC'
+    ).all(prefix, end) as Array<{ path: string; value: string }>;
+
+    return rows.map(r => ({
+      key: r.path.slice(prefix.length),
+      path: r.path,
+      data: JSON.parse(r.value),
+      deliveryCount: 0,
+      status: 'dead' as const,
+    }));
+  }
+
+  /** Delete messages in a queue. Default: pending only. Pass { all: true } to include inflight + DLQ. */
+  purge(queue: string, opts?: { all?: boolean }): number {
+    queue = validatePath(queue);
+    const prefix = queue + '/';
+    const end = prefixEnd(prefix);
+    const result = (opts?.all ? this.stmtPurgeAll : this.stmtPurge).run(prefix, end);
+    if (result.changes > 0) this.emit([queue]);
+    return result.changes;
+  }
+
+  /** Reclaim expired inflight messages; move exhausted ones to DLQ. */
+  sweep(): { reclaimed: number; dlqd: number } {
+    const nowMs = Date.now();
+    let reclaimed = 0;
+    let dlqd = 0;
+
+    const expired = this.stmtSweepFetch.all(nowMs) as Array<{ path: string; value: string; mq_delivery_count: number }>;
+
+    const tx = this.db.transaction(() => {
+      for (const row of expired) {
+        // Determine which queue this belongs to and get options
+        const opts = this.optionsForPath(row.path);
+        if (row.mq_delivery_count >= opts.maxDeliveries) {
+          // Move to DLQ
+          const dlqPath = this.dlqPathForMessage(row.path, opts.dlqSuffix);
+          this.stmtDlqInsert.run(dlqPath, row.value, Date.now());
+          this.stmtDlqDelete.run(row.path);
+          dlqd++;
+        } else {
+          // Release back to pending
+          this.stmtRelease.run(row.path);
+          reclaimed++;
+        }
+      }
+    });
+    tx();
+
+    if (expired.length > 0) {
+      this.emit(expired.map(r => r.path));
+    }
+    return { reclaimed, dlqd };
+  }
+
+  private emit(paths: string[]): void {
+    this.options.notify?.(paths);
+  }
+
+  /** Get effective options for a queue path (longest prefix match from queues config). */
+  private optionsFor(queue: string): MQEngineOptions {
+    if (!this.options.queues) return this.options;
+    let bestMatch = '';
+    let bestOpts: Partial<MQEngineOptions> | undefined;
+    for (const [prefix, opts] of Object.entries(this.options.queues)) {
+      if (queue.startsWith(prefix) && prefix.length > bestMatch.length) {
+        bestMatch = prefix;
+        bestOpts = opts;
+      }
+    }
+    return bestOpts ? { ...this.options, ...bestOpts } as MQEngineOptions : this.options;
+  }
+
+  /** Get options for a full message path (e.g. queues/jobs/abc123). */
+  private optionsForPath(fullPath: string): MQEngineOptions {
+    // Try to find the queue by stripping the last segment (the push key)
+    const lastSlash = fullPath.lastIndexOf('/');
+    if (lastSlash < 0) return this.options;
+    const queue = fullPath.slice(0, lastSlash);
+    return this.optionsFor(queue);
+  }
+
+  /** Build DLQ path from a message's full path. */
+  private dlqPathForMessage(fullPath: string, dlqSuffix: string): string {
+    const lastSlash = fullPath.lastIndexOf('/');
+    const queue = fullPath.slice(0, lastSlash);
+    const key = fullPath.slice(lastSlash + 1);
+    return `${queue}/${dlqSuffix}/${key}`;
+  }
+}

package/src/server/QueryEngine.ts

@@ -0,0 +1,45 @@
+import type { QueryFilter, OrderClause } from '../shared/protocol.ts';
+import type { StorageEngine } from './StorageEngine.ts';
+
+/** Fluent query builder that delegates to StorageEngine.query */
+export class QueryEngine {
+  private filters: QueryFilter[] = [];
+  private orderClause?: OrderClause;
+  private limitVal?: number;
+  private offsetVal?: number;
+
+  constructor(
+    private storage: StorageEngine,
+    private basePath: string,
+  ) {}
+
+  where(field: string, op: QueryFilter['op'], value: unknown): QueryEngine {
+    this.filters.push({ field, op, value });
+    return this;
+  }
+
+  order(field: string, dir: 'asc' | 'desc' = 'asc'): QueryEngine {
+    this.orderClause = { field, dir };
+    return this;
+  }
+
+  limit(n: number): QueryEngine {
+    this.limitVal = n;
+    return this;
+  }
+
+  offset(n: number): QueryEngine {
+    this.offsetVal = n;
+    return this;
+  }
+
+  get() {
+    return this.storage.query(
+      this.basePath,
+      this.filters.length ? this.filters : undefined,
+      this.orderClause,
+      this.limitVal,
+      this.offsetVal,
+    );
+  }
+}

package/src/server/RulesEngine.ts

@@ -0,0 +1,108 @@
+import { normalizePath } from '../shared/pathUtils.ts';
+import { compileRule } from './ExpressionRules.ts';
+
+export interface RuleContext {
+  auth: Record<string, unknown> | null;
+  path: string;
+  params: Record<string, string>;
+  data: unknown;
+  newData: unknown;
+}
+
+/** Rule can be a function (V1) or expression string/boolean (V2) */
+export interface PathRule {
+  read?: ((ctx: RuleContext) => boolean) | string | boolean;
+  write?: ((ctx: RuleContext) => boolean) | string | boolean;
+}
+
+export class RulesEngineOptions {
+  rules: Record<string, PathRule> = {};
+}
+
+interface CompiledPathRule {
+  read?: (ctx: RuleContext) => boolean;
+  write?: (ctx: RuleContext) => boolean;
+}
+
+export class RulesEngine {
+  readonly options: RulesEngineOptions;
+  private compiled: Map<string, CompiledPathRule>;
+
+  constructor(options?: Partial<RulesEngineOptions>) {
+    this.options = { ...new RulesEngineOptions(), ...options };
+    this.compiled = new Map();
+
+    // Pre-compile expression rules
+    for (const [pattern, rule] of Object.entries(this.options.rules)) {
+      const compiled: CompiledPathRule = {};
+      if (rule.read !== undefined) {
+        compiled.read = typeof rule.read === 'function' ? rule.read : compileRule(rule.read);
+      }
+      if (rule.write !== undefined) {
+        compiled.write = typeof rule.write === 'function' ? rule.write : compileRule(rule.write);
+      }
+      this.compiled.set(pattern, compiled);
+    }
+  }
+
+  /** Check if an operation is allowed. Returns true if no rules match (open by default). */
+  check(
+    op: 'read' | 'write',
+    path: string,
+    auth: Record<string, unknown> | null,
+    data?: unknown,
+    newData?: unknown,
+  ): boolean {
+    path = normalizePath(path);
+    const pathParts = path.split('/');
+
+    let bestMatch: { rule: CompiledPathRule; params: Record<string, string>; specificity: number } | null = null;
+
+    for (const [pattern, rule] of this.compiled) {
+      const patternParts = normalizePath(pattern).split('/');
+      const params = matchPattern(patternParts, pathParts);
+      if (params !== null) {
+        const specificity = patternParts.length;
+        if (!bestMatch || specificity > bestMatch.specificity) {
+          bestMatch = { rule, params, specificity };
+        }
+      }
+    }
+
+    if (!bestMatch) return true;
+
+    const fn = bestMatch.rule[op];
+    if (fn === undefined) return true;
+
+    const ctx: RuleContext = {
+      auth,
+      path,
+      params: bestMatch.params,
+      data: data ?? null,
+      newData: newData ?? null,
+    };
+
+    try {
+      return fn(ctx);
+    } catch {
+      return false;
+    }
+  }
+}
+
+function matchPattern(
+  patternParts: string[],
+  pathParts: string[],
+): Record<string, string> | null {
+  if (patternParts.length > pathParts.length) return null;
+
+  const params: Record<string, string> = {};
+  for (let i = 0; i < patternParts.length; i++) {
+    if (patternParts[i].startsWith('$')) {
+      params[patternParts[i].slice(1)] = pathParts[i];
+    } else if (patternParts[i] !== pathParts[i]) {
+      return null;
+    }
+  }
+  return params;
+}