@bod.ee/db 0.11.1 → 0.12.2

package/src/server/Transport.ts

@@ -159,7 +159,14 @@ export class Transport {
         if (this.options.keyAuth && path.startsWith(AUTH_PREFIX)) {
           return Response.json({ ok: false, error: `Write to ${AUTH_PREFIX} is reserved`, code: Errors.PERMISSION_DENIED }, { status: 403 });
         }
-        if (this.db.replication?.isReplica) {
+        const auth = await this.extractAuth(req);
+        if (this.rules && !this.rules.check('write', path, auth)) {
+          return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
+        }
+        if (this.db.replication?.shouldRejectPath(path)) {
+          return Response.json({ ok: false, error: 'Path is readonly', code: Errors.PERMISSION_DENIED }, { status: 403 });
+        }
+        if (this.db.replication?.shouldProxyPath(path)) {
           try {
             const body = await req.json();
             await this.db.replication.proxyWrite({ op: 'set', path, value: body });
@@ -168,10 +175,6 @@ export class Transport {
             return Response.json({ ok: false, error: e.message, code: Errors.INTERNAL }, { status: 500 });
           }
         }
-        const auth = await this.extractAuth(req);
-        if (this.rules && !this.rules.check('write', path, auth)) {
-          return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
-        }
         const body = await req.json();
         this.db.set(path, body);
         return Response.json({ ok: true });
@@ -185,7 +188,14 @@ export class Transport {
         if (this.options.keyAuth && path.startsWith(AUTH_PREFIX)) {
           return Response.json({ ok: false, error: `Write to ${AUTH_PREFIX} is reserved`, code: Errors.PERMISSION_DENIED }, { status: 403 });
         }
-        if (this.db.replication?.isReplica) {
+        const auth = await this.extractAuth(req);
+        if (this.rules && !this.rules.check('write', path, auth)) {
+          return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
+        }
+        if (this.db.replication?.shouldRejectPath(path)) {
+          return Response.json({ ok: false, error: 'Path is readonly', code: Errors.PERMISSION_DENIED }, { status: 403 });
+        }
+        if (this.db.replication?.shouldProxyPath(path)) {
           try {
             await this.db.replication.proxyWrite({ op: 'delete', path });
             return Response.json({ ok: true });
@@ -193,10 +203,6 @@ export class Transport {
             return Response.json({ ok: false, error: e.message, code: Errors.INTERNAL }, { status: 500 });
           }
         }
-        const auth = await this.extractAuth(req);
-        if (this.rules && !this.rules.check('write', path, auth)) {
-          return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
-        }
         this.db.delete(path);
         return Response.json({ ok: true });
       })();
@@ -418,13 +424,14 @@ export class Transport {
 
             case 'set': {
               if (guardAuthPrefix(msg.path)) return;
-              if (self.db.replication?.isReplica) {
-                try { return reply(await self.db.replication.proxyWrite(msg)); }
-                catch (e: any) { return error(e.message, Errors.INTERNAL); }
-              }
               if (self.rules && !self.rules.check('write', msg.path, ws.data.auth, self.db.get(msg.path), msg.value)) {
                 return error('Permission denied', Errors.PERMISSION_DENIED);
               }
+              if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
+              if (self.db.replication?.shouldProxyPath(msg.path)) {
+                try { return reply(await self.db.replication.proxyWrite(msg)); }
+                catch (e: any) { return error(e.message, Errors.INTERNAL); }
+              }
               self.db.set(msg.path, msg.value, msg.ttl ? { ttl: msg.ttl } : undefined);
               return reply(null);
             }
@@ -433,28 +440,34 @@ export class Transport {
               for (const path of Object.keys(msg.updates)) {
                 if (guardAuthPrefix(path)) return;
               }
-              if (self.db.replication?.isReplica) {
-                try { return reply(await self.db.replication.proxyWrite(msg)); }
-                catch (e: any) { return error(e.message, Errors.INTERNAL); }
-              }
               for (const path of Object.keys(msg.updates)) {
                 if (self.rules && !self.rules.check('write', path, ws.data.auth)) {
                   return error(`Permission denied for ${path}`, Errors.PERMISSION_DENIED);
                 }
               }
+              // Per-path: if ANY path needs reject, reject; if ANY needs proxy, proxy all (known limitation: greedy)
+              if (self.db.replication) {
+                const paths = Object.keys(msg.updates);
+                if (paths.some(p => self.db.replication!.shouldRejectPath(p))) return error('Path is readonly', Errors.PERMISSION_DENIED);
+                if (paths.some(p => self.db.replication!.shouldProxyPath(p))) {
+                  try { return reply(await self.db.replication.proxyWrite(msg)); }
+                  catch (e: any) { return error(e.message, Errors.INTERNAL); }
+                }
+              }
               self.db.update(msg.updates);
               return reply(null);
             }
 
             case 'delete': {
               if (guardAuthPrefix(msg.path)) return;
-              if (self.db.replication?.isReplica) {
-                try { return reply(await self.db.replication.proxyWrite(msg)); }
-                catch (e: any) { return error(e.message, Errors.INTERNAL); }
-              }
               if (self.rules && !self.rules.check('write', msg.path, ws.data.auth)) {
                 return error('Permission denied', Errors.PERMISSION_DENIED);
               }
+              if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
+              if (self.db.replication?.shouldProxyPath(msg.path)) {
+                try { return reply(await self.db.replication.proxyWrite(msg)); }
+                catch (e: any) { return error(e.message, Errors.INTERNAL); }
+              }
               self.db.delete(msg.path);
               return reply(null);
             }
@@ -502,9 +515,23 @@ export class Transport {
             }
 
             case 'batch': {
-              if (self.db.replication?.isReplica) {
-                try { return reply(await self.db.replication.proxyWrite(msg)); }
-                catch (e: any) { return error(e.message, Errors.INTERNAL); }
+              // Upfront rules check before proxy (defense-in-depth)
+              for (const batchOp of msg.operations) {
+                const opPaths = batchOp.op === 'update' ? Object.keys(batchOp.updates) : [batchOp.path];
+                for (const p of opPaths) {
+                  if (self.rules && !self.rules.check('write', p, ws.data.auth)) {
+                    return error(`Permission denied for ${p}`, Errors.PERMISSION_DENIED);
+                  }
+                }
+              }
+              // Per-path topology: if ANY path needs reject, reject; if ANY needs proxy, proxy all (known limitation: greedy)
+              if (self.db.replication) {
+                const batchPaths = msg.operations.map((o: any) => o.path != null ? [o.path] : (o.updates ? Object.keys(o.updates) : [])).flat();
+                if (batchPaths.some((p: string) => self.db.replication!.shouldRejectPath(p))) return error('Path is readonly', Errors.PERMISSION_DENIED);
+                if (batchPaths.some((p: string) => self.db.replication!.shouldProxyPath(p))) {
+                  try { return reply(await self.db.replication.proxyWrite(msg)); }
+                  catch (e: any) { return error(e.message, Errors.INTERNAL); }
+                }
               }
               const results: unknown[] = [];
               self.db.transaction((tx) => {
@@ -546,13 +573,14 @@ export class Transport {
 
             case 'push': {
               if (guardAuthPrefix(msg.path)) return;
-              if (self.db.replication?.isReplica) {
-                try { return reply(await self.db.replication.proxyWrite(msg)); }
-                catch (e: any) { return error(e.message, Errors.INTERNAL); }
-              }
               if (self.rules && !self.rules.check('write', msg.path, ws.data.auth)) {
                 return error('Permission denied', Errors.PERMISSION_DENIED);
               }
+              if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
+              if (self.db.replication?.shouldProxyPath(msg.path)) {
+                try { return reply(await self.db.replication.proxyWrite(msg)); }
+                catch (e: any) { return error(e.message, Errors.INTERNAL); }
+              }
               const key = self.db.push(msg.path, msg.value, msg.idempotencyKey ? { idempotencyKey: msg.idempotencyKey } : undefined);
               return reply(key);
             }
@@ -753,6 +781,14 @@ export class Transport {
               }
               return reply(self.db.vfs.list(msg.path));
             }
+            case 'vfs-tree': {
+              if (!self.db.vfs) return error('VFS not configured', Errors.INTERNAL);
+              if (self.rules && !self.rules.check('read', `_vfs/${msg.path}`, ws.data.auth)) {
+                return error('Permission denied', Errors.PERMISSION_DENIED);
+              }
+              const hiddenPaths = msg.hiddenPaths ? new Set(msg.hiddenPaths) : undefined;
+              return reply(self.db.vfs.tree(msg.path, { hiddenPaths, hideDotfiles: msg.hideDotfiles }));
+            }
             case 'vfs-delete': {
               if (!self.db.vfs) return error('VFS not configured', Errors.INTERNAL);
               if (self.rules && !self.rules.check('write', `_vfs/${msg.path}`, ws.data.auth)) {

package/src/server/VFSEngine.ts

@@ -133,9 +133,14 @@ export class VFSEngine {
 
   async write(virtualPath: string, data: Uint8Array, mime?: string): Promise<FileStat> {
     const vp = normalizePath(virtualPath);
-    const existing = this.db.get(this.metaPath(vp)) as Record<string, unknown> | null;
+    const existing = this.db.get(this.metaPath(vp)) as FileStat | null;
     const fileId = this.options.pathAsFileId ? vp : ((existing?.fileId as string) || generatePushId());
 
+    const hash = await computeHash(data);
+
+    // Skip write entirely if content hasn't changed (prevents unnecessary replication events)
+    if (existing?.hash === hash) return existing;
+
     await this.backend.write(fileId, data);
 
     const name = vp.split('/').pop()!;
@@ -147,6 +152,7 @@ export class VFSEngine {
       mtime: Date.now(),
       fileId,
       isDir: false,
+      hash,
     };
     this.db.set(this.metaPath(vp), stat);
     return stat;
@@ -183,6 +189,33 @@ export class VFSEngine {
     return results;
   }
 
+  tree(virtualPath: string, opts?: { hiddenPaths?: Set<string>; hideDotfiles?: boolean }): any[] {
+    const hiddenPaths = opts?.hiddenPaths ?? new Set<string>();
+    const hideDotfiles = opts?.hideDotfiles ?? false;
+
+    const walk = (vPath: string): any[] => {
+      const stats = this.list(vPath);
+      stats.sort((a, b) => {
+        if (a.isDir !== b.isDir) return a.isDir ? -1 : 1;
+        return a.name.localeCompare(b.name);
+      });
+      const result: any[] = [];
+      for (const s of stats) {
+        if (hiddenPaths.has(s.name)) continue;
+        if (hideDotfiles && s.name.startsWith('.')) continue;
+        if (s.isDir) {
+          const childPath = vPath ? `${vPath}/${s.name}` : s.name;
+          result.push({ name: s.name, type: 'directory', children: walk(childPath) });
+        } else {
+          result.push({ name: s.name, type: 'file' });
+        }
+      }
+      return result;
+    };
+
+    return walk(normalizePath(virtualPath));
+  }
+
   mkdir(virtualPath: string): FileStat {
     const vp = normalizePath(virtualPath);
     const name = vp.split('/').pop()!;
@@ -224,13 +257,18 @@ export class VFSEngine {
 
     const newName = dstPath.split('/').pop()!;
     const fileId = this.options.pathAsFileId ? dstPath : meta.fileId;
-    const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now() };
+    const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now(), hash: meta.hash };
     this.db.set(this.metaPath(dstPath), updated);
     this.db.delete(this.containerPath(srcPath));
     return updated;
   }
 }
 
+async function computeHash(data: Uint8Array): Promise<string> {
+  const digest = await crypto.subtle.digest('SHA-256', data);
+  return Array.from(new Uint8Array(digest)).map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
 function guessMime(name: string): string {
   const ext = name.split('.').pop()?.toLowerCase();
   const mimes: Record<string, string> = {

package/src/shared/protocol.ts

@@ -36,6 +36,7 @@ export type ClientMessage =
   | { id: string; op: 'vfs-download-init'; path: string }
   | { id: string; op: 'vfs-stat'; path: string }
   | { id: string; op: 'vfs-list'; path: string }
+  | { id: string; op: 'vfs-tree'; path: string; hiddenPaths?: string[]; hideDotfiles?: boolean }
   | { id: string; op: 'vfs-delete'; path: string }
   | { id: string; op: 'vfs-mkdir'; path: string }
   | { id: string; op: 'vfs-move'; path: string; dst: string }
@@ -100,6 +101,7 @@ export interface FileStat {
   mtime: number;
   fileId?: string;
   isDir: boolean;
+  hash?: string;
 }
 
 export type BatchOp =

package/tests/repl-stream-bloat.test.ts

@@ -0,0 +1,270 @@
+import { describe, it, expect, afterEach } from 'bun:test';
+import { BodDB } from '../src/server/BodDB.ts';
+import { BodClient } from '../src/client/BodClient.ts';
+
+const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
+let nextPort = 26400 + Math.floor(Math.random() * 1000);
+
+/**
+ * Stress tests for _repl stream bloat.
+ *
+ * The real-world issue: _repl grows unbounded → streamMaterialize produces
+ * a massive WS response → client requestTimeout (30s) fires or WS chokes.
+ *
+ * Locally we can't easily reproduce network latency, but we CAN:
+ * 1. Push the entry count + payload size to stress serialization/parsing
+ * 2. Use short requestTimeout on the replica client to simulate the real failure
+ * 3. Measure materialization time scaling (O(n) proof)
+ * 4. Verify compaction actually fixes it
+ */
+describe('_repl stream bloat', () => {
+  const instances: BodDB[] = [];
+  const clients: BodClient[] = [];
+
+  afterEach(() => {
+    for (const c of clients) c.disconnect();
+    clients.length = 0;
+    for (const db of [...instances].reverse()) db.close();
+    instances.length = 0;
+  });
+
+  function getPort() { return nextPort++; }
+
+  /** Primary with auto-compact DISABLED so _repl grows unbounded */
+  function createPrimary(opts?: { maxMessageSize?: number }) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', compact: {} },
+    });
+    db.replication!.start();
+    db.serve({ port, maxMessageSize: opts?.maxMessageSize });
+    instances.push(db);
+    return { db, port };
+  }
+
+  function createReplica(primaryPort: number) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: {
+        role: 'replica',
+        primaryUrl: `ws://localhost:${primaryPort}`,
+        replicaId: `bloat-replica-${port}`,
+      },
+    });
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  // Helper: fill _repl with N entries, each ~payloadBytes
+  function fillRepl(db: BodDB, count: number, payloadBytes: number) {
+    const padding = 'x'.repeat(Math.max(0, payloadBytes - 100));
+    for (let i = 0; i < count; i++) {
+      db.set(`vfs/files/project/src/deep/nested/path/module${i}/component.tsx`, {
+        size: 1024 + i,
+        mime: 'application/typescript',
+        mtime: Date.now(),
+        fileId: `fid_${i}_${padding}`,
+        hash: `sha256_${i.toString(16).padStart(64, '0')}`,
+        isDir: false,
+        metadata: { author: 'test', tags: ['a', 'b', 'c'], version: i },
+      });
+    }
+  }
+
+  // --- Accumulation ---
+
+  it('_repl grows unbounded without compaction', () => {
+    const { db } = createPrimary();
+    fillRepl(db, 5000, 300);
+    const repl = db.get('_repl') as Record<string, any>;
+    expect(Object.keys(repl).length).toBe(5000);
+  });
+
+  // --- Scaling: measure materialize time at increasing sizes ---
+
+  it('materialize time scales linearly (O(n) proof)', async () => {
+    const sizes = [500, 2000, 5000];
+    const timings: { count: number; ms: number; bytes: number }[] = [];
+
+    for (const count of sizes) {
+      const { db, port } = createPrimary();
+      fillRepl(db, count, 500);
+
+      const client = new BodClient({ url: `ws://localhost:${port}` });
+      clients.push(client);
+      await client.connect();
+
+      const start = Date.now();
+      const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+      const elapsed = Date.now() - start;
+
+      const json = JSON.stringify(result);
+      timings.push({ count, ms: elapsed, bytes: json.length });
+    }
+
+    for (const t of timings) {
+      console.log(`  ${t.count} entries → ${t.ms}ms, ${(t.bytes / 1024 / 1024).toFixed(2)}MB response`);
+    }
+
+    // Response size should scale roughly linearly
+    const ratio = timings[2].bytes / timings[0].bytes;
+    console.log(`  Size ratio (${sizes[2]}/${sizes[0]}): ${ratio.toFixed(1)}x (expect ~${(sizes[2] / sizes[0]).toFixed(1)}x)`);
+    expect(ratio).toBeGreaterThan(5); // 5000/500 = 10x, allow some dedup
+  });
+
+  // --- Timeout reproduction: short requestTimeout simulates real-world failure ---
+
+  it('short requestTimeout causes streamMaterialize to fail on bloated _repl', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 10000, 1000); // 10k entries × 1KB = ~10MB response
+
+    const replCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
+    expect(replCount).toBe(10000);
+
+    // Direct client with 1ms timeout — guaranteed to fail, proving the
+    // timeout path exists and that materialize has no retry/fallback
+    const client = new BodClient({
+      url: `ws://localhost:${primaryPort}`,
+      requestTimeout: 1,
+    });
+    clients.push(client);
+    await client.connect();
+
+    let error: Error | null = null;
+    try {
+      await client.streamMaterialize('_repl', { keepKey: 'path' });
+    } catch (e) {
+      error = e as Error;
+    }
+
+    expect(error).not.toBeNull();
+    console.log(`  REPRODUCED: streamMaterialize failed: ${error!.message}`);
+    expect(error!.message).toContain('timeout');
+  });
+
+  it('streamMaterialize response exceeds typical WS comfort zone', async () => {
+    const { db, port } = createPrimary();
+    fillRepl(db, 10000, 1000);
+
+    const client = new BodClient({ url: `ws://localhost:${port}` });
+    clients.push(client);
+    await client.connect();
+
+    const start = Date.now();
+    const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+    const elapsed = Date.now() - start;
+    const responseSize = JSON.stringify(result).length;
+
+    console.log(`  10k entries × 1KB: ${(responseSize / 1024 / 1024).toFixed(1)}MB response in ${elapsed}ms`);
+    // On real networks with latency, this 10MB+ response would easily exceed 30s timeout
+    expect(responseSize).toBeGreaterThan(5 * 1024 * 1024); // >5MB
+  });
+
+  // --- Payload size bomb: fewer entries but huge payloads ---
+
+  it('large payloads per entry amplify the problem', async () => {
+    const { db, port } = createPrimary();
+
+    // 1000 entries but 2KB each → ~2MB+ materialize response
+    const bigPadding = 'y'.repeat(2000);
+    for (let i = 0; i < 1000; i++) {
+      db.set(`data/big${i}`, {
+        content: bigPadding,
+        nested: { deep: { value: bigPadding.slice(0, 500) } },
+        index: i,
+      });
+    }
+
+    const client = new BodClient({ url: `ws://localhost:${port}` });
+    clients.push(client);
+    await client.connect();
+
+    const start = Date.now();
+    const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+    const elapsed = Date.now() - start;
+    const responseSize = JSON.stringify(result).length;
+
+    console.log(`  1000 entries × 2KB = ${(responseSize / 1024 / 1024).toFixed(2)}MB, ${elapsed}ms`);
+    expect(responseSize).toBeGreaterThan(2 * 1024 * 1024); // >2MB
+  });
+
+  // --- Compaction fixes it ---
+
+  it('compaction reduces _repl and speeds up bootstrap', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 5000, 500);
+
+    const beforeCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
+    expect(beforeCount).toBe(5000);
+
+    // Compact down to 500
+    primary.stream.compact('_repl', { maxCount: 500, keepKey: 'path' });
+    const afterRepl = primary.get('_repl') as Record<string, any>;
+    const afterCount = afterRepl ? Object.keys(afterRepl).length : 0;
+    console.log(`  Compacted: ${beforeCount} → ${afterCount} entries`);
+    expect(afterCount).toBeLessThanOrEqual(500);
+
+    // Bootstrap should now work fast with compacted stream
+    const { db: replica } = createReplica(primaryPort);
+    const start = Date.now();
+    await replica.replication!.start();
+    const elapsed = Date.now() - start;
+
+    console.log(`  Compacted bootstrap: ${elapsed}ms`);
+    expect(elapsed).toBeLessThan(3000);
+
+    await wait(300);
+    // Verify latest writes are present (compaction keeps newest by keepKey)
+    const val = replica.get('vfs/files/project/src/deep/nested/path/module4999/component.tsx') as any;
+    expect(val?.size).toBe(1024 + 4999);
+  });
+
+  // --- Repeated writes to same paths: worst case for non-compacted stream ---
+
+  it('repeated writes to same paths bloat _repl with duplicates', () => {
+    const { db } = createPrimary();
+
+    // 100 paths × 50 writes each = 5000 _repl entries, but only 100 unique paths
+    for (let round = 0; round < 50; round++) {
+      for (let i = 0; i < 100; i++) {
+        db.set(`config/setting${i}`, { value: round, updated: Date.now() });
+      }
+    }
+
+    const repl = db.get('_repl') as Record<string, any>;
+    const totalEntries = Object.keys(repl).length;
+    console.log(`  100 paths × 50 writes = ${totalEntries} _repl entries`);
+    expect(totalEntries).toBe(5000);
+
+    // Compact with keepKey deduplicates to 100
+    db.stream.compact('_repl', { keepKey: 'path' });
+    const after = db.get('_repl') as Record<string, any>;
+    const afterCount = after ? Object.keys(after).length : 0;
+    // snapshot (1) + remaining entries
+    console.log(`  After compact: ${afterCount} entries (expect ~100 unique paths)`);
+    expect(afterCount).toBeLessThanOrEqual(150);
+  });
+
+  // --- No bootstrap protection: replica.start() has no timeout ---
+
+  it('replica.start() has no built-in timeout (current gap)', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 3000, 300);
+
+    const { db: replica } = createReplica(primaryPort);
+
+    // Measure: start() blocks until materialize completes — no internal timeout
+    const start = Date.now();
+    await replica.replication!.start();
+    const elapsed = Date.now() - start;
+
+    console.log(`  replica.start() blocked for ${elapsed}ms (no internal timeout)`);
+    // This documents the gap: there's no way to bail out of a slow bootstrap
+    // P0 fix should add a configurable timeout here
+  });
+});