@bod.ee/db 0.7.0 → 0.8.0

package/tests/replication.test.ts

@@ -0,0 +1,404 @@
+import { describe, it, expect, afterEach } from 'bun:test';
+import { BodDB } from '../src/server/BodDB.ts';
+import { BodClient } from '../src/client/BodClient.ts';
+
+const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
+let nextPort = 24400 + Math.floor(Math.random() * 1000);
+
+describe('ReplicationEngine', () => {
+  const instances: BodDB[] = [];
+  const clients: BodClient[] = [];
+
+  afterEach(() => {
+    for (const c of clients) c.disconnect();
+    clients.length = 0;
+    for (const db of [...instances].reverse()) db.close();
+    instances.length = 0;
+  });
+
+  function getPort() { return nextPort++; }
+
+  function createPrimary() {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    db.replication!.start();
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  function createReplica(primaryPort: number) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: {
+        role: 'replica',
+        primaryUrl: `ws://localhost:${primaryPort}`,
+        replicaId: `test-replica-${port}`,
+      },
+    });
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  // 1. Primary emits events to _repl on set/delete/update/push
+  it('primary emits replication events on set/delete/update/push', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.set('users/u1', { name: 'Eli' });
+    db.update({ 'users/u2/age': 30 });
+    db.delete('users/u1');
+    db.push('logs', { msg: 'hello' });
+
+    const replData = db.get('_repl');
+    expect(replData).toBeTruthy();
+    const entries = Object.values(replData as Record<string, any>);
+    // update emits per-path set events: set + set(from update) + delete + push = 4
+    expect(entries.length).toBe(4);
+    expect(entries[0].op).toBe('set');
+    expect(entries[0].path).toBe('users/u1');
+    expect(entries[1].op).toBe('set');
+    expect(entries[1].path).toBe('users/u2/age');
+    expect(entries[2].op).toBe('delete');
+    expect(entries[3].op).toBe('push');
+  });
+
+  // 2. Loop prevention — _replaying flag prevents re-emission
+  it('does not emit events when replaying', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.setReplaying(true);
+    db.set('users/u1', { name: 'Test' });
+    db.setReplaying(false);
+
+    const replData = db.get('_repl');
+    expect(replData).toBeNull();
+  });
+
+  // 3. Excluded prefixes are not replicated
+  it('skips excluded prefixes', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', excludePrefixes: ['_repl', '_streams', '_mq', 'internal'] },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.set('internal/config', { foo: 1 });
+    db.set('users/u1', { name: 'Eli' });
+
+    const replData = db.get('_repl') as Record<string, any>;
+    const entries = Object.values(replData);
+    expect(entries.length).toBe(1);
+    expect(entries[0].op).toBe('set');
+    expect(entries[0].path).toBe('users/u1');
+  });
+
+  // 4. Integration: primary + replica bootstrap
+  it('replica receives data from primary via replication stream', async () => {
+    const { db: primary, port: pPort } = createPrimary();
+
+    primary.set('users/u1', { name: 'Eli' });
+    primary.set('users/u2', { name: 'Dan' });
+
+    const { db: replica } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(500);
+
+    expect(replica.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(replica.get('users/u2')).toEqual({ name: 'Dan' });
+  });
+
+  // 5. Proxy test: write to replica goes to primary
+  it('write to replica is proxied to primary', async () => {
+    const { db: primary, port: pPort } = createPrimary();
+    const { db: replica, port: rPort } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(200);
+
+    const client = new BodClient({ url: `ws://localhost:${rPort}` });
+    clients.push(client);
+    await client.connect();
+    await client.set('items/i1', { price: 42 });
+    await wait(200);
+
+    expect(primary.get('items/i1')).toEqual({ price: 42 });
+
+    await wait(500);
+    expect(replica.get('items/i1')).toEqual({ price: 42 });
+  });
+
+  // 6. Push replication preserves keys
+  it('push events replicate with exact pushKey', async () => {
+    const { db: primary, port: pPort } = createPrimary();
+    const key = primary.push('events', { type: 'click' });
+
+    const { db: replica } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(500);
+
+    expect(replica.get(`events/${key}`)).toEqual({ type: 'click' });
+  });
+
+  // 7. Transaction events are replicated
+  it('transaction write events are replicated', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.transaction(tx => {
+      tx.set('a/1', 'one');
+      tx.set('a/2', 'two');
+    });
+
+    const replData = db.get('_repl') as Record<string, any>;
+    const entries = Object.values(replData);
+    expect(entries.length).toBe(2);
+  });
+
+  // 8. Delete replication
+  it('delete events are replicated to replica', async () => {
+    const { db: primary, port: pPort } = createPrimary();
+    primary.set('tmp/x', 'hello');
+
+    const { db: replica } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(500);
+    expect(replica.get('tmp/x')).toBe('hello');
+
+    // Delete on primary
+    primary.delete('tmp/x');
+    await wait(500);
+    expect(replica.get('tmp/x')).toBeNull();
+  });
+
+  // 9. Update replication (flattened to per-path sets)
+  it('update events replicate as per-path sets', async () => {
+    const { db: primary, port: pPort } = createPrimary();
+
+    const { db: replica } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(200);
+
+    primary.update({ 'conf/a': 1, 'conf/b': 2 });
+    await wait(500);
+
+    expect(replica.get('conf/a')).toBe(1);
+    expect(replica.get('conf/b')).toBe(2);
+  });
+
+  // 10. Multi-path update exclusion check (all paths checked, not just first)
+  it('update with mixed excluded/non-excluded paths emits only non-excluded', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', excludePrefixes: ['_repl', '_streams', '_mq', '_internal'] },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.update({ '_internal/x': 1, 'data/y': 2 });
+
+    const replData = db.get('_repl') as Record<string, any>;
+    const entries = Object.values(replData);
+    expect(entries.length).toBe(1);
+    expect(entries[0].path).toBe('data/y');
+  });
+
+  // 11. Sweep fires delete events for expired paths
+  it('sweep expired paths are replicated as deletes', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.set('sessions/s1', { token: 'abc' }, { ttl: 1 });
+    // Force expiry by manipulating the DB directly
+    db.storage.db.prepare('UPDATE nodes SET expires_at = 1 WHERE path LIKE ?').run('sessions/s1%');
+    db.sweep();
+
+    const replData = db.get('_repl') as Record<string, any>;
+    const entries = Object.values(replData);
+    // set event + delete event(s) from sweep
+    const deleteEvents = entries.filter((e: any) => e.op === 'delete');
+    expect(deleteEvents.length).toBeGreaterThan(0);
+  });
+
+  // 12. Recursion guard — emit doesn't infinite loop
+  it('emit does not cause infinite recursion', () => {
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary' },
+    });
+    instances.push(db);
+    db.replication!.start();
+
+    db.set('users/u1', 'test');
+    expect(db.get('users/u1')).toBe('test');
+    const replData = db.get('_repl') as Record<string, any>;
+    expect(Object.values(replData).length).toBe(1);
+  });
+
+  // --- Source feed subscription tests ---
+
+  function createSourceLocal(sources: Array<{ url: string; paths: string[]; localPrefix?: string; excludePrefixes?: string[] }>) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', sources },
+    });
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  it('source only syncs matching paths', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+    remote.set('orders/o1', { total: 100 });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'] }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('orders/o1')).toBeNull();
+  });
+
+  it('source remaps paths with localPrefix', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'], localPrefix: 'db-a' }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('db-a/users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('users/u1')).toBeNull();
+  });
+
+  it('multi-source syncs from multiple remotes', async () => {
+    const { db: remoteA, port: portA } = createPrimary();
+    const { db: remoteB, port: portB } = createPrimary();
+    remoteA.set('users/u1', { name: 'Eli' });
+    remoteB.set('products/p1', { price: 42 });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${portA}`, paths: ['users'], localPrefix: 'a' },
+      { url: `ws://localhost:${portB}`, paths: ['products'], localPrefix: 'b' },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('a/users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('b/products/p1')).toEqual({ price: 42 });
+  });
+
+  it('primary with sources: own writes emit, source data does not', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'], localPrefix: 'ext' }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('ext/users/u1')).toEqual({ name: 'Eli' });
+
+    local.set('mydata/x', 'hello');
+    const replData = local.get('_repl') as Record<string, any>;
+    const entries = Object.values(replData);
+    expect(entries.some((e: any) => e.path === 'mydata/x')).toBe(true);
+    expect(entries.some((e: any) => e.path === 'ext/users/u1')).toBe(false);
+  });
+
+  it('source receives ongoing events after bootstrap', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'] }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+
+    remote.set('users/u2', { name: 'Dan' });
+    await wait(500);
+    expect(local.get('users/u2')).toEqual({ name: 'Dan' });
+  });
+
+  it('source excludePrefixes filters out matching paths', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+    remote.set('users/internal/secret', { key: '123' });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${rPort}`, paths: ['users'], excludePrefixes: ['users/internal'] },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('users/internal/secret')).toBeNull();
+  });
+
+  it('source replicates delete events with remap', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'], localPrefix: 'src' }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('src/users/u1')).toEqual({ name: 'Eli' });
+
+    remote.delete('users/u1');
+    await wait(500);
+    expect(local.get('src/users/u1')).toBeNull();
+  });
+
+  it('stop() disconnects source clients', async () => {
+    const { db: remote, port: rPort } = createPrimary();
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([{ url: `ws://localhost:${rPort}`, paths: ['users'] }]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+
+    local.replication!.stop();
+
+    // After stop, ongoing writes should NOT sync
+    remote.set('users/u2', { name: 'Dan' });
+    await wait(500);
+    expect(local.get('users/u2')).toBeNull();
+  });
+});

package/tests/vfs.test.ts

@@ -0,0 +1,166 @@
+import { describe, it, expect, beforeEach, afterEach, afterAll } from 'bun:test';
+import { BodDB } from '../src/server/BodDB.ts';
+import { BodClient } from '../src/client/BodClient.ts';
+import { rmSync } from 'fs';
+
+const VFS_ROOT = '.tmp/test-vfs-' + Date.now();
+let db: BodDB;
+let client: BodClient;
+let port: number;
+
+beforeEach(async () => {
+  db = new BodDB({ path: ':memory:', vfs: { storageRoot: VFS_ROOT } });
+  const server = db.serve({ port: 0 });
+  port = server.port;
+  client = new BodClient({ url: `ws://localhost:${port}` });
+  await client.connect();
+});
+
+afterEach(() => {
+  client.disconnect();
+  db.close();
+});
+
+afterAll(() => {
+  try { rmSync(VFS_ROOT, { recursive: true }); } catch {}
+});
+
+describe('VFSEngine direct', () => {
+  it('write + read + stat round-trip', async () => {
+    const data = new TextEncoder().encode('hello vfs');
+    const stat = await db.vfs!.write('docs/readme.txt', data);
+    expect(stat.name).toBe('readme.txt');
+    expect(stat.path).toBe('docs/readme.txt');
+    expect(stat.size).toBe(9);
+    expect(stat.mime).toBe('text/plain');
+    expect(stat.isDir).toBe(false);
+
+    const read = await db.vfs!.read('docs/readme.txt');
+    expect(new TextDecoder().decode(read)).toBe('hello vfs');
+
+    const s = db.vfs!.stat('docs/readme.txt');
+    expect(s?.fileId).toBeDefined();
+  });
+
+  it('mkdir + list', () => {
+    db.vfs!.mkdir('projects');
+    db.vfs!.mkdir('projects/alpha');
+    const items = db.vfs!.list('projects');
+    expect(items.length).toBe(1);
+    expect(items[0].name).toBe('alpha');
+  });
+
+  it('move renames without file I/O', async () => {
+    const data = new TextEncoder().encode('moveme');
+    await db.vfs!.write('a.txt', data);
+    const oldStat = db.vfs!.stat('a.txt')!;
+    const newStat = await db.vfs!.move('a.txt', 'b.txt');
+    expect(newStat.fileId).toBe(oldStat.fileId); // same file on disk
+    expect(newStat.path).toBe('b.txt');
+    expect(db.vfs!.stat('a.txt')).toBeNull();
+  });
+
+  it('delete removes file + metadata', async () => {
+    const data = new TextEncoder().encode('del');
+    await db.vfs!.write('tmp.txt', data);
+    await db.vfs!.remove('tmp.txt');
+    expect(db.vfs!.stat('tmp.txt')).toBeNull();
+  });
+});
+
+describe('VFS REST transport', () => {
+  it('upload + download via REST', async () => {
+    const data = new TextEncoder().encode('rest-upload');
+    const uploadRes = await fetch(`http://localhost:${port}/files/docs/test.txt`, {
+      method: 'POST',
+      body: data,
+      headers: { 'Content-Type': 'text/plain' },
+    });
+    const uploadJson = await uploadRes.json() as any;
+    expect(uploadJson.ok).toBe(true);
+    expect(uploadJson.data.size).toBe(11);
+
+    const dlRes = await fetch(`http://localhost:${port}/files/docs/test.txt`);
+    expect(dlRes.ok).toBe(true);
+    const dlData = new Uint8Array(await dlRes.arrayBuffer());
+    expect(new TextDecoder().decode(dlData)).toBe('rest-upload');
+  });
+
+  it('stat + list + delete via REST', async () => {
+    await fetch(`http://localhost:${port}/files/a/b.txt`, { method: 'POST', body: new Uint8Array([1, 2, 3]) });
+
+    const statRes = await fetch(`http://localhost:${port}/files/a/b.txt?stat=1`);
+    const statJson = await statRes.json() as any;
+    expect(statJson.ok).toBe(true);
+    expect(statJson.data.name).toBe('b.txt');
+
+    const listRes = await fetch(`http://localhost:${port}/files/a?list=1`);
+    const listJson = await listRes.json() as any;
+    expect(listJson.data.length).toBe(1);
+
+    const delRes = await fetch(`http://localhost:${port}/files/a/b.txt`, { method: 'DELETE' });
+    expect((await delRes.json() as any).ok).toBe(true);
+  });
+
+  it('mkdir + move via REST', async () => {
+    const mkdirRes = await fetch(`http://localhost:${port}/files/newdir?mkdir=1`, { method: 'POST' });
+    expect((await mkdirRes.json() as any).ok).toBe(true);
+
+    await fetch(`http://localhost:${port}/files/x.txt`, { method: 'POST', body: new Uint8Array([9]) });
+    const moveRes = await fetch(`http://localhost:${port}/files/x.txt?move=y.txt`, { method: 'PUT' });
+    const moveJson = await moveRes.json() as any;
+    expect(moveJson.ok).toBe(true);
+    expect(moveJson.data.path).toBe('y.txt');
+  });
+});
+
+describe('VFS Client SDK', () => {
+  it('vfs().upload + download via REST', async () => {
+    const vfs = client.vfs();
+    const data = new TextEncoder().encode('client-upload');
+    const stat = await vfs.upload('sdk/file.txt', data, 'text/plain');
+    expect(stat.size).toBe(13);
+
+    const dl = await vfs.download('sdk/file.txt');
+    expect(new TextDecoder().decode(dl)).toBe('client-upload');
+  });
+
+  it('vfs().uploadWS + downloadWS via WS chunks', async () => {
+    const vfs = client.vfs();
+    const data = new TextEncoder().encode('ws-chunked-data');
+    const stat = await vfs.uploadWS('ws/file.bin', data);
+    expect(stat.size).toBe(data.byteLength);
+
+    const dl = await vfs.downloadWS('ws/file.bin');
+    expect(new TextDecoder().decode(dl)).toBe('ws-chunked-data');
+  });
+
+  it('vfs().stat + list + mkdir + delete + move', async () => {
+    const vfs = client.vfs();
+    await vfs.mkdir('testdir');
+    await vfs.upload('testdir/a.txt', new TextEncoder().encode('a'));
+
+    const stat = await vfs.stat('testdir/a.txt');
+    expect(stat?.name).toBe('a.txt');
+
+    const list = await vfs.list('testdir');
+    expect(list.length).toBe(1);
+
+    const moved = await vfs.move('testdir/a.txt', 'testdir/b.txt');
+    expect(moved.path).toBe('testdir/b.txt');
+
+    await vfs.delete('testdir/b.txt');
+    const stat2 = await vfs.stat('testdir/b.txt');
+    expect(stat2).toBeNull();
+  });
+});
+
+describe('VFS metadata gets subscriptions for free', () => {
+  it('subscribing to _vfs path fires on file write', async () => {
+    const events: unknown[] = [];
+    db.on('_vfs/watched', (snap) => events.push(snap.val()));
+    await db.vfs!.write('watched/hello.txt', new TextEncoder().encode('hi'));
+    // Subscription fires synchronously for local ops
+    expect(events.length).toBeGreaterThan(0);
+  });
+});