@bod.ee/db 0.7.0 → 0.8.0

package/cli.ts

@@ -59,6 +59,15 @@ const db = await BodDB.create(options as any);
 const server = db.serve();
 const port = server?.port ?? options.port ?? 4400;
 
+// Start replication if configured
+if (db.replication) {
+  db.replication.start().then(() => {
+    console.log(`  Replication: ${db.replication!.isPrimary ? 'primary' : `replica → ${(options as any).replication?.primaryUrl}`}`);
+  }).catch((e: Error) => {
+    console.error(`  Replication failed: ${e.message}`);
+  });
+}
+
 const line = (s: string) => `  │ ${s.padEnd(36)}│`;
 console.log([
   '',
@@ -110,6 +119,7 @@ if (options.transport?.staticRoutes) {
       system: { cpuCores: cpus().length, totalMemMb: +(totalmem() / 1024 / 1024).toFixed(0), cpuPercent: systemCpuPercent() },
       subs: db.subs.subscriberCount(),
       clients: db.transport?.clientCount ?? 0,
+      repl: db.replication?.stats() ?? null,
       ts: Date.now(),
     });
   }, 1000);

package/client.ts

@@ -1,2 +1,3 @@
-export { BodClient, BodClientOptions, ValueSnapshot, ClientQueryBuilder, StreamReader, MQReader, MQMessageSnapshot } from './src/client/BodClient.ts';
-export type { ChildEvent, StreamEventSnapshot } from './src/client/BodClient.ts';
+export { BodClient, BodClientOptions, ValueSnapshot, ClientQueryBuilder, StreamReader, MQReader, MQMessageSnapshot, VFSClient } from './src/client/BodClient.ts';
+export type { ChildEvent, StreamEventSnapshot, FileStat } from './src/client/BodClient.ts';
+export { CachedClient, CachedClientOptions } from './src/client/CachedClient.ts';

package/config.ts

@@ -14,6 +14,7 @@ export default {
   fts: {},
   vectors: { dimensions: 384 },
   mq: { visibilityTimeout: 30, maxDeliveries: 5 },
+  vfs: { storageRoot: './.tmp/vfs' },
   compact: {
     'events/logs': { maxAge: 86400 },
   },

package/deploy/boddb-il.yaml

@@ -0,0 +1,14 @@
+droplet:
+  host: 151.145.81.254
+ssh:
+  privateKey: ~/.ssh/oracle_cloud
+app:
+  name: boddb-il
+  dir: /opt/boddb-il
+runtime:
+  port: 4400
+service:
+  name: boddb-il
+  execStart: /home/ubuntu/.bun/bin/bun run cli.ts deploy/prod-il.config.ts
+https:
+  domain: db-il.bod.ee

package/deploy/prod-il.config.ts

@@ -0,0 +1,19 @@
+export default {
+  path: './data/bod.db',
+  port: 4400,
+  sweepInterval: 60000,
+  rules: { '': { read: true, write: true } },
+  indexes: {},
+  fts: {},
+  replication: {
+    role: 'replica' as const,
+    primaryUrl: 'wss://db-main.bod.ee',
+    replicaId: 'il-replica-1',
+  },
+  transport: {
+    staticRoutes: {
+      '/admin': './admin/ui.html',
+      '/': './admin/ui.html',
+    },
+  },
+}

package/deploy/prod.config.ts

@@ -6,6 +6,7 @@ export default {
   indexes: {},
   fts: {},
   mq: { visibilityTimeout: 30, maxDeliveries: 5 },
+  replication: { role: 'primary' as const },
   transport: {
     staticRoutes: {
       '/admin': './admin/ui.html',

package/index.ts

@@ -6,6 +6,7 @@ export { QueryEngine } from './src/server/QueryEngine.ts';
 export { RulesEngine, RulesEngineOptions } from './src/server/RulesEngine.ts';
 export type { RuleContext, PathRule } from './src/server/RulesEngine.ts';
 export { Transport, TransportOptions } from './src/server/Transport.ts';
+export type { WsData } from './src/server/Transport.ts';
 export { FTSEngine, FTSEngineOptions } from './src/server/FTSEngine.ts';
 export { VectorEngine, VectorEngineOptions } from './src/server/VectorEngine.ts';
 export { StreamEngine, StreamEngineOptions } from './src/server/StreamEngine.ts';
@@ -13,6 +14,8 @@ export { MQEngine, MQEngineOptions } from './src/server/MQEngine.ts';
 export type { MQMessage } from './src/server/MQEngine.ts';
 export type { StreamEvent, CompactOptions, CompactResult, StreamSnapshot } from './src/server/StreamEngine.ts';
 export { FileAdapter, FileAdapterOptions } from './src/server/FileAdapter.ts';
+export { ReplicationEngine, ReplicationOptions, ReplicationSource } from './src/server/ReplicationEngine.ts';
+export type { ReplEvent, WriteEvent } from './src/server/ReplicationEngine.ts';
 export { compileRule } from './src/server/ExpressionRules.ts';
 export { increment, serverTimestamp, arrayUnion, arrayRemove, ref } from './src/shared/transforms.ts';
 export * from './src/shared/protocol.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bod.ee/db",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "module": "index.ts",
   "type": "module",
   "bin": {
@@ -13,6 +13,7 @@
   },
   "scripts": {
     "admin": "bun --watch run admin/server.ts",
+    "admin:remote": "bun run admin/proxy.ts",
     "serve": "bun run cli.ts",
     "start": "bun run cli.ts config.ts",
     "publish-lib": "bun publish --access public",
@@ -21,7 +22,11 @@
     "deploy-logs-db": "bun run deploy/deploy.ts boddb-logs ",
     "deploy:bootstrap": "bun run deploy/deploy.ts boddb bootstrap",
     "deploy:logs": "bun run deploy/deploy.ts boddb logs",
-    "deploy:ssh": "bun run deploy/deploy.ts boddb ssh"
+    "deploy:ssh": "bun run deploy/deploy.ts boddb ssh",
+    "deploy-il": "bun run deploy/deploy.ts boddb-il deploy",
+    "deploy-il:bootstrap": "bun run deploy/deploy.ts boddb-il bootstrap",
+    "deploy-il:logs": "bun run deploy/deploy.ts boddb-il logs",
+    "deploy-il:ssh": "bun run deploy/deploy.ts boddb-il ssh"
   },
   "peerDependencies": {
     "typescript": "^5"

package/src/client/BodClient.ts

@@ -1,4 +1,4 @@
-import type { QueryFilter, OrderClause, BatchOp } from '../shared/protocol.ts';
+import type { QueryFilter, OrderClause, BatchOp, FileStat } from '../shared/protocol.ts';
 import { pathKey } from '../shared/pathUtils.ts';
 
 export class BodClientOptions {
@@ -24,6 +24,7 @@ export class BodClient {
   private activeSubs = new Set<string>(); // tracks 'value:path' and 'child:path' keys
   private streamCbs = new Map<string, Set<(events: Array<{ key: string; data: unknown }>) => void>>();
   private activeStreamSubs = new Set<string>(); // tracks 'path:groupId' keys
+  private vfsDownloadCbs = new Map<string, Set<(chunk: { data: string; seq: number; done: boolean }) => void>>();
   private closed = false;
   private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
   private reconnectDelay: number;
@@ -78,7 +79,7 @@ export class BodClient {
         if (msg.type === 'value') {
           const cbs = this.valueCbs.get(msg.path);
           if (cbs) {
-            const snap = new ValueSnapshot(msg.path, msg.data);
+            const snap = new ValueSnapshot(msg.path, msg.data, msg.updatedAt);
             for (const cb of cbs) cb(snap);
           }
           return;
@@ -104,12 +105,17 @@ export class BodClient {
           }
           return;
         }
+        if (msg.type === 'vfs-download-chunk') {
+          const cbs = this.vfsDownloadCbs.get(msg.transferId);
+          if (cbs) for (const cb of cbs) cb(msg);
+          return;
+        }
 
         // Response message
         const p = this.pending.get(msg.id);
         if (p) {
           this.pending.delete(msg.id);
-          if (msg.ok) p.resolve(msg.data ?? null);
+          if (msg.ok) p.resolve(msg);
           else p.reject(new Error(msg.error || 'Unknown error'));
         }
       };
@@ -165,23 +171,33 @@ export class BodClient {
     return String(++this.msgId);
   }
 
-  private send(op: string, params: Record<string, unknown> = {}): Promise<unknown> {
+  private sendRaw(op: string, params: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
     return new Promise((resolve, reject) => {
       if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
         return reject(new Error('Not connected'));
       }
       const id = this.nextId();
-      this.pending.set(id, { resolve, reject });
+      this.pending.set(id, { resolve: resolve as (v: unknown) => void, reject });
       this.ws.send(JSON.stringify({ id, op, ...params }));
     });
   }
 
+  private async send(op: string, params: Record<string, unknown> = {}): Promise<unknown> {
+    const msg = await this.sendRaw(op, params);
+    return msg.data ?? null;
+  }
+
   // CRUD
 
   async get(path: string): Promise<unknown> {
     return this.send('get', { path });
   }
 
+  async getSnapshot(path: string): Promise<ValueSnapshot> {
+    const msg = await this.sendRaw('get', { path });
+    return new ValueSnapshot(path, msg.data ?? null, msg.updatedAt as number | undefined);
+  }
+
   async getShallow(path?: string): Promise<Array<{ key: string; isLeaf: boolean; value?: unknown }>> {
     return this.send('get', { path: path ?? '', shallow: true }) as Promise<Array<{ key: string; isLeaf: boolean; value?: unknown }>>;
   }
@@ -373,13 +389,43 @@ export class BodClient {
   get connected(): boolean {
     return this.ws?.readyState === WebSocket.OPEN;
   }
+
+  /** Get the HTTP base URL derived from the WS URL */
+  get httpUrl(): string {
+    return this.options.url.replace(/^ws(s?):\/\//, 'http$1://');
+  }
+
+  /** VFS client facade */
+  vfs(): VFSClient {
+    return new VFSClient(this);
+  }
+
+  /** @internal */
+  _send(op: string, params: Record<string, unknown> = {}): Promise<unknown> {
+    return this.send(op, params);
+  }
+
+  /** @internal — listen for download chunks */
+  _onDownloadChunk(transferId: string, cb: (chunk: { data: string; seq: number; done: boolean }) => void): () => void {
+    if (!this.vfsDownloadCbs.has(transferId)) this.vfsDownloadCbs.set(transferId, new Set());
+    this.vfsDownloadCbs.get(transferId)!.add(cb);
+    return () => {
+      this.vfsDownloadCbs.get(transferId)?.delete(cb);
+      if (this.vfsDownloadCbs.get(transferId)?.size === 0) this.vfsDownloadCbs.delete(transferId);
+    };
+  }
 }
 
 export class ValueSnapshot {
+  readonly updatedAt?: number;
+
   constructor(
     readonly path: string,
     private data: unknown,
-  ) {}
+    updatedAt?: number,
+  ) {
+    this.updatedAt = updatedAt;
+  }
 
   val(): unknown { return this.data; }
   get key(): string { return pathKey(this.path); }
@@ -476,6 +522,83 @@ export class MQMessageSnapshot {
   exists(): boolean { return this.data !== null && this.data !== undefined; }
 }
 
+export type { FileStat };
+
+export class VFSClient {
+  constructor(private client: BodClient) {}
+
+  /** Upload via REST (preferred — binary streaming) */
+  async upload(path: string, data: Uint8Array, mime?: string): Promise<FileStat> {
+    const url = `${this.client.httpUrl}/files/${path}`;
+    const res = await fetch(url, {
+      method: 'POST',
+      body: data,
+      headers: mime ? { 'Content-Type': mime } : {},
+    });
+    const json = await res.json() as { ok: boolean; data: FileStat; error?: string };
+    if (!json.ok) throw new Error(json.error || 'Upload failed');
+    return json.data;
+  }
+
+  /** Download via REST (preferred — binary streaming) */
+  async download(path: string): Promise<Uint8Array> {
+    const url = `${this.client.httpUrl}/files/${path}`;
+    const res = await fetch(url);
+    if (!res.ok) throw new Error('Download failed');
+    return new Uint8Array(await res.arrayBuffer());
+  }
+
+  /** Upload via WS (chunked, base64-encoded fallback) */
+  async uploadWS(path: string, data: Uint8Array, mime?: string): Promise<FileStat> {
+    const transferId = await this.client._send('vfs-upload-init', { path, size: data.byteLength, mime }) as string;
+    const b64 = Buffer.from(data).toString('base64');
+    const CHUNK_SIZE = 48000;
+    const totalChunks = Math.ceil(b64.length / CHUNK_SIZE) || 1;
+    for (let i = 0; i < totalChunks; i++) {
+      const chunk = b64.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+      await this.client._send('vfs-upload-chunk', { transferId, data: chunk, seq: i });
+    }
+    return await this.client._send('vfs-upload-done', { transferId }) as FileStat;
+  }
+
+  /** Download via WS (chunked, base64-encoded fallback) */
+  async downloadWS(path: string): Promise<Uint8Array> {
+    const result = await this.client._send('vfs-download-init', { path }) as { transferId: string; totalChunks: number };
+    const chunks: string[] = [];
+    return new Promise<Uint8Array>((resolve, reject) => {
+      const timeout = setTimeout(() => { off(); reject(new Error('Download timeout')); }, 30000);
+      const off = this.client._onDownloadChunk(result.transferId, (chunk) => {
+        chunks[chunk.seq] = chunk.data;
+        if (chunk.done) {
+          clearTimeout(timeout);
+          off();
+          resolve(new Uint8Array(Buffer.from(chunks.join(''), 'base64')));
+        }
+      });
+    });
+  }
+
+  async stat(path: string): Promise<FileStat | null> {
+    return await this.client._send('vfs-stat', { path }) as FileStat | null;
+  }
+
+  async list(path: string): Promise<FileStat[]> {
+    return await this.client._send('vfs-list', { path }) as FileStat[];
+  }
+
+  async delete(path: string): Promise<void> {
+    await this.client._send('vfs-delete', { path });
+  }
+
+  async mkdir(path: string): Promise<FileStat> {
+    return await this.client._send('vfs-mkdir', { path }) as FileStat;
+  }
+
+  async move(src: string, dst: string): Promise<FileStat> {
+    return await this.client._send('vfs-move', { path: src, dst }) as FileStat;
+  }
+}
+
 export class MQReader {
   constructor(
     private client: BodClient,

package/src/client/CachedClient.ts

@@ -0,0 +1,228 @@
+import { BodClient, ValueSnapshot } from './BodClient.ts';
+import type { ChildEvent } from './BodClient.ts';
+import { ancestors } from '../shared/pathUtils.ts';
+
+export class CachedClientOptions {
+  maxAge: number = 7 * 24 * 60 * 60 * 1000; // 7 days in ms
+  maxMemoryEntries: number = 500;
+  dbName: string = 'boddb-cache';
+  enabled: boolean = true;
+}
+
+interface CacheEntry {
+  path: string;
+  data: unknown;
+  updatedAt: number;
+  cachedAt: number;
+}
+
+export class CachedClient {
+  readonly options: CachedClientOptions;
+  readonly client: BodClient;
+  private memory = new Map<string, CacheEntry>();
+  private subscribedPaths = new Set<string>();
+  private idb: IDBDatabase | null = null;
+
+  constructor(client: BodClient, options?: Partial<CachedClientOptions>) {
+    this.options = { ...new CachedClientOptions(), ...options };
+    this.client = client;
+  }
+
+  async init(): Promise<void> {
+    if (!this.options.enabled) return;
+    this.idb = await this.openIDB();
+    await this.sweepExpired();
+  }
+
+  private openIDB(): Promise<IDBDatabase> {
+    return new Promise((resolve, reject) => {
+      const req = indexedDB.open(this.options.dbName, 1);
+      req.onupgradeneeded = () => {
+        const db = req.result;
+        if (!db.objectStoreNames.contains('entries')) {
+          db.createObjectStore('entries', { keyPath: 'path' });
+        }
+      };
+      req.onsuccess = () => resolve(req.result);
+      req.onerror = () => reject(req.error);
+    });
+  }
+
+  private async sweepExpired(): Promise<void> {
+    if (!this.idb) return;
+    const cutoff = Date.now() - this.options.maxAge;
+    const tx = this.idb.transaction('entries', 'readwrite');
+    const store = tx.objectStore('entries');
+    const req = store.openCursor();
+    return new Promise((resolve) => {
+      req.onsuccess = () => {
+        const cursor = req.result;
+        if (!cursor) { resolve(); return; }
+        const entry = cursor.value as CacheEntry;
+        if (entry.cachedAt < cutoff) cursor.delete();
+        cursor.continue();
+      };
+      req.onerror = () => resolve();
+    });
+  }
+
+  // --- get: stale-while-revalidate ---
+
+  async get(path: string): Promise<unknown> {
+    if (!this.options.enabled) return this.client.get(path);
+
+    // Memory hit + subscribed = fresh
+    const mem = this.memory.get(path);
+    if (mem && this.subscribedPaths.has(path)) return mem.data;
+
+    // Memory hit + not subscribed = return stale, revalidate
+    if (mem) {
+      this.revalidate(path);
+      return mem.data;
+    }
+
+    // IDB check
+    const idbEntry = await this.idbGet(path);
+    if (idbEntry && (Date.now() - idbEntry.cachedAt) < this.options.maxAge) {
+      this.memSet(path, idbEntry);
+      this.revalidate(path);
+      return idbEntry.data;
+    }
+
+    // Network fetch
+    const snap = await this.client.getSnapshot(path);
+    const entry: CacheEntry = { path, data: snap.val(), updatedAt: snap.updatedAt ?? Date.now(), cachedAt: Date.now() };
+    this.memSet(path, entry);
+    this.idbSet(entry);
+    return entry.data;
+  }
+
+  private revalidate(path: string): void {
+    this.client.getSnapshot(path).then((snap) => {
+      const entry: CacheEntry = { path, data: snap.val(), updatedAt: snap.updatedAt ?? Date.now(), cachedAt: Date.now() };
+      this.memSet(path, entry);
+      this.idbSet(entry);
+    }).catch(() => {}); // silent background refetch
+  }
+
+  // --- subscriptions ---
+
+  on(path: string, cb: (snap: ValueSnapshot) => void): () => void {
+    this.subscribedPaths.add(path);
+    const off = this.client.on(path, (snap) => {
+      const entry: CacheEntry = { path, data: snap.val(), updatedAt: snap.updatedAt ?? Date.now(), cachedAt: Date.now() };
+      this.memSet(path, entry);
+      this.idbSet(entry);
+      cb(snap);
+    });
+    return () => {
+      off();
+      this.subscribedPaths.delete(path);
+    };
+  }
+
+  onChild(path: string, cb: (event: ChildEvent) => void): () => void {
+    const off = this.client.onChild(path, (event) => {
+      // Invalidate parent + ancestors on any child event
+      this.invalidatePath(path);
+      cb(event);
+    });
+    return off;
+  }
+
+  // --- writes: passthrough + invalidate ---
+
+  async set(path: string, value: unknown, opts?: { ttl?: number }): Promise<void> {
+    await this.client.set(path, value, opts);
+    this.invalidatePath(path);
+  }
+
+  async update(updates: Record<string, unknown>): Promise<void> {
+    await this.client.update(updates);
+    for (const path of Object.keys(updates)) this.invalidatePath(path);
+  }
+
+  async delete(path: string): Promise<void> {
+    await this.client.delete(path);
+    this.invalidatePath(path);
+  }
+
+  private invalidatePath(path: string): void {
+    this.invalidate(path);
+    for (const ancestor of ancestors(path)) this.invalidate(ancestor);
+  }
+
+  private invalidate(path: string): void {
+    this.memory.delete(path);
+    this.idbDelete(path);
+  }
+
+  // --- warmup ---
+
+  async warmup(paths: string[]): Promise<void> {
+    if (!this.idb) return;
+    const cutoff = Date.now() - this.options.maxAge;
+    // Fire all gets in a single transaction synchronously, then collect results
+    const tx = this.idb.transaction('entries', 'readonly');
+    const store = tx.objectStore('entries');
+    const promises = paths.map(path => new Promise<void>((resolve) => {
+      const req = store.get(path);
+      req.onsuccess = () => {
+        const entry = req.result as CacheEntry | undefined;
+        if (entry && entry.cachedAt >= cutoff) this.memSet(path, entry);
+        resolve();
+      };
+      req.onerror = () => resolve();
+    }));
+    await Promise.all(promises);
+  }
+
+  close(): void {
+    this.memory.clear();
+    this.subscribedPaths.clear();
+    this.idb?.close();
+    this.idb = null;
+  }
+
+  // --- memory LRU ---
+
+  private memSet(path: string, entry: CacheEntry): void {
+    // Delete first to re-insert at end (insertion order)
+    this.memory.delete(path);
+    this.memory.set(path, entry);
+    // Evict oldest if over limit
+    while (this.memory.size > this.options.maxMemoryEntries) {
+      const first = this.memory.keys().next().value;
+      if (first !== undefined) this.memory.delete(first);
+      else break;
+    }
+  }
+
+  // --- IDB helpers ---
+
+  private idbGet(path: string): Promise<CacheEntry | null> {
+    if (!this.idb) return Promise.resolve(null);
+    const tx = this.idb.transaction('entries', 'readonly');
+    const req = tx.objectStore('entries').get(path);
+    return new Promise((resolve) => {
+      req.onsuccess = () => resolve(req.result ?? null);
+      req.onerror = () => resolve(null);
+    });
+  }
+
+  private idbSet(entry: CacheEntry): void {
+    if (!this.idb) return;
+    try {
+      const tx = this.idb.transaction('entries', 'readwrite');
+      tx.objectStore('entries').put(entry);
+    } catch {}
+  }
+
+  private idbDelete(path: string): void {
+    if (!this.idb) return;
+    try {
+      const tx = this.idb.transaction('entries', 'readwrite');
+      tx.objectStore('entries').delete(path);
+    } catch {}
+  }
+}