@bod.ee/db 0.11.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/config-file.md +6 -0
- package/.claude/skills/deploying-bod-db.md +21 -0
- package/.claude/skills/using-bod-db.md +30 -0
- package/CLAUDE.md +3 -1
- package/README.md +14 -0
- package/docs/keyauth-integration.md +141 -0
- package/docs/para-chat-integration.md +198 -0
- package/docs/repl-stream-bloat-spec.md +48 -0
- package/package.json +1 -1
- package/src/client/BodClient.ts +4 -0
- package/src/client/BodClientCached.ts +91 -3
- package/src/server/BodDB.ts +3 -3
- package/src/server/ReplicationEngine.ts +254 -18
- package/src/server/Transport.ts +65 -29
- package/src/server/VFSEngine.ts +40 -2
- package/src/shared/protocol.ts +2 -0
- package/tests/repl-stream-bloat.test.ts +270 -0
- package/tests/replication-topology.test.ts +835 -0
|
@@ -182,7 +182,7 @@ export class BodClientCached {
|
|
|
182
182
|
|
|
183
183
|
vfs(): CachedVFSClient {
|
|
184
184
|
if (!this._vfs) {
|
|
185
|
-
this._vfs = new CachedVFSClient(this.client.vfs(), this.options.maxAge);
|
|
185
|
+
this._vfs = new CachedVFSClient(this.client.vfs(), this.options.maxAge, this.idb);
|
|
186
186
|
this._vfsUnsub = this.client.onChild('_vfs', () => {
|
|
187
187
|
// _vfs events are coarse (top-level key only), clear all VFS caches
|
|
188
188
|
this._vfs?.clear();
|
|
@@ -252,18 +252,34 @@ export class CachedVFSClient {
|
|
|
252
252
|
constructor(
|
|
253
253
|
private raw: VFSClient,
|
|
254
254
|
private maxAge: number,
|
|
255
|
+
private idb: IDBDatabase | null = null,
|
|
255
256
|
) {}
|
|
256
257
|
|
|
257
258
|
async stat(path: string): Promise<FileStat | null> {
|
|
258
259
|
const cached = this.statCache.get(path);
|
|
259
260
|
if (cached && Date.now() - cached.cachedAt < this.maxAge) {
|
|
260
261
|
this._stats.hits++;
|
|
261
|
-
this.raw.stat(path).then(r =>
|
|
262
|
+
this.raw.stat(path).then(r => {
|
|
263
|
+
this.statCache.set(path, { data: r, cachedAt: Date.now() });
|
|
264
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: r, cachedAt: Date.now() });
|
|
265
|
+
}).catch(() => {});
|
|
262
266
|
return cached.data;
|
|
263
267
|
}
|
|
268
|
+
// Check IDB before network
|
|
269
|
+
const idbEntry = await this.idbGetVfs(`vfs:stat:${path}`);
|
|
270
|
+
if (idbEntry && Date.now() - idbEntry.cachedAt < this.maxAge) {
|
|
271
|
+
this._stats.hits++;
|
|
272
|
+
this.statCache.set(path, idbEntry);
|
|
273
|
+
this.raw.stat(path).then(r => {
|
|
274
|
+
this.statCache.set(path, { data: r, cachedAt: Date.now() });
|
|
275
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: r, cachedAt: Date.now() });
|
|
276
|
+
}).catch(() => {});
|
|
277
|
+
return idbEntry.data as FileStat | null;
|
|
278
|
+
}
|
|
264
279
|
this._stats.misses++;
|
|
265
280
|
const result = await this.raw.stat(path);
|
|
266
281
|
this.statCache.set(path, { data: result, cachedAt: Date.now() });
|
|
282
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: result, cachedAt: Date.now() });
|
|
267
283
|
return result;
|
|
268
284
|
}
|
|
269
285
|
|
|
@@ -271,15 +287,34 @@ export class CachedVFSClient {
|
|
|
271
287
|
const cached = this.listCache.get(path);
|
|
272
288
|
if (cached && Date.now() - cached.cachedAt < this.maxAge) {
|
|
273
289
|
this._stats.hits++;
|
|
274
|
-
this.raw.list(path).then(r =>
|
|
290
|
+
this.raw.list(path).then(r => {
|
|
291
|
+
this.listCache.set(path, { data: r, cachedAt: Date.now() });
|
|
292
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: r, cachedAt: Date.now() });
|
|
293
|
+
}).catch(() => {});
|
|
275
294
|
return cached.data;
|
|
276
295
|
}
|
|
296
|
+
// Check IDB before network
|
|
297
|
+
const idbEntry = await this.idbGetVfs(`vfs:list:${path}`);
|
|
298
|
+
if (idbEntry && Date.now() - idbEntry.cachedAt < this.maxAge) {
|
|
299
|
+
this._stats.hits++;
|
|
300
|
+
this.listCache.set(path, idbEntry as { data: FileStat[]; cachedAt: number });
|
|
301
|
+
this.raw.list(path).then(r => {
|
|
302
|
+
this.listCache.set(path, { data: r, cachedAt: Date.now() });
|
|
303
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: r, cachedAt: Date.now() });
|
|
304
|
+
}).catch(() => {});
|
|
305
|
+
return idbEntry.data as FileStat[];
|
|
306
|
+
}
|
|
277
307
|
this._stats.misses++;
|
|
278
308
|
const result = await this.raw.list(path);
|
|
279
309
|
this.listCache.set(path, { data: result, cachedAt: Date.now() });
|
|
310
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: result, cachedAt: Date.now() });
|
|
280
311
|
return result;
|
|
281
312
|
}
|
|
282
313
|
|
|
314
|
+
async tree(path: string, opts?: { hiddenPaths?: string[]; hideDotfiles?: boolean }): Promise<any[]> {
|
|
315
|
+
return this.raw.tree(path, opts);
|
|
316
|
+
}
|
|
317
|
+
|
|
283
318
|
async upload(path: string, data: Uint8Array, mime?: string): Promise<FileStat> {
|
|
284
319
|
const r = await this.raw.upload(path, data, mime);
|
|
285
320
|
this.invalidatePath(path);
|
|
@@ -296,6 +331,7 @@ export class CachedVFSClient {
|
|
|
296
331
|
this.invalidatePath(path);
|
|
297
332
|
// Also invalidate the dir's own list (was empty/nonexistent before)
|
|
298
333
|
this.listCache.delete(path);
|
|
334
|
+
this.idbDeleteVfs(`vfs:list:${path}`);
|
|
299
335
|
return r;
|
|
300
336
|
}
|
|
301
337
|
|
|
@@ -313,14 +349,17 @@ export class CachedVFSClient {
|
|
|
313
349
|
invalidatePath(filePath: string) {
|
|
314
350
|
this._stats.invalidations++;
|
|
315
351
|
this.statCache.delete(filePath);
|
|
352
|
+
this.idbDeleteVfs(`vfs:stat:${filePath}`);
|
|
316
353
|
const parent = filePath.includes('/') ? filePath.slice(0, filePath.lastIndexOf('/')) : '';
|
|
317
354
|
this.listCache.delete(parent);
|
|
355
|
+
this.idbDeleteVfs(`vfs:list:${parent}`);
|
|
318
356
|
}
|
|
319
357
|
|
|
320
358
|
clear() {
|
|
321
359
|
this._stats.pushClears++;
|
|
322
360
|
this.listCache.clear();
|
|
323
361
|
this.statCache.clear();
|
|
362
|
+
this.idbClearVfs();
|
|
324
363
|
}
|
|
325
364
|
|
|
326
365
|
/** Cache stats for diagnostics. Use in browser: `__bodCache.vfs().stats` */
|
|
@@ -334,4 +373,53 @@ export class CachedVFSClient {
|
|
|
334
373
|
statCacheSize: this.statCache.size,
|
|
335
374
|
};
|
|
336
375
|
}
|
|
376
|
+
|
|
377
|
+
// --- IDB helpers for VFS cache ---
|
|
378
|
+
|
|
379
|
+
private idbGetVfs(key: string): Promise<{ data: unknown; cachedAt: number } | null> {
|
|
380
|
+
if (!this.idb) return Promise.resolve(null);
|
|
381
|
+
try {
|
|
382
|
+
const tx = this.idb.transaction('entries', 'readonly');
|
|
383
|
+
const req = tx.objectStore('entries').get(key);
|
|
384
|
+
return new Promise(resolve => {
|
|
385
|
+
req.onsuccess = () => {
|
|
386
|
+
const entry = req.result;
|
|
387
|
+
resolve(entry ? { data: entry.data, cachedAt: entry.cachedAt } : null);
|
|
388
|
+
};
|
|
389
|
+
req.onerror = () => resolve(null);
|
|
390
|
+
});
|
|
391
|
+
} catch { return Promise.resolve(null); }
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
private idbSetVfs(key: string, value: { data: unknown; cachedAt: number }): void {
|
|
395
|
+
if (!this.idb) return;
|
|
396
|
+
try {
|
|
397
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
398
|
+
tx.objectStore('entries').put({ path: key, data: value.data, cachedAt: value.cachedAt, updatedAt: Date.now() });
|
|
399
|
+
} catch {}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
private idbDeleteVfs(key: string): void {
|
|
403
|
+
if (!this.idb) return;
|
|
404
|
+
try {
|
|
405
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
406
|
+
tx.objectStore('entries').delete(key);
|
|
407
|
+
} catch {}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
private idbClearVfs(): void {
|
|
411
|
+
if (!this.idb) return;
|
|
412
|
+
try {
|
|
413
|
+
// Delete all vfs: prefixed entries
|
|
414
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
415
|
+
const store = tx.objectStore('entries');
|
|
416
|
+
const req = store.openCursor();
|
|
417
|
+
req.onsuccess = () => {
|
|
418
|
+
const cursor = req.result;
|
|
419
|
+
if (!cursor) return;
|
|
420
|
+
if (typeof cursor.key === 'string' && cursor.key.startsWith('vfs:')) cursor.delete();
|
|
421
|
+
cursor.continue();
|
|
422
|
+
};
|
|
423
|
+
} catch {}
|
|
424
|
+
}
|
|
337
425
|
}
|
package/src/server/BodDB.ts
CHANGED
|
@@ -145,12 +145,12 @@ export class BodDB {
|
|
|
145
145
|
// Init replication
|
|
146
146
|
if (this.options.replication) {
|
|
147
147
|
this.replication = new ReplicationEngine(this, this.options.replication);
|
|
148
|
-
// Auto-add _repl compaction for
|
|
149
|
-
if (this.replication.
|
|
148
|
+
// Auto-add _repl compaction for any node that emits to _repl
|
|
149
|
+
if (this.replication.emitsToRepl) {
|
|
150
150
|
const compactOpts = this.options.replication.compact ?? { keepKey: 'path', maxCount: 10000 };
|
|
151
151
|
this.stream.options.compact = { ...this.stream.options.compact, _repl: compactOpts };
|
|
152
152
|
}
|
|
153
|
-
_log.info(`Replication enabled (role: ${this.replication.
|
|
153
|
+
_log.info(`Replication enabled (role: ${this.options.replication.role}${this.replication.router ? ', per-path topology' : ''})`);
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
_log.info('BodDB ready');
|
|
@@ -2,6 +2,7 @@ import type { BodDB } from './BodDB.ts';
|
|
|
2
2
|
import { BodClient } from '../client/BodClient.ts';
|
|
3
3
|
import type { ClientMessage } from '../shared/protocol.ts';
|
|
4
4
|
import type { CompactOptions } from './StreamEngine.ts';
|
|
5
|
+
import type { ComponentLogger } from '../shared/logger.ts';
|
|
5
6
|
|
|
6
7
|
export interface ReplEvent {
|
|
7
8
|
op: 'set' | 'delete' | 'push';
|
|
@@ -20,6 +21,94 @@ export interface WriteEvent {
|
|
|
20
21
|
ttl?: number;
|
|
21
22
|
}
|
|
22
23
|
|
|
24
|
+
// --- Per-path topology types ---
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Per-path replication mode:
|
|
28
|
+
* - `primary`: local-authoritative, emits to _repl, not pulled from remote
|
|
29
|
+
* - `replica`: remote-authoritative, pulled from remote, client writes proxied to primary
|
|
30
|
+
* - `sync`: emits + pulls (one-directional pull — remote must also configure a source pointing back for true bidirectional)
|
|
31
|
+
* - `readonly`: pulled from remote, client writes rejected
|
|
32
|
+
* - `writeonly`: emits only, not pulled from remote, client writes allowed locally
|
|
33
|
+
*/
|
|
34
|
+
export type PathMode = 'primary' | 'replica' | 'sync' | 'readonly' | 'writeonly';
|
|
35
|
+
|
|
36
|
+
export interface PathTopology {
|
|
37
|
+
path: string;
|
|
38
|
+
mode: PathMode;
|
|
39
|
+
/** Override write behavior: 'reject' rejects writes even on replica paths (default: proxy for replica, reject for readonly) */
|
|
40
|
+
writeProxy?: 'proxy' | 'optimistic' | 'reject';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Resolves per-path replication mode via longest-prefix match */
|
|
44
|
+
export class PathTopologyRouter {
|
|
45
|
+
private entries: PathTopology[];
|
|
46
|
+
private fallbackMode: PathMode;
|
|
47
|
+
|
|
48
|
+
constructor(paths: Array<string | PathTopology>, fallbackRole: 'primary' | 'replica') {
|
|
49
|
+
this.fallbackMode = fallbackRole === 'primary' ? 'primary' : 'replica';
|
|
50
|
+
this.entries = paths.map(p =>
|
|
51
|
+
typeof p === 'string' ? { path: p, mode: 'sync' as PathMode } : p
|
|
52
|
+
);
|
|
53
|
+
// Sort longest-first for prefix matching
|
|
54
|
+
this.entries.sort((a, b) => b.path.length - a.path.length);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Longest-prefix match, falls back to role-based mode */
|
|
58
|
+
resolve(path: string): PathTopology {
|
|
59
|
+
for (const e of this.entries) {
|
|
60
|
+
if (path === e.path || path.startsWith(e.path + '/')) {
|
|
61
|
+
return e;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return { path: '', mode: this.fallbackMode };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Should this path's writes be emitted to _repl? */
|
|
68
|
+
shouldEmit(path: string): boolean {
|
|
69
|
+
const { mode } = this.resolve(path);
|
|
70
|
+
return mode === 'primary' || mode === 'sync' || mode === 'writeonly';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Should incoming repl events for this path be applied? */
|
|
74
|
+
shouldApply(path: string): boolean {
|
|
75
|
+
const { mode } = this.resolve(path);
|
|
76
|
+
return mode === 'replica' || mode === 'sync' || mode === 'readonly';
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Should client writes to this path be proxied to primary? */
|
|
80
|
+
shouldProxy(path: string): boolean {
|
|
81
|
+
const { mode, writeProxy } = this.resolve(path);
|
|
82
|
+
if (writeProxy === 'reject') return false;
|
|
83
|
+
if (mode === 'readonly') return false; // readonly rejects, not proxies
|
|
84
|
+
return mode === 'replica';
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Should client writes to this path be rejected? */
|
|
88
|
+
shouldReject(path: string): boolean {
|
|
89
|
+
const { mode, writeProxy } = this.resolve(path);
|
|
90
|
+
if (writeProxy === 'reject') return true;
|
|
91
|
+
return mode === 'readonly';
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Get all configured entries */
|
|
95
|
+
getEntries(): readonly PathTopology[] { return this.entries; }
|
|
96
|
+
|
|
97
|
+
/** Get paths that need pulling from primary (ongoing stream subscription) */
|
|
98
|
+
getReplicaPaths(): string[] {
|
|
99
|
+
return this.entries
|
|
100
|
+
.filter(e => e.mode === 'replica' || e.mode === 'readonly' || e.mode === 'sync')
|
|
101
|
+
.map(e => e.path);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Get paths that need full bootstrap from primary (excludes sync — sync paths get ongoing events only, avoiding overwrite of local state) */
|
|
105
|
+
getBootstrapPaths(): string[] {
|
|
106
|
+
return this.entries
|
|
107
|
+
.filter(e => e.mode === 'replica' || e.mode === 'readonly')
|
|
108
|
+
.map(e => e.path);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
23
112
|
export class ReplicationSource {
|
|
24
113
|
url: string = '';
|
|
25
114
|
auth?: () => string | Promise<string>;
|
|
@@ -34,17 +123,21 @@ export class ReplicationOptions {
|
|
|
34
123
|
primaryUrl?: string;
|
|
35
124
|
primaryAuth?: () => string | Promise<string>;
|
|
36
125
|
replicaId?: string;
|
|
126
|
+
/** Paths excluded from replication. Note: when `paths` explicitly configures a prefix (e.g. `_auth`), it overrides this exclusion. */
|
|
37
127
|
excludePrefixes: string[] = ['_repl', '_streams', '_mq', '_admin', '_auth'];
|
|
38
128
|
/** Bootstrap replica from primary's full state before applying _repl stream */
|
|
39
129
|
fullBootstrap: boolean = true;
|
|
40
130
|
compact?: CompactOptions;
|
|
41
131
|
sources?: ReplicationSource[];
|
|
132
|
+
/** Per-path topology: strings default to 'sync', objects specify mode. When absent, role governs all paths. */
|
|
133
|
+
paths?: Array<string | PathTopology>;
|
|
42
134
|
}
|
|
43
135
|
|
|
44
136
|
type ProxyableMessage = Extract<ClientMessage, { op: 'set' | 'delete' | 'update' | 'push' | 'batch' }>;
|
|
45
137
|
|
|
46
138
|
export class ReplicationEngine {
|
|
47
139
|
readonly options: ReplicationOptions;
|
|
140
|
+
readonly router: PathTopologyRouter | null = null;
|
|
48
141
|
private client: BodClient | null = null;
|
|
49
142
|
private unsubWrite: (() => void) | null = null;
|
|
50
143
|
private unsubStream: (() => void) | null = null;
|
|
@@ -53,9 +146,15 @@ export class ReplicationEngine {
|
|
|
53
146
|
private _seq = 0;
|
|
54
147
|
private _emitting = false;
|
|
55
148
|
private _pendingReplEvents: WriteEvent[] | null = null;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
149
|
+
private log: ComponentLogger;
|
|
150
|
+
|
|
151
|
+
/** When no router, falls back to role-based check. With router, both return false — use emitsToRepl/pullsFromPrimary instead. */
|
|
152
|
+
get isReplica(): boolean { return !this.router && this.options.role === 'replica'; }
|
|
153
|
+
get isPrimary(): boolean { return !this.router && this.options.role === 'primary'; }
|
|
154
|
+
/** True if this node emits write events to _repl (primary, router with emitting paths, or any node with startPrimary called) */
|
|
155
|
+
get emitsToRepl(): boolean { return this.isPrimary || !!this.router; }
|
|
156
|
+
/** True if this node pulls events from a primary */
|
|
157
|
+
get pullsFromPrimary(): boolean { return this.isReplica || (!!this.router && this.router.getReplicaPaths().length > 0); }
|
|
59
158
|
get started(): boolean { return this._started; }
|
|
60
159
|
get seq(): number { return this._seq; }
|
|
61
160
|
|
|
@@ -65,6 +164,7 @@ export class ReplicationEngine {
|
|
|
65
164
|
role: this.options.role,
|
|
66
165
|
started: this._started,
|
|
67
166
|
seq: this._seq,
|
|
167
|
+
topology: this.router ? this.router.getEntries().map(e => ({ path: e.path, mode: e.mode })) : null,
|
|
68
168
|
sources: (this.options.sources ?? []).map((s, i) => {
|
|
69
169
|
const conn = this.sourceConns[i];
|
|
70
170
|
return {
|
|
@@ -84,9 +184,42 @@ export class ReplicationEngine {
|
|
|
84
184
|
options?: Partial<ReplicationOptions>,
|
|
85
185
|
) {
|
|
86
186
|
this.options = { ...new ReplicationOptions(), ...options };
|
|
187
|
+
this.log = db.log.forComponent('repl');
|
|
188
|
+
if (this.options.paths?.length) {
|
|
189
|
+
this.router = new PathTopologyRouter(this.options.paths, this.options.role);
|
|
190
|
+
}
|
|
87
191
|
if (!this.options.replicaId) {
|
|
88
|
-
|
|
192
|
+
// Derive stable ID from config when possible, fall back to random
|
|
193
|
+
if (this.options.primaryUrl) {
|
|
194
|
+
const seed = `${this.options.primaryUrl}:${(this.options.paths ?? []).map(p => typeof p === 'string' ? p : p.path).sort().join('+')}`;
|
|
195
|
+
// Simple hash for stability across restarts
|
|
196
|
+
let h = 0;
|
|
197
|
+
for (let i = 0; i < seed.length; i++) h = ((h << 5) - h + seed.charCodeAt(i)) | 0;
|
|
198
|
+
this.options.replicaId = `replica_${(h >>> 0).toString(36)}`;
|
|
199
|
+
} else {
|
|
200
|
+
this.options.replicaId = `replica_${Math.random().toString(36).slice(2, 10)}`;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Should client writes to this path be proxied to primary? */
|
|
206
|
+
shouldProxyPath(path: string): boolean {
|
|
207
|
+
if (this.router) {
|
|
208
|
+
// Only proxy if we have a client connection to proxy through
|
|
209
|
+
return this.router.shouldProxy(path) && !!this.client;
|
|
89
210
|
}
|
|
211
|
+
return this.options.role === 'replica';
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Should client writes to this path be rejected? */
|
|
215
|
+
shouldRejectPath(path: string): boolean {
|
|
216
|
+
if (this.router) {
|
|
217
|
+
const shouldProxy = this.router.shouldProxy(path);
|
|
218
|
+
// If path wants proxy but no client → reject instead of crashing
|
|
219
|
+
if (shouldProxy && !this.client) return true;
|
|
220
|
+
return this.router.shouldReject(path);
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
90
223
|
}
|
|
91
224
|
|
|
92
225
|
/** Start replication — primary listens for writes, replica connects and consumes */
|
|
@@ -94,9 +227,19 @@ export class ReplicationEngine {
|
|
|
94
227
|
if (this._started) return;
|
|
95
228
|
this._started = true;
|
|
96
229
|
|
|
97
|
-
|
|
230
|
+
// Always hook writes for emit (router or primary will filter)
|
|
231
|
+
if (this.router || this.isPrimary) {
|
|
98
232
|
this.startPrimary();
|
|
99
|
-
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// Connect to primary for replica/readonly/sync paths
|
|
236
|
+
if (this.router) {
|
|
237
|
+
const pullPaths = this.router.getReplicaPaths();
|
|
238
|
+
if (pullPaths.length) {
|
|
239
|
+
if (!this.options.primaryUrl) throw new Error('primaryUrl is required when paths include replica/readonly/sync modes');
|
|
240
|
+
await this.startReplicaForPaths(pullPaths);
|
|
241
|
+
}
|
|
242
|
+
} else if (this.options.role === 'replica') {
|
|
100
243
|
await this.startReplica();
|
|
101
244
|
}
|
|
102
245
|
|
|
@@ -108,6 +251,7 @@ export class ReplicationEngine {
|
|
|
108
251
|
/** Stop replication */
|
|
109
252
|
stop(): void {
|
|
110
253
|
this._started = false;
|
|
254
|
+
if (this._compactTimer) { clearInterval(this._compactTimer); this._compactTimer = null; }
|
|
111
255
|
this.unsubWrite?.();
|
|
112
256
|
this.unsubWrite = null;
|
|
113
257
|
this.unsubStream?.();
|
|
@@ -123,7 +267,7 @@ export class ReplicationEngine {
|
|
|
123
267
|
|
|
124
268
|
/** Proxy a write operation to the primary (replica mode) */
|
|
125
269
|
async proxyWrite(msg: ProxyableMessage): Promise<unknown> {
|
|
126
|
-
if (!this.client) throw new Error('Replica not connected to primary');
|
|
270
|
+
if (!this.client) throw new Error('Replica not connected to primary — ensure primaryUrl is set and start() has been called');
|
|
127
271
|
switch (msg.op) {
|
|
128
272
|
case 'set':
|
|
129
273
|
await this.client.set(msg.path, msg.value, msg.ttl ? { ttl: msg.ttl } : undefined);
|
|
@@ -145,20 +289,51 @@ export class ReplicationEngine {
|
|
|
145
289
|
|
|
146
290
|
// --- Primary mode ---
|
|
147
291
|
|
|
292
|
+
private _compactTimer: ReturnType<typeof setInterval> | null = null;
|
|
293
|
+
|
|
148
294
|
private startPrimary(): void {
|
|
149
295
|
this.unsubWrite = this.db.onWrite((ev: WriteEvent) => {
|
|
150
296
|
this.emit(ev);
|
|
151
297
|
});
|
|
298
|
+
|
|
299
|
+
// Auto-compact _repl stream to prevent unbounded growth
|
|
300
|
+
const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
|
|
301
|
+
if (compact.maxCount || compact.maxAge) {
|
|
302
|
+
// Compact on startup
|
|
303
|
+
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
304
|
+
// Then periodically (every 5 minutes)
|
|
305
|
+
this._compactTimer = setInterval(() => {
|
|
306
|
+
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
307
|
+
}, 5 * 60_000);
|
|
308
|
+
}
|
|
152
309
|
}
|
|
153
310
|
|
|
154
311
|
private isExcluded(path: string): boolean {
|
|
155
|
-
|
|
312
|
+
if (this.options.excludePrefixes.some(p => path.startsWith(p))) {
|
|
313
|
+
// If router explicitly configures this path, don't exclude it
|
|
314
|
+
if (this.router) {
|
|
315
|
+
const resolved = this.router.resolve(path);
|
|
316
|
+
if (resolved.path !== '') return false; // explicitly configured → not excluded
|
|
317
|
+
}
|
|
318
|
+
return true;
|
|
319
|
+
}
|
|
320
|
+
return false;
|
|
156
321
|
}
|
|
157
322
|
|
|
158
323
|
/** Buffer replication events during transactions, emit immediately otherwise */
|
|
159
324
|
private emit(ev: WriteEvent): void {
|
|
160
325
|
if (this._emitting) return;
|
|
161
|
-
if (this.
|
|
326
|
+
if (this.router) {
|
|
327
|
+
// Single resolve: check exclusion override + shouldEmit together
|
|
328
|
+
const resolved = this.router.resolve(ev.path);
|
|
329
|
+
const isConfigured = resolved.path !== '';
|
|
330
|
+
// If in excludePrefixes but not explicitly configured, skip
|
|
331
|
+
if (!isConfigured && this.options.excludePrefixes.some(p => ev.path.startsWith(p))) return;
|
|
332
|
+
const mode = resolved.mode;
|
|
333
|
+
if (mode !== 'primary' && mode !== 'sync' && mode !== 'writeonly') return;
|
|
334
|
+
} else {
|
|
335
|
+
if (this.isExcluded(ev.path)) return;
|
|
336
|
+
}
|
|
162
337
|
|
|
163
338
|
// If buffering (transaction in progress), collect events
|
|
164
339
|
if (this._pendingReplEvents) {
|
|
@@ -202,6 +377,62 @@ export class ReplicationEngine {
|
|
|
202
377
|
|
|
203
378
|
// --- Replica mode ---
|
|
204
379
|
|
|
380
|
+
/** Start replica for specific path prefixes (router-based) */
|
|
381
|
+
private async startReplicaForPaths(pathPrefixes: string[]): Promise<void> {
|
|
382
|
+
if (!this.options.primaryUrl) throw new Error('primaryUrl required for per-path replication');
|
|
383
|
+
|
|
384
|
+
this.client = new BodClient({
|
|
385
|
+
url: this.options.primaryUrl,
|
|
386
|
+
auth: this.options.primaryAuth,
|
|
387
|
+
});
|
|
388
|
+
await this.client.connect();
|
|
389
|
+
this.log.info(`Connected to primary ${this.options.primaryUrl} (paths: ${pathPrefixes.join(', ')})`);
|
|
390
|
+
|
|
391
|
+
// Bootstrap only replica/readonly paths (sync paths get ongoing events only — avoids overwriting local state)
|
|
392
|
+
const bootstrapPaths = this.router!.getBootstrapPaths();
|
|
393
|
+
if (this.options.fullBootstrap && bootstrapPaths.length) {
|
|
394
|
+
await this.bootstrapFullState(bootstrapPaths);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Stream bootstrap filtered
|
|
398
|
+
const snapshot = await this.client!.streamMaterialize('_repl', { keepKey: 'path' });
|
|
399
|
+
if (snapshot) {
|
|
400
|
+
this.db.setReplaying(true);
|
|
401
|
+
try {
|
|
402
|
+
for (const [, event] of Object.entries(snapshot)) {
|
|
403
|
+
const ev = event as ReplEvent;
|
|
404
|
+
if (this.matchesPathPrefixes(ev.path, pathPrefixes)) {
|
|
405
|
+
this.applyEvent(ev);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
} finally {
|
|
409
|
+
this.db.setReplaying(false);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
// Subscribe to ongoing events, filter by paths
|
|
414
|
+
const groupId = this.options.replicaId!;
|
|
415
|
+
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
416
|
+
this.db.setReplaying(true);
|
|
417
|
+
try {
|
|
418
|
+
for (const e of events) {
|
|
419
|
+
const ev = e.val() as ReplEvent;
|
|
420
|
+
if (this.matchesPathPrefixes(ev.path, pathPrefixes)) {
|
|
421
|
+
this.applyEvent(ev);
|
|
422
|
+
}
|
|
423
|
+
this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
|
|
424
|
+
}
|
|
425
|
+
} finally {
|
|
426
|
+
this.db.setReplaying(false);
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Check if path matches any of the given prefixes */
|
|
432
|
+
private matchesPathPrefixes(path: string, prefixes: string[]): boolean {
|
|
433
|
+
return prefixes.some(p => path === p || path.startsWith(p + '/'));
|
|
434
|
+
}
|
|
435
|
+
|
|
205
436
|
private async startReplica(): Promise<void> {
|
|
206
437
|
if (!this.options.primaryUrl) throw new Error('primaryUrl required for replica mode');
|
|
207
438
|
|
|
@@ -211,7 +442,7 @@ export class ReplicationEngine {
|
|
|
211
442
|
});
|
|
212
443
|
|
|
213
444
|
await this.client.connect();
|
|
214
|
-
|
|
445
|
+
this.log.info(`Connected to primary ${this.options.primaryUrl}`);
|
|
215
446
|
|
|
216
447
|
// Full state bootstrap: fetch all data from primary (catches pre-replication data)
|
|
217
448
|
if (this.options.fullBootstrap) {
|
|
@@ -220,7 +451,7 @@ export class ReplicationEngine {
|
|
|
220
451
|
|
|
221
452
|
// Stream bootstrap: apply _repl events on top (catches recent writes, deduped by idempotent set)
|
|
222
453
|
const snapshot = await this.client!.streamMaterialize('_repl', { keepKey: 'path' });
|
|
223
|
-
|
|
454
|
+
this.log.info(`Stream bootstrap: ${snapshot ? Object.keys(snapshot).length + ' events' : 'no events'}`);
|
|
224
455
|
if (snapshot) {
|
|
225
456
|
this.db.setReplaying(true);
|
|
226
457
|
try {
|
|
@@ -235,9 +466,9 @@ export class ReplicationEngine {
|
|
|
235
466
|
|
|
236
467
|
// Subscribe to ongoing events
|
|
237
468
|
const groupId = this.options.replicaId!;
|
|
238
|
-
|
|
469
|
+
this.log.info(`Subscribing to stream as '${groupId}'`);
|
|
239
470
|
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
240
|
-
|
|
471
|
+
this.log.debug(`Received ${events.length} events`);
|
|
241
472
|
this.db.setReplaying(true);
|
|
242
473
|
try {
|
|
243
474
|
for (const e of events) {
|
|
@@ -252,13 +483,16 @@ export class ReplicationEngine {
|
|
|
252
483
|
});
|
|
253
484
|
}
|
|
254
485
|
|
|
255
|
-
/** Fetch full DB state from primary and apply locally */
|
|
256
|
-
private async bootstrapFullState(): Promise<void> {
|
|
486
|
+
/** Fetch full DB state from primary and apply locally. Optional pathPrefixes filters which top-level keys to pull. */
|
|
487
|
+
private async bootstrapFullState(pathPrefixes?: string[]): Promise<void> {
|
|
257
488
|
const topLevel = await this.client!.getShallow();
|
|
258
|
-
|
|
489
|
+
let keys = topLevel
|
|
259
490
|
.map(e => e.key)
|
|
260
491
|
.filter(k => !this.isExcluded(k));
|
|
261
|
-
|
|
492
|
+
if (pathPrefixes) {
|
|
493
|
+
keys = keys.filter(k => this.matchesPathPrefixes(k, pathPrefixes));
|
|
494
|
+
}
|
|
495
|
+
this.log.info(`Full bootstrap: ${keys.length} top-level keys [${keys.join(', ')}]`);
|
|
262
496
|
if (keys.length === 0) return;
|
|
263
497
|
|
|
264
498
|
this.db.setReplaying(true);
|
|
@@ -283,7 +517,7 @@ export class ReplicationEngine {
|
|
|
283
517
|
for (let i = 0; i < results.length; i++) {
|
|
284
518
|
if (results[i].status === 'rejected') {
|
|
285
519
|
const src = this.options.sources![i];
|
|
286
|
-
|
|
520
|
+
this.log.error(`source ${src.url} paths=[${src.paths}] failed:`, (results[i] as PromiseRejectedResult).reason);
|
|
287
521
|
}
|
|
288
522
|
}
|
|
289
523
|
}
|
|
@@ -345,6 +579,8 @@ export class ReplicationEngine {
|
|
|
345
579
|
|
|
346
580
|
private applyEvent(ev: ReplEvent, source?: ReplicationSource): void {
|
|
347
581
|
const path = source ? this.remapPath(ev.path, source) : ev.path;
|
|
582
|
+
// Defense-in-depth: skip events for paths we shouldn't apply (primary/writeonly)
|
|
583
|
+
if (!source && this.router && !this.router.shouldApply(path)) return;
|
|
348
584
|
switch (ev.op) {
|
|
349
585
|
case 'set':
|
|
350
586
|
this.db.set(path, ev.value, ev.ttl ? { ttl: ev.ttl } : undefined);
|