@bod.ee/db 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +7 -1
- package/.claude/skills/config-file.md +7 -0
- package/.claude/skills/deploying-bod-db.md +34 -0
- package/.claude/skills/developing-bod-db.md +20 -2
- package/.claude/skills/using-bod-db.md +165 -0
- package/.github/workflows/test-and-publish.yml +111 -0
- package/CLAUDE.md +10 -1
- package/README.md +57 -2
- package/admin/proxy.ts +79 -0
- package/admin/rules.ts +1 -1
- package/admin/server.ts +134 -50
- package/admin/ui.html +729 -18
- package/cli.ts +10 -0
- package/client.ts +3 -2
- package/config.ts +1 -0
- package/deploy/boddb-il.yaml +14 -0
- package/deploy/prod-il.config.ts +19 -0
- package/deploy/prod.config.ts +1 -0
- package/index.ts +3 -0
- package/package.json +11 -2
- package/src/client/BodClient.ts +129 -6
- package/src/client/CachedClient.ts +228 -0
- package/src/server/BodDB.ts +145 -1
- package/src/server/ReplicationEngine.ts +332 -0
- package/src/server/StorageEngine.ts +19 -0
- package/src/server/Transport.ts +577 -360
- package/src/server/VFSEngine.ts +172 -0
- package/src/shared/protocol.ts +25 -4
- package/tests/cached-client.test.ts +143 -0
- package/tests/replication.test.ts +404 -0
- package/tests/vfs.test.ts +166 -0
package/src/server/BodDB.ts
CHANGED
|
@@ -7,6 +7,8 @@ import { FTSEngine, type FTSEngineOptions } from './FTSEngine.ts';
|
|
|
7
7
|
import { VectorEngine, type VectorEngineOptions } from './VectorEngine.ts';
|
|
8
8
|
import { StreamEngine, type CompactOptions } from './StreamEngine.ts';
|
|
9
9
|
import { MQEngine, type MQEngineOptions } from './MQEngine.ts';
|
|
10
|
+
import { ReplicationEngine, type ReplicationOptions, type WriteEvent } from './ReplicationEngine.ts';
|
|
11
|
+
import { VFSEngine, type VFSEngineOptions } from './VFSEngine.ts';
|
|
10
12
|
import { validatePath } from '../shared/pathUtils.ts';
|
|
11
13
|
|
|
12
14
|
export interface TransactionProxy {
|
|
@@ -32,6 +34,10 @@ export class BodDBOptions {
|
|
|
32
34
|
compact?: Record<string, CompactOptions>;
|
|
33
35
|
/** Message queue config */
|
|
34
36
|
mq?: Partial<MQEngineOptions>;
|
|
37
|
+
/** Replication config */
|
|
38
|
+
replication?: Partial<ReplicationOptions>;
|
|
39
|
+
/** VFS config */
|
|
40
|
+
vfs?: Partial<VFSEngineOptions>;
|
|
35
41
|
port?: number;
|
|
36
42
|
auth?: TransportOptions['auth'];
|
|
37
43
|
transport?: Partial<TransportOptions>;
|
|
@@ -45,10 +51,17 @@ export class BodDB {
|
|
|
45
51
|
readonly mq: MQEngine;
|
|
46
52
|
readonly fts: FTSEngine | null = null;
|
|
47
53
|
readonly vectors: VectorEngine | null = null;
|
|
54
|
+
readonly vfs: VFSEngine | null = null;
|
|
48
55
|
readonly options: BodDBOptions;
|
|
56
|
+
replication: ReplicationEngine | null = null;
|
|
57
|
+
private _replaying = false;
|
|
58
|
+
private _onWriteHooks: Array<(ev: WriteEvent) => void> = [];
|
|
49
59
|
private _transport: Transport | null = null;
|
|
50
60
|
get transport(): Transport | null { return this._transport; }
|
|
51
61
|
private sweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
62
|
+
private _statsInterval: ReturnType<typeof setInterval> | null = null;
|
|
63
|
+
private _lastCpuUsage = process.cpuUsage();
|
|
64
|
+
private _lastCpuTime = performance.now();
|
|
52
65
|
|
|
53
66
|
constructor(options?: Partial<BodDBOptions>) {
|
|
54
67
|
this.options = { ...new BodDBOptions(), ...options };
|
|
@@ -89,10 +102,25 @@ export class BodDB {
|
|
|
89
102
|
this.vectors = new VectorEngine(this.storage.db, this.options.vectors);
|
|
90
103
|
}
|
|
91
104
|
|
|
105
|
+
// Initialize VFS if configured
|
|
106
|
+
if (this.options.vfs) {
|
|
107
|
+
(this as { vfs: VFSEngine }).vfs = new VFSEngine(this, this.options.vfs);
|
|
108
|
+
}
|
|
109
|
+
|
|
92
110
|
// Start TTL sweep
|
|
93
111
|
if (this.options.sweepInterval > 0) {
|
|
94
112
|
this.sweepTimer = setInterval(() => this.sweep(), this.options.sweepInterval);
|
|
95
113
|
}
|
|
114
|
+
|
|
115
|
+
// Init replication
|
|
116
|
+
if (this.options.replication) {
|
|
117
|
+
this.replication = new ReplicationEngine(this, this.options.replication);
|
|
118
|
+
// Auto-add _repl compaction for primary
|
|
119
|
+
if (this.replication.isPrimary) {
|
|
120
|
+
const compactOpts = this.options.replication.compact ?? { keepKey: 'path', maxCount: 10000 };
|
|
121
|
+
this.stream.options.compact = { ...this.stream.options.compact, _repl: compactOpts };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
96
124
|
}
|
|
97
125
|
|
|
98
126
|
/** Load rules from a JSON or TS file synchronously */
|
|
@@ -137,6 +165,21 @@ export class BodDB {
|
|
|
137
165
|
throw new Error(`Unsupported config file format: ${filePath}. Use .ts, .js, or .json.`);
|
|
138
166
|
}
|
|
139
167
|
|
|
168
|
+
/** Set replaying flag (used by ReplicationEngine to prevent re-emission) */
|
|
169
|
+
setReplaying(v: boolean): void { this._replaying = v; }
|
|
170
|
+
|
|
171
|
+
/** Register a write hook. Returns unsubscribe function. */
|
|
172
|
+
onWrite(cb: (ev: WriteEvent) => void): () => void {
|
|
173
|
+
this._onWriteHooks.push(cb);
|
|
174
|
+
return () => { this._onWriteHooks = this._onWriteHooks.filter(h => h !== cb); };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Fire write hooks (skipped during replication replay) */
|
|
178
|
+
private _fireWrite(ev: WriteEvent): void {
|
|
179
|
+
if (this._replaying) return;
|
|
180
|
+
for (const cb of this._onWriteHooks) cb(ev);
|
|
181
|
+
}
|
|
182
|
+
|
|
140
183
|
get(path: string): unknown {
|
|
141
184
|
return this.storage.get(path);
|
|
142
185
|
}
|
|
@@ -155,6 +198,7 @@ export class BodDB {
|
|
|
155
198
|
if (this.subs.hasSubscriptions) {
|
|
156
199
|
this.subs.notify(changed, (pp) => this.storage.get(pp), existedBefore);
|
|
157
200
|
}
|
|
201
|
+
this._fireWrite({ op: 'set', path: p, value, ttl: options?.ttl });
|
|
158
202
|
}
|
|
159
203
|
|
|
160
204
|
/** Manually trigger TTL sweep + stream auto-compact, returns expired paths */
|
|
@@ -163,6 +207,10 @@ export class BodDB {
|
|
|
163
207
|
if (expired.length > 0 && this.subs.hasSubscriptions) {
|
|
164
208
|
this.subs.notify(expired, (p) => this.storage.get(p));
|
|
165
209
|
}
|
|
210
|
+
// Fire delete events for expired paths (replication)
|
|
211
|
+
for (const p of expired) {
|
|
212
|
+
this._fireWrite({ op: 'delete', path: p });
|
|
213
|
+
}
|
|
166
214
|
this.stream.autoCompact();
|
|
167
215
|
this.mq.sweep();
|
|
168
216
|
return expired;
|
|
@@ -180,6 +228,10 @@ export class BodDB {
|
|
|
180
228
|
if (this.subs.hasSubscriptions) {
|
|
181
229
|
this.subs.notify(changed, (p) => this.storage.get(p), existedBefore);
|
|
182
230
|
}
|
|
231
|
+
// Emit per-path set events for replication (each path becomes its own repl event)
|
|
232
|
+
for (const [path, value] of Object.entries(updates)) {
|
|
233
|
+
this._fireWrite({ op: 'set', path: validatePath(path), value });
|
|
234
|
+
}
|
|
183
235
|
}
|
|
184
236
|
|
|
185
237
|
delete(path: string): void {
|
|
@@ -189,6 +241,7 @@ export class BodDB {
|
|
|
189
241
|
if (this.subs.hasSubscriptions) {
|
|
190
242
|
this.subs.notify([p], (pp) => this.storage.get(pp), existedBefore);
|
|
191
243
|
}
|
|
244
|
+
this._fireWrite({ op: 'delete', path: p });
|
|
192
245
|
}
|
|
193
246
|
|
|
194
247
|
/** Push a value with auto-generated time-sortable key (stored as single JSON row, not flattened) */
|
|
@@ -199,6 +252,9 @@ export class BodDB {
|
|
|
199
252
|
if (!duplicate && this.subs.hasSubscriptions) {
|
|
200
253
|
this.subs.notify(changedPaths, (pp) => this.storage.get(pp), existedBefore);
|
|
201
254
|
}
|
|
255
|
+
if (!duplicate) {
|
|
256
|
+
this._fireWrite({ op: 'push', path: p, value, pushKey: key });
|
|
257
|
+
}
|
|
202
258
|
return key;
|
|
203
259
|
}
|
|
204
260
|
|
|
@@ -247,6 +303,7 @@ export class BodDB {
|
|
|
247
303
|
transaction<T>(fn: (tx: TransactionProxy) => T): T {
|
|
248
304
|
const allChanged: string[] = [];
|
|
249
305
|
const allExistedBefore = new Set<string>();
|
|
306
|
+
const txWriteEvents: WriteEvent[] = [];
|
|
250
307
|
const proxy: TransactionProxy = {
|
|
251
308
|
get: (path: string) => this.storage.get(path),
|
|
252
309
|
set: (path: string, value: unknown) => {
|
|
@@ -255,6 +312,7 @@ export class BodDB {
|
|
|
255
312
|
for (const ep of this.snapshotExisting(p)) allExistedBefore.add(ep);
|
|
256
313
|
}
|
|
257
314
|
allChanged.push(...this.storage.set(path, value));
|
|
315
|
+
txWriteEvents.push({ op: 'set', path: p, value });
|
|
258
316
|
},
|
|
259
317
|
update: (updates: Record<string, unknown>) => {
|
|
260
318
|
if (this.subs.hasChildSubscriptions) {
|
|
@@ -263,6 +321,9 @@ export class BodDB {
|
|
|
263
321
|
}
|
|
264
322
|
}
|
|
265
323
|
allChanged.push(...this.storage.update(updates));
|
|
324
|
+
for (const [path, value] of Object.entries(updates)) {
|
|
325
|
+
txWriteEvents.push({ op: 'set', path: validatePath(path), value });
|
|
326
|
+
}
|
|
266
327
|
},
|
|
267
328
|
delete: (path: string) => {
|
|
268
329
|
const p = validatePath(path);
|
|
@@ -271,6 +332,7 @@ export class BodDB {
|
|
|
271
332
|
}
|
|
272
333
|
this.storage.delete(path);
|
|
273
334
|
allChanged.push(p);
|
|
335
|
+
txWriteEvents.push({ op: 'delete', path: p });
|
|
274
336
|
},
|
|
275
337
|
};
|
|
276
338
|
|
|
@@ -280,27 +342,109 @@ export class BodDB {
|
|
|
280
342
|
if (this.subs.hasSubscriptions && allChanged.length > 0) {
|
|
281
343
|
this.subs.notify(allChanged, (p) => this.storage.get(p), allExistedBefore);
|
|
282
344
|
}
|
|
345
|
+
for (const ev of txWriteEvents) this._fireWrite(ev);
|
|
283
346
|
return result;
|
|
284
347
|
}
|
|
285
348
|
|
|
286
|
-
/** Start
|
|
349
|
+
/** Start publishing process/db stats to `_admin/stats` (only when subscribers exist). */
|
|
350
|
+
private _startStatsPublisher(): void {
|
|
351
|
+
if (this._statsInterval) return;
|
|
352
|
+
const { statSync } = require('fs');
|
|
353
|
+
const { cpus, totalmem } = require('os');
|
|
354
|
+
let lastOsCpus = cpus();
|
|
355
|
+
|
|
356
|
+
this._statsInterval = setInterval(() => {
|
|
357
|
+
if (!this.subs.subscriberCount('_admin')) return;
|
|
358
|
+
|
|
359
|
+
const now = performance.now();
|
|
360
|
+
const cpu = process.cpuUsage();
|
|
361
|
+
const elapsedUs = (now - this._lastCpuTime) * 1000;
|
|
362
|
+
const cpuPercent = +((cpu.user - this._lastCpuUsage.user + cpu.system - this._lastCpuUsage.system) / elapsedUs * 100).toFixed(1);
|
|
363
|
+
this._lastCpuUsage = cpu;
|
|
364
|
+
this._lastCpuTime = now;
|
|
365
|
+
|
|
366
|
+
const mem = process.memoryUsage();
|
|
367
|
+
let nodeCount = 0;
|
|
368
|
+
try { nodeCount = (this.storage.db.query('SELECT COUNT(*) as n FROM nodes WHERE mq_status IS NULL').get() as any).n; } catch {}
|
|
369
|
+
let dbSizeMb = 0;
|
|
370
|
+
try { dbSizeMb = +(statSync(this.options.path).size / 1024 / 1024).toFixed(2); } catch {}
|
|
371
|
+
|
|
372
|
+
// System CPU
|
|
373
|
+
const cur = cpus();
|
|
374
|
+
let idleDelta = 0, totalDelta = 0;
|
|
375
|
+
for (let i = 0; i < cur.length; i++) {
|
|
376
|
+
const prev = lastOsCpus[i]?.times ?? cur[i].times;
|
|
377
|
+
const c = cur[i].times;
|
|
378
|
+
idleDelta += c.idle - prev.idle;
|
|
379
|
+
totalDelta += (c.user + c.nice + c.sys + c.irq + c.idle) - (prev.user + prev.nice + prev.sys + prev.irq + prev.idle);
|
|
380
|
+
}
|
|
381
|
+
lastOsCpus = cur;
|
|
382
|
+
const sysCpu = totalDelta > 0 ? +((1 - idleDelta / totalDelta) * 100).toFixed(1) : 0;
|
|
383
|
+
|
|
384
|
+
this.set('_admin/stats', {
|
|
385
|
+
process: { cpuPercent, heapUsedMb: +(mem.heapUsed / 1024 / 1024).toFixed(2), rssMb: +(mem.rss / 1024 / 1024).toFixed(2), uptimeSec: Math.floor(process.uptime()) },
|
|
386
|
+
db: { nodeCount, sizeMb: dbSizeMb },
|
|
387
|
+
system: { cpuCores: cur.length, totalMemMb: +(totalmem() / 1024 / 1024).toFixed(0), cpuPercent: sysCpu },
|
|
388
|
+
subs: this.subs.subscriberCount(),
|
|
389
|
+
ts: Date.now(),
|
|
390
|
+
});
|
|
391
|
+
}, 1000);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Start standalone WebSocket + REST server on its own port */
|
|
287
395
|
serve(options?: Partial<TransportOptions>) {
|
|
288
396
|
const port = options?.port ?? this.options.port;
|
|
289
397
|
const auth = options?.auth ?? this.options.auth;
|
|
290
398
|
this._transport = new Transport(this, this.rules, { ...this.options.transport, ...options, port, auth });
|
|
399
|
+
this._startStatsPublisher();
|
|
291
400
|
return this._transport.start();
|
|
292
401
|
}
|
|
293
402
|
|
|
403
|
+
/**
|
|
404
|
+
* Get transport handlers for mounting on an external Bun.serve().
|
|
405
|
+
* Does NOT start a server — caller is responsible for wiring fetch/websocket.
|
|
406
|
+
*
|
|
407
|
+
* Usage:
|
|
408
|
+
* ```ts
|
|
409
|
+
* const { handleFetch, websocketConfig, newWsData } = db.getHandlers({ auth });
|
|
410
|
+
* Bun.serve({
|
|
411
|
+
* fetch(req, server) {
|
|
412
|
+
* if (url.pathname.startsWith('/db') || url.pathname.startsWith('/sse')) {
|
|
413
|
+
* return handleFetch(req, server);
|
|
414
|
+
* }
|
|
415
|
+
* return myAppHandler(req);
|
|
416
|
+
* },
|
|
417
|
+
* websocket: websocketConfig,
|
|
418
|
+
* });
|
|
419
|
+
* ```
|
|
420
|
+
*/
|
|
421
|
+
getHandlers(options?: Partial<TransportOptions>) {
|
|
422
|
+
const auth = options?.auth ?? this.options.auth;
|
|
423
|
+
this._transport = new Transport(this, this.rules, { ...this.options.transport, ...options, auth });
|
|
424
|
+
this._startStatsPublisher();
|
|
425
|
+
return {
|
|
426
|
+
handleFetch: this._transport.handleFetch.bind(this._transport),
|
|
427
|
+
websocketConfig: this._transport.websocketConfig,
|
|
428
|
+
newWsData: this._transport.newWsData.bind(this._transport),
|
|
429
|
+
transport: this._transport,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
294
433
|
stop() {
|
|
295
434
|
this._transport?.stop();
|
|
296
435
|
this._transport = null;
|
|
297
436
|
}
|
|
298
437
|
|
|
299
438
|
close(): void {
|
|
439
|
+
this.replication?.stop();
|
|
300
440
|
if (this.sweepTimer) {
|
|
301
441
|
clearInterval(this.sweepTimer);
|
|
302
442
|
this.sweepTimer = null;
|
|
303
443
|
}
|
|
444
|
+
if (this._statsInterval) {
|
|
445
|
+
clearInterval(this._statsInterval);
|
|
446
|
+
this._statsInterval = null;
|
|
447
|
+
}
|
|
304
448
|
this.stop();
|
|
305
449
|
this.subs.clear();
|
|
306
450
|
this.storage.close();
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
import type { BodDB } from './BodDB.ts';
|
|
2
|
+
import { BodClient } from '../client/BodClient.ts';
|
|
3
|
+
import type { ClientMessage } from '../shared/protocol.ts';
|
|
4
|
+
import type { CompactOptions } from './StreamEngine.ts';
|
|
5
|
+
|
|
6
|
+
export interface ReplEvent {
|
|
7
|
+
op: 'set' | 'delete' | 'push';
|
|
8
|
+
path: string;
|
|
9
|
+
value?: unknown;
|
|
10
|
+
pushKey?: string;
|
|
11
|
+
ttl?: number;
|
|
12
|
+
ts: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface WriteEvent {
|
|
16
|
+
op: 'set' | 'delete' | 'push';
|
|
17
|
+
path: string;
|
|
18
|
+
value?: unknown;
|
|
19
|
+
pushKey?: string;
|
|
20
|
+
ttl?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class ReplicationSource {
|
|
24
|
+
url: string = '';
|
|
25
|
+
auth?: () => string | Promise<string>;
|
|
26
|
+
id?: string;
|
|
27
|
+
paths: string[] = [];
|
|
28
|
+
localPrefix?: string;
|
|
29
|
+
excludePrefixes?: string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export class ReplicationOptions {
|
|
33
|
+
role: 'primary' | 'replica' = 'primary';
|
|
34
|
+
primaryUrl?: string;
|
|
35
|
+
primaryAuth?: () => string | Promise<string>;
|
|
36
|
+
replicaId?: string;
|
|
37
|
+
excludePrefixes: string[] = ['_repl', '_streams', '_mq', '_admin'];
|
|
38
|
+
/** Bootstrap replica from primary's full state before applying _repl stream */
|
|
39
|
+
fullBootstrap: boolean = true;
|
|
40
|
+
compact?: CompactOptions;
|
|
41
|
+
sources?: ReplicationSource[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
type ProxyableMessage = Extract<ClientMessage, { op: 'set' | 'delete' | 'update' | 'push' | 'batch' }>;
|
|
45
|
+
|
|
46
|
+
export class ReplicationEngine {
|
|
47
|
+
readonly options: ReplicationOptions;
|
|
48
|
+
private client: BodClient | null = null;
|
|
49
|
+
private unsubWrite: (() => void) | null = null;
|
|
50
|
+
private unsubStream: (() => void) | null = null;
|
|
51
|
+
private sourceConns: Array<{ client: BodClient; unsub: () => void; lastEventTs: number; eventsApplied: number; pending: number }> = [];
|
|
52
|
+
private _started = false;
|
|
53
|
+
private _seq = 0;
|
|
54
|
+
private _emitting = false;
|
|
55
|
+
|
|
56
|
+
get isReplica(): boolean { return this.options.role === 'replica'; }
|
|
57
|
+
get isPrimary(): boolean { return this.options.role === 'primary'; }
|
|
58
|
+
get started(): boolean { return this._started; }
|
|
59
|
+
get seq(): number { return this._seq; }
|
|
60
|
+
|
|
61
|
+
/** Live stats snapshot for monitoring */
|
|
62
|
+
stats() {
|
|
63
|
+
return {
|
|
64
|
+
role: this.options.role,
|
|
65
|
+
started: this._started,
|
|
66
|
+
seq: this._seq,
|
|
67
|
+
sources: (this.options.sources ?? []).map((s, i) => {
|
|
68
|
+
const conn = this.sourceConns[i];
|
|
69
|
+
return {
|
|
70
|
+
url: s.url,
|
|
71
|
+
paths: s.paths,
|
|
72
|
+
localPrefix: s.localPrefix,
|
|
73
|
+
connected: !!conn?.client?.connected,
|
|
74
|
+
pending: conn?.pending ?? 0,
|
|
75
|
+
eventsApplied: conn?.eventsApplied ?? 0,
|
|
76
|
+
};
|
|
77
|
+
}),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
constructor(
|
|
82
|
+
private db: BodDB,
|
|
83
|
+
options?: Partial<ReplicationOptions>,
|
|
84
|
+
) {
|
|
85
|
+
this.options = { ...new ReplicationOptions(), ...options };
|
|
86
|
+
if (!this.options.replicaId) {
|
|
87
|
+
this.options.replicaId = `replica_${Math.random().toString(36).slice(2, 10)}`;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Start replication — primary listens for writes, replica connects and consumes */
|
|
92
|
+
async start(): Promise<void> {
|
|
93
|
+
if (this._started) return;
|
|
94
|
+
this._started = true;
|
|
95
|
+
|
|
96
|
+
if (this.isPrimary) {
|
|
97
|
+
this.startPrimary();
|
|
98
|
+
} else {
|
|
99
|
+
await this.startReplica();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (this.options.sources?.length) {
|
|
103
|
+
await this.startSources();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Stop replication */
|
|
108
|
+
stop(): void {
|
|
109
|
+
this._started = false;
|
|
110
|
+
this.unsubWrite?.();
|
|
111
|
+
this.unsubWrite = null;
|
|
112
|
+
this.unsubStream?.();
|
|
113
|
+
this.unsubStream = null;
|
|
114
|
+
this.client?.disconnect();
|
|
115
|
+
this.client = null;
|
|
116
|
+
for (const sc of this.sourceConns) {
|
|
117
|
+
sc.unsub();
|
|
118
|
+
sc.client.disconnect();
|
|
119
|
+
}
|
|
120
|
+
this.sourceConns = [];
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Proxy a write operation to the primary (replica mode) */
|
|
124
|
+
async proxyWrite(msg: ProxyableMessage): Promise<unknown> {
|
|
125
|
+
if (!this.client) throw new Error('Replica not connected to primary');
|
|
126
|
+
switch (msg.op) {
|
|
127
|
+
case 'set':
|
|
128
|
+
await this.client.set(msg.path, msg.value, msg.ttl ? { ttl: msg.ttl } : undefined);
|
|
129
|
+
return null;
|
|
130
|
+
case 'delete':
|
|
131
|
+
await this.client.delete(msg.path);
|
|
132
|
+
return null;
|
|
133
|
+
case 'update':
|
|
134
|
+
await this.client.update(msg.updates);
|
|
135
|
+
return null;
|
|
136
|
+
case 'push':
|
|
137
|
+
return await this.client.push(msg.path, msg.value, msg.idempotencyKey ? { idempotencyKey: msg.idempotencyKey } : undefined);
|
|
138
|
+
case 'batch':
|
|
139
|
+
return await this.client.batch(msg.operations);
|
|
140
|
+
default:
|
|
141
|
+
throw new Error(`Cannot proxy op: ${(msg as any).op}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// --- Primary mode ---
|
|
146
|
+
|
|
147
|
+
private startPrimary(): void {
|
|
148
|
+
this.unsubWrite = this.db.onWrite((ev: WriteEvent) => {
|
|
149
|
+
this.emit(ev);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private isExcluded(path: string): boolean {
|
|
154
|
+
return this.options.excludePrefixes.some(p => path.startsWith(p));
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
private emit(ev: WriteEvent): void {
|
|
158
|
+
// Guard against recursion (db.push('_repl') triggers _fireWrite → emit)
|
|
159
|
+
if (this._emitting) return;
|
|
160
|
+
if (this.isExcluded(ev.path)) return;
|
|
161
|
+
|
|
162
|
+
this._emitting = true;
|
|
163
|
+
try {
|
|
164
|
+
const replEvent: ReplEvent = { ...ev, ts: Date.now() };
|
|
165
|
+
const seq = this._seq++;
|
|
166
|
+
const idempotencyKey = `${replEvent.ts}:${seq}:${ev.path}`;
|
|
167
|
+
this.db.push('_repl', replEvent, { idempotencyKey });
|
|
168
|
+
} finally {
|
|
169
|
+
this._emitting = false;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// --- Replica mode ---
|
|
174
|
+
|
|
175
|
+
private async startReplica(): Promise<void> {
|
|
176
|
+
if (!this.options.primaryUrl) throw new Error('primaryUrl required for replica mode');
|
|
177
|
+
|
|
178
|
+
this.client = new BodClient({
|
|
179
|
+
url: this.options.primaryUrl,
|
|
180
|
+
auth: this.options.primaryAuth,
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
await this.client.connect();
|
|
184
|
+
console.log(` [REPL] Connected to primary ${this.options.primaryUrl}`);
|
|
185
|
+
|
|
186
|
+
// Full state bootstrap: fetch all data from primary (catches pre-replication data)
|
|
187
|
+
if (this.options.fullBootstrap) {
|
|
188
|
+
await this.bootstrapFullState();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Stream bootstrap: apply _repl events on top (catches recent writes, deduped by idempotent set)
|
|
192
|
+
const snapshot = await this.client!.streamMaterialize('_repl', { keepKey: 'path' });
|
|
193
|
+
console.log(` [REPL] Stream bootstrap: ${snapshot ? Object.keys(snapshot).length + ' events' : 'no events'}`);
|
|
194
|
+
if (snapshot) {
|
|
195
|
+
this.db.setReplaying(true);
|
|
196
|
+
try {
|
|
197
|
+
for (const [, event] of Object.entries(snapshot)) {
|
|
198
|
+
const ev = event as ReplEvent;
|
|
199
|
+
this.applyEvent(ev);
|
|
200
|
+
}
|
|
201
|
+
} finally {
|
|
202
|
+
this.db.setReplaying(false);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Subscribe to ongoing events
|
|
207
|
+
const groupId = this.options.replicaId!;
|
|
208
|
+
console.log(` [REPL] Subscribing to stream as '${groupId}'`);
|
|
209
|
+
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
210
|
+
console.log(` [REPL] Received ${events.length} events`);
|
|
211
|
+
this.db.setReplaying(true);
|
|
212
|
+
try {
|
|
213
|
+
for (const e of events) {
|
|
214
|
+
const ev = e.val() as ReplEvent;
|
|
215
|
+
this.applyEvent(ev);
|
|
216
|
+
// Ack after applying
|
|
217
|
+
this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
|
|
218
|
+
}
|
|
219
|
+
} finally {
|
|
220
|
+
this.db.setReplaying(false);
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Fetch full DB state from primary and apply locally */
|
|
226
|
+
private async bootstrapFullState(): Promise<void> {
|
|
227
|
+
const topLevel = await this.client!.getShallow();
|
|
228
|
+
const keys = topLevel
|
|
229
|
+
.map(e => e.key)
|
|
230
|
+
.filter(k => !this.isExcluded(k));
|
|
231
|
+
console.log(` [REPL] Full bootstrap: ${keys.length} top-level keys [${keys.join(', ')}]`);
|
|
232
|
+
if (keys.length === 0) return;
|
|
233
|
+
|
|
234
|
+
this.db.setReplaying(true);
|
|
235
|
+
try {
|
|
236
|
+
for (const key of keys) {
|
|
237
|
+
const value = await this.client!.get(key);
|
|
238
|
+
if (value != null) {
|
|
239
|
+
this.db.set(key, value);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
} finally {
|
|
243
|
+
this.db.setReplaying(false);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// --- Source feed subscriptions ---
|
|
248
|
+
|
|
249
|
+
private async startSources(): Promise<void> {
|
|
250
|
+
const results = await Promise.allSettled(
|
|
251
|
+
this.options.sources!.map(source => this.startSource(source)),
|
|
252
|
+
);
|
|
253
|
+
for (let i = 0; i < results.length; i++) {
|
|
254
|
+
if (results[i].status === 'rejected') {
|
|
255
|
+
const src = this.options.sources![i];
|
|
256
|
+
console.error(`[REPL] source ${src.url} paths=[${src.paths}] failed:`, (results[i] as PromiseRejectedResult).reason);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
private async startSource(source: ReplicationSource): Promise<void> {
|
|
262
|
+
const client = new BodClient({ url: source.url, auth: source.auth });
|
|
263
|
+
await client.connect();
|
|
264
|
+
|
|
265
|
+
// Bootstrap: materialize _repl, filter by source paths
|
|
266
|
+
const snapshot = await client.streamMaterialize('_repl', { keepKey: 'path' });
|
|
267
|
+
if (snapshot) {
|
|
268
|
+
this.db.setReplaying(true);
|
|
269
|
+
try {
|
|
270
|
+
for (const [, event] of Object.entries(snapshot)) {
|
|
271
|
+
const ev = event as ReplEvent;
|
|
272
|
+
if (this.matchesSourcePaths(ev.path, source)) {
|
|
273
|
+
this.applyEvent(ev, source);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
} finally {
|
|
277
|
+
this.db.setReplaying(false);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Subscribe to ongoing events
|
|
282
|
+
const groupId = source.id || `source_${source.url}_${source.paths.sort().join('+')}`;
|
|
283
|
+
const connState = { client, unsub: () => {}, lastEventTs: Date.now(), eventsApplied: 0, pending: 0 };
|
|
284
|
+
const unsub = client.stream('_repl', groupId).on((events) => {
|
|
285
|
+
connState.pending += events.length;
|
|
286
|
+
this.db.setReplaying(true);
|
|
287
|
+
try {
|
|
288
|
+
for (const e of events) {
|
|
289
|
+
const ev = e.val() as ReplEvent;
|
|
290
|
+
connState.lastEventTs = ev.ts || Date.now();
|
|
291
|
+
if (this.matchesSourcePaths(ev.path, source)) {
|
|
292
|
+
this.applyEvent(ev, source);
|
|
293
|
+
connState.eventsApplied++;
|
|
294
|
+
}
|
|
295
|
+
connState.pending--;
|
|
296
|
+
client.stream('_repl', groupId).ack(e.key).catch(() => {});
|
|
297
|
+
}
|
|
298
|
+
} finally {
|
|
299
|
+
this.db.setReplaying(false);
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
connState.unsub = unsub;
|
|
303
|
+
|
|
304
|
+
this.sourceConns.push(connState);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
private matchesSourcePaths(path: string, source: ReplicationSource): boolean {
|
|
308
|
+
if (source.excludePrefixes?.some(p => path.startsWith(p))) return false;
|
|
309
|
+
return source.paths.some(p => path === p || path.startsWith(p + '/'));
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private remapPath(path: string, source: ReplicationSource): string {
|
|
313
|
+
return source.localPrefix ? `${source.localPrefix}/${path}` : path;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
private applyEvent(ev: ReplEvent, source?: ReplicationSource): void {
|
|
317
|
+
const path = source ? this.remapPath(ev.path, source) : ev.path;
|
|
318
|
+
switch (ev.op) {
|
|
319
|
+
case 'set':
|
|
320
|
+
this.db.set(path, ev.value, ev.ttl ? { ttl: ev.ttl } : undefined);
|
|
321
|
+
break;
|
|
322
|
+
case 'delete':
|
|
323
|
+
this.db.delete(path);
|
|
324
|
+
break;
|
|
325
|
+
case 'push':
|
|
326
|
+
if (ev.pushKey) {
|
|
327
|
+
this.db.set(`${path}/${ev.pushKey}`, ev.value);
|
|
328
|
+
}
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
@@ -24,6 +24,8 @@ export class StorageEngine {
|
|
|
24
24
|
private stmtPushIdempotent!: ReturnType<Database['prepare']>;
|
|
25
25
|
private stmtGetByIdempotencyKey!: ReturnType<Database['prepare']>;
|
|
26
26
|
private stmtQueryAfterKey!: ReturnType<Database['prepare']>;
|
|
27
|
+
private stmtGetWithMeta!: ReturnType<Database['prepare']>;
|
|
28
|
+
private stmtPrefixMaxUpdated!: ReturnType<Database['prepare']>;
|
|
27
29
|
|
|
28
30
|
constructor(options?: Partial<StorageEngineOptions>) {
|
|
29
31
|
this.options = { ...new StorageEngineOptions(), ...options };
|
|
@@ -71,6 +73,8 @@ export class StorageEngine {
|
|
|
71
73
|
this.stmtPushIdempotent = this.db.prepare('INSERT OR IGNORE INTO nodes (path, value, updated_at, idempotency_key) VALUES (?, ?, ?, ?)');
|
|
72
74
|
this.stmtGetByIdempotencyKey = this.db.prepare('SELECT path FROM nodes WHERE idempotency_key = ?');
|
|
73
75
|
this.stmtQueryAfterKey = this.db.prepare('SELECT path, value FROM nodes WHERE path > ? AND path < ? AND mq_status IS NULL ORDER BY path ASC LIMIT ?');
|
|
76
|
+
this.stmtGetWithMeta = this.db.prepare('SELECT path, value, updated_at FROM nodes WHERE path = ? AND mq_status IS NULL');
|
|
77
|
+
this.stmtPrefixMaxUpdated = this.db.prepare('SELECT MAX(updated_at) as max_ua FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL');
|
|
74
78
|
}
|
|
75
79
|
|
|
76
80
|
/** Check if a path has any data (exact or subtree) */
|
|
@@ -82,6 +86,21 @@ export class StorageEngine {
|
|
|
82
86
|
return !!this.stmtExistsPrefix.get(prefix, prefixEnd(prefix));
|
|
83
87
|
}
|
|
84
88
|
|
|
89
|
+
/** Get value at path with metadata (updated_at) */
|
|
90
|
+
getWithMeta(path: string): { data: unknown; updatedAt: number } | null {
|
|
91
|
+
path = validatePath(path);
|
|
92
|
+
const exact = this.stmtGetWithMeta.get(path) as { path: string; value: string; updated_at: number } | null;
|
|
93
|
+
if (exact) {
|
|
94
|
+
return { data: JSON.parse(exact.value), updatedAt: exact.updated_at };
|
|
95
|
+
}
|
|
96
|
+
// Try prefix (subtree) — use max updated_at across all leaves
|
|
97
|
+
const prefix = path + '/';
|
|
98
|
+
const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string; value: string }>;
|
|
99
|
+
if (rows.length === 0) return null;
|
|
100
|
+
const maxUpdated = this.stmtPrefixMaxUpdated.get(prefix, prefixEnd(prefix)) as { max_ua: number } | null;
|
|
101
|
+
return { data: reconstruct(path, rows), updatedAt: maxUpdated?.max_ua ?? 0 };
|
|
102
|
+
}
|
|
103
|
+
|
|
85
104
|
/** Get value at path (reconstructs subtree if needed) */
|
|
86
105
|
get(path: string, options?: { resolve?: boolean | string[] }): unknown {
|
|
87
106
|
path = validatePath(path);
|