@bod.ee/db 0.12.6 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +2 -1
- package/admin/admin.ts +24 -4
- package/admin/bun.lock +248 -0
- package/admin/index.html +12 -0
- package/admin/package.json +22 -0
- package/admin/src/App.tsx +23 -0
- package/admin/src/client/ZuzClient.ts +183 -0
- package/admin/src/client/types.ts +28 -0
- package/admin/src/components/MetricsBar.tsx +167 -0
- package/admin/src/components/Sparkline.tsx +72 -0
- package/admin/src/components/TreePane.tsx +287 -0
- package/admin/src/components/tabs/Advanced.tsx +222 -0
- package/admin/src/components/tabs/AuthRules.tsx +104 -0
- package/admin/src/components/tabs/Cache.tsx +113 -0
- package/admin/src/components/tabs/KeyAuth.tsx +462 -0
- package/admin/src/components/tabs/MessageQueue.tsx +237 -0
- package/admin/src/components/tabs/Query.tsx +75 -0
- package/admin/src/components/tabs/ReadWrite.tsx +177 -0
- package/admin/src/components/tabs/Replication.tsx +94 -0
- package/admin/src/components/tabs/Streams.tsx +329 -0
- package/admin/src/components/tabs/StressTests.tsx +209 -0
- package/admin/src/components/tabs/Subscriptions.tsx +69 -0
- package/admin/src/components/tabs/TabPane.tsx +151 -0
- package/admin/src/components/tabs/VFS.tsx +435 -0
- package/admin/src/components/tabs/View.tsx +14 -0
- package/admin/src/components/tabs/utils.ts +25 -0
- package/admin/src/context/DbContext.tsx +33 -0
- package/admin/src/context/StatsContext.tsx +56 -0
- package/admin/src/main.tsx +10 -0
- package/admin/src/styles.css +96 -0
- package/admin/tsconfig.app.json +21 -0
- package/admin/tsconfig.json +7 -0
- package/admin/tsconfig.node.json +15 -0
- package/admin/vite.config.ts +42 -0
- package/deploy/base.yaml +1 -1
- package/deploy/prod-il.config.ts +5 -2
- package/deploy/prod.config.ts +5 -2
- package/package.json +5 -1
- package/src/server/BodDB.ts +62 -5
- package/src/server/ReplicationEngine.ts +149 -34
- package/src/server/StorageEngine.ts +12 -4
- package/src/server/StreamEngine.ts +2 -2
- package/src/server/Transport.ts +60 -0
- package/tests/replication.test.ts +162 -1
- package/admin/ui.html +0 -3547
|
@@ -116,6 +116,8 @@ export class ReplicationSource {
|
|
|
116
116
|
paths: string[] = [];
|
|
117
117
|
localPrefix?: string;
|
|
118
118
|
excludePrefixes?: string[];
|
|
119
|
+
/** Subscribe to `_repl_scoped/<prefix>` instead of filtering `_repl`. Requires primary to have matching `scopedPrefixes`. */
|
|
120
|
+
scopedStream?: boolean;
|
|
119
121
|
}
|
|
120
122
|
|
|
121
123
|
export class ReplicationOptions {
|
|
@@ -124,7 +126,7 @@ export class ReplicationOptions {
|
|
|
124
126
|
primaryAuth?: () => string | Promise<string>;
|
|
125
127
|
replicaId?: string;
|
|
126
128
|
/** Paths excluded from replication. Note: when `paths` explicitly configures a prefix (e.g. `_auth`), it overrides this exclusion. */
|
|
127
|
-
excludePrefixes: string[] = ['_repl', '_streams', '_mq', '_admin', '_auth'];
|
|
129
|
+
excludePrefixes: string[] = ['_repl', '_repl_scoped', '_streams', '_mq', '_admin', '_auth'];
|
|
128
130
|
/** Bootstrap replica from primary's full state before applying _repl stream */
|
|
129
131
|
fullBootstrap: boolean = true;
|
|
130
132
|
compact?: CompactOptions;
|
|
@@ -133,6 +135,8 @@ export class ReplicationOptions {
|
|
|
133
135
|
sources?: ReplicationSource[];
|
|
134
136
|
/** Per-path topology: strings default to 'sync', objects specify mode. When absent, role governs all paths. */
|
|
135
137
|
paths?: Array<string | PathTopology>;
|
|
138
|
+
/** Path prefixes that get their own `_repl_scoped/<prefix>` stream. Auto-derived from paths/sources if not set. */
|
|
139
|
+
scopedPrefixes?: string[];
|
|
136
140
|
}
|
|
137
141
|
|
|
138
142
|
const BOOTSTRAP_BATCH_SIZE = 200;
|
|
@@ -142,6 +146,8 @@ type ProxyableMessage = Extract<ClientMessage, { op: 'set' | 'delete' | 'update'
|
|
|
142
146
|
export class ReplicationEngine {
|
|
143
147
|
readonly options: ReplicationOptions;
|
|
144
148
|
readonly router: PathTopologyRouter | null = null;
|
|
149
|
+
/** Resolved scoped prefixes (sorted longest-first for matching) */
|
|
150
|
+
private _scopedPrefixes: string[] = [];
|
|
145
151
|
private client: BodClient | null = null;
|
|
146
152
|
private unsubWrite: (() => void) | null = null;
|
|
147
153
|
private unsubStream: (() => void) | null = null;
|
|
@@ -193,6 +199,8 @@ export class ReplicationEngine {
|
|
|
193
199
|
if (this.options.paths?.length) {
|
|
194
200
|
this.router = new PathTopologyRouter(this.options.paths, this.options.role);
|
|
195
201
|
}
|
|
202
|
+
// Resolve scoped prefixes
|
|
203
|
+
this._scopedPrefixes = this._resolveScopedPrefixes();
|
|
196
204
|
if (!this.options.replicaId) {
|
|
197
205
|
// Derive stable ID from config when possible, fall back to random
|
|
198
206
|
if (this.options.primaryUrl) {
|
|
@@ -293,6 +301,29 @@ export class ReplicationEngine {
|
|
|
293
301
|
}
|
|
294
302
|
}
|
|
295
303
|
|
|
304
|
+
/** Derive scoped prefixes for primary emit. Only auto-derives from topology paths when role is 'primary'.
|
|
305
|
+
* Replicas must use explicit `scopedPrefixes` config since they don't control what the primary emits. */
|
|
306
|
+
private _resolveScopedPrefixes(): string[] {
|
|
307
|
+
if (this.options.scopedPrefixes) return [...this.options.scopedPrefixes].sort((a, b) => b.length - a.length);
|
|
308
|
+
// Only auto-derive for primary role — replica/sync nodes don't control primary's emit topics
|
|
309
|
+
if (this.options.role !== 'primary') return [];
|
|
310
|
+
const set = new Set<string>();
|
|
311
|
+
if (this.router) {
|
|
312
|
+
for (const e of this.router.getEntries()) {
|
|
313
|
+
if (e.path) set.add(e.path);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return [...set].sort((a, b) => b.length - a.length);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** Longest-prefix match against scoped prefixes */
|
|
320
|
+
private _matchScopedPrefix(path: string): string | null {
|
|
321
|
+
for (const p of this._scopedPrefixes) {
|
|
322
|
+
if (path === p || path.startsWith(p + '/')) return p;
|
|
323
|
+
}
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
|
|
296
327
|
// --- Primary mode ---
|
|
297
328
|
|
|
298
329
|
private startPrimary(): void {
|
|
@@ -307,6 +338,9 @@ export class ReplicationEngine {
|
|
|
307
338
|
const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
|
|
308
339
|
if (compact.maxCount || compact.maxAge) {
|
|
309
340
|
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
341
|
+
for (const prefix of this._scopedPrefixes) {
|
|
342
|
+
try { this.db.stream.compact(`_repl_scoped/${prefix}`, compact); } catch {}
|
|
343
|
+
}
|
|
310
344
|
}
|
|
311
345
|
}
|
|
312
346
|
|
|
@@ -324,6 +358,8 @@ export class ReplicationEngine {
|
|
|
324
358
|
|
|
325
359
|
/** Buffer replication events during transactions, emit immediately otherwise */
|
|
326
360
|
private emit(ev: WriteEvent): void {
|
|
361
|
+
// _repl/_repl_scoped writes must never emit (infinite loop)
|
|
362
|
+
if (ev.path.startsWith('_repl')) return;
|
|
327
363
|
if (this._emitting) {
|
|
328
364
|
this.log.debug('emit: skipped (re-entrant)', { path: ev.path });
|
|
329
365
|
return;
|
|
@@ -361,6 +397,19 @@ export class ReplicationEngine {
|
|
|
361
397
|
const idempotencyKey = `${replEvent.ts}:${seq}:${ev.path}`;
|
|
362
398
|
this.db.push('_repl', replEvent, { idempotencyKey });
|
|
363
399
|
this.log.info('_repl emit', { seq, op: ev.op, path: ev.path });
|
|
400
|
+
|
|
401
|
+
// Dual emit to scoped stream
|
|
402
|
+
const scopedPrefix = this._matchScopedPrefix(ev.path);
|
|
403
|
+
if (scopedPrefix) {
|
|
404
|
+
const scopedTopic = `_repl_scoped/${scopedPrefix}`;
|
|
405
|
+
const scopedKey = `${replEvent.ts}:${seq}:scoped:${ev.path}`;
|
|
406
|
+
try {
|
|
407
|
+
this.db.push(scopedTopic, replEvent, { idempotencyKey: scopedKey });
|
|
408
|
+
this.log.debug('scoped emit', { topic: scopedTopic, path: ev.path });
|
|
409
|
+
} catch (e: any) {
|
|
410
|
+
this.log.error('scoped emit failed', { topic: scopedTopic, error: e.message });
|
|
411
|
+
}
|
|
412
|
+
}
|
|
364
413
|
} catch (e: any) {
|
|
365
414
|
this.log.error('_repl emit failed', { path: ev.path, error: e.message });
|
|
366
415
|
} finally {
|
|
@@ -373,6 +422,10 @@ export class ReplicationEngine {
|
|
|
373
422
|
this._emitCount = 0;
|
|
374
423
|
const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
|
|
375
424
|
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
425
|
+
// Compact scoped streams too
|
|
426
|
+
for (const prefix of this._scopedPrefixes) {
|
|
427
|
+
try { this.db.stream.compact(`_repl_scoped/${prefix}`, compact); } catch {}
|
|
428
|
+
}
|
|
376
429
|
}
|
|
377
430
|
}
|
|
378
431
|
|
|
@@ -414,30 +467,67 @@ export class ReplicationEngine {
|
|
|
414
467
|
await this.bootstrapFullState(bootstrapPaths);
|
|
415
468
|
}
|
|
416
469
|
|
|
417
|
-
//
|
|
418
|
-
const applied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, pathPrefixes) });
|
|
419
|
-
this.log.info(`Stream bootstrap (paths): ${applied} events applied`);
|
|
420
|
-
|
|
421
|
-
// Subscribe to ongoing events, filter by paths
|
|
470
|
+
// Use scoped streams for paths that have them, fall back to filtered _repl
|
|
422
471
|
const groupId = this.options.replicaId!;
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
472
|
+
const scopedSubs: Array<() => void> = [];
|
|
473
|
+
const unscopedPaths: string[] = [];
|
|
474
|
+
|
|
475
|
+
// Also bootstrap scoped paths from _repl (catches historical events from before scoped streams existed)
|
|
476
|
+
const scopedPathPrefixes = pathPrefixes.filter(p => this._scopedPrefixes.includes(p));
|
|
477
|
+
if (scopedPathPrefixes.length > 0) {
|
|
478
|
+
const fallbackApplied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, scopedPathPrefixes), groupId });
|
|
479
|
+
if (fallbackApplied > 0) this.log.info(`Scoped _repl fallback bootstrap: ${fallbackApplied} events`);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
for (const prefix of pathPrefixes) {
|
|
483
|
+
if (this._scopedPrefixes.includes(prefix)) {
|
|
484
|
+
const scopedTopic = `_repl_scoped/${prefix}`;
|
|
485
|
+
// Bootstrap from scoped stream (may apply duplicates from _repl fallback — applyEvent is idempotent via set/delete)
|
|
486
|
+
const applied = await this.bootstrapFromStream(this.client!, { streamTopic: scopedTopic, groupId });
|
|
487
|
+
this.log.info(`Scoped stream bootstrap: ${scopedTopic} → ${applied} events`);
|
|
488
|
+
|
|
489
|
+
// Subscribe to scoped stream (no filter needed)
|
|
490
|
+
const unsub = this.client!.stream(scopedTopic, groupId).on((events) => {
|
|
491
|
+
this.db.setReplaying(true);
|
|
492
|
+
try {
|
|
493
|
+
for (const e of events) {
|
|
494
|
+
this.applyEvent(e.val() as ReplEvent);
|
|
495
|
+
this.client!.stream(scopedTopic, groupId).ack(e.key).catch(() => {});
|
|
496
|
+
}
|
|
497
|
+
} finally {
|
|
498
|
+
this.db.setReplaying(false);
|
|
434
499
|
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
}
|
|
438
|
-
|
|
500
|
+
});
|
|
501
|
+
scopedSubs.push(unsub);
|
|
502
|
+
} else {
|
|
503
|
+
unscopedPaths.push(prefix);
|
|
439
504
|
}
|
|
440
|
-
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// Fall back to _repl with filter for paths without scoped streams
|
|
508
|
+
if (unscopedPaths.length > 0) {
|
|
509
|
+
const applied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, unscopedPaths), groupId });
|
|
510
|
+
this.log.info(`Stream bootstrap (filtered _repl): ${applied} events for [${unscopedPaths.join(', ')}]`);
|
|
511
|
+
|
|
512
|
+
const unsub = this.client!.stream('_repl', groupId).on((events) => {
|
|
513
|
+
this.db.setReplaying(true);
|
|
514
|
+
try {
|
|
515
|
+
for (const e of events) {
|
|
516
|
+
const ev = e.val() as ReplEvent;
|
|
517
|
+
if (this.matchesPathPrefixes(ev.path, unscopedPaths)) {
|
|
518
|
+
this.applyEvent(ev);
|
|
519
|
+
}
|
|
520
|
+
this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
|
|
521
|
+
}
|
|
522
|
+
} finally {
|
|
523
|
+
this.db.setReplaying(false);
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
scopedSubs.push(unsub);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// Combine all unsub functions
|
|
530
|
+
this.unsubStream = () => { for (const u of scopedSubs) u(); };
|
|
441
531
|
}
|
|
442
532
|
|
|
443
533
|
/** Check if path matches any of the given prefixes */
|
|
@@ -462,11 +552,11 @@ export class ReplicationEngine {
|
|
|
462
552
|
}
|
|
463
553
|
|
|
464
554
|
// Stream bootstrap: cursor-based to avoid huge single response
|
|
465
|
-
const
|
|
555
|
+
const groupId = this.options.replicaId!;
|
|
556
|
+
const applied = await this.bootstrapFromStream(this.client!, { groupId });
|
|
466
557
|
this.log.info(`Stream bootstrap: ${applied} events applied`);
|
|
467
558
|
|
|
468
559
|
// Subscribe to ongoing events
|
|
469
|
-
const groupId = this.options.replicaId!;
|
|
470
560
|
this.log.info(`Subscribing to stream as '${groupId}'`);
|
|
471
561
|
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
472
562
|
this.log.debug(`Received ${events.length} events`);
|
|
@@ -527,28 +617,39 @@ export class ReplicationEngine {
|
|
|
527
617
|
const client = new BodClient({ url: source.url, auth: source.auth });
|
|
528
618
|
await client.connect();
|
|
529
619
|
|
|
530
|
-
|
|
620
|
+
const groupId = source.id || `source_${source.url}_${source.paths.sort().join('+')}`;
|
|
621
|
+
const connState = { client, unsub: () => {}, lastEventTs: Date.now(), eventsApplied: 0, pending: 0 };
|
|
622
|
+
|
|
623
|
+
// Use scoped stream if explicitly enabled and source has a single path
|
|
624
|
+
if (source.scopedStream && source.paths.length > 1) {
|
|
625
|
+
this.log.warn('scopedStream only supported for single-path sources, falling back to filtered _repl', { paths: source.paths });
|
|
626
|
+
}
|
|
627
|
+
const useScoped = source.scopedStream && source.paths.length === 1;
|
|
628
|
+
const topic = useScoped ? `_repl_scoped/${source.paths[0]}` : '_repl';
|
|
629
|
+
const needsFilter = !useScoped;
|
|
630
|
+
|
|
631
|
+
// Bootstrap
|
|
531
632
|
await this.bootstrapFromStream(client, {
|
|
532
|
-
filter: ev => this.matchesSourcePaths(ev.path, source),
|
|
633
|
+
filter: needsFilter ? (ev => this.matchesSourcePaths(ev.path, source)) : undefined,
|
|
533
634
|
source,
|
|
635
|
+
streamTopic: topic,
|
|
636
|
+
groupId,
|
|
534
637
|
});
|
|
535
638
|
|
|
536
639
|
// Subscribe to ongoing events
|
|
537
|
-
const
|
|
538
|
-
const connState = { client, unsub: () => {}, lastEventTs: Date.now(), eventsApplied: 0, pending: 0 };
|
|
539
|
-
const unsub = client.stream('_repl', groupId).on((events) => {
|
|
640
|
+
const unsub = client.stream(topic, groupId).on((events) => {
|
|
540
641
|
connState.pending += events.length;
|
|
541
642
|
this.db.setReplaying(true);
|
|
542
643
|
try {
|
|
543
644
|
for (const e of events) {
|
|
544
645
|
const ev = e.val() as ReplEvent;
|
|
545
646
|
connState.lastEventTs = ev.ts || Date.now();
|
|
546
|
-
if (this.matchesSourcePaths(ev.path, source)) {
|
|
647
|
+
if (!needsFilter || this.matchesSourcePaths(ev.path, source)) {
|
|
547
648
|
this.applyEvent(ev, source);
|
|
548
649
|
connState.eventsApplied++;
|
|
549
650
|
}
|
|
550
651
|
connState.pending--;
|
|
551
|
-
client.stream(
|
|
652
|
+
client.stream(topic, groupId).ack(e.key).catch(() => {});
|
|
552
653
|
}
|
|
553
654
|
} finally {
|
|
554
655
|
this.db.setReplaying(false);
|
|
@@ -568,8 +669,22 @@ export class ReplicationEngine {
|
|
|
568
669
|
return source.localPrefix ? `${source.localPrefix}/${path}` : path;
|
|
569
670
|
}
|
|
570
671
|
|
|
571
|
-
/** Cursor-based stream bootstrap: pages through _repl materialize to avoid huge single responses
|
|
572
|
-
|
|
672
|
+
/** Cursor-based stream bootstrap: pages through _repl (or scoped stream) materialize to avoid huge single responses.
|
|
673
|
+
* Skips bootstrap entirely when the consumer group already has a stored offset (subscribe handles incremental replay). */
|
|
674
|
+
private async bootstrapFromStream(client: BodClient, opts?: { filter?: (ev: ReplEvent) => boolean; source?: ReplicationSource; streamTopic?: string; groupId?: string }): Promise<number> {
|
|
675
|
+
const topic = opts?.streamTopic ?? '_repl';
|
|
676
|
+
const groupId = opts?.groupId;
|
|
677
|
+
|
|
678
|
+
// If consumer group has a stored offset, skip full bootstrap — subscribe() will replay incrementally from that offset
|
|
679
|
+
if (groupId) {
|
|
680
|
+
const offsetPath = `_streams/${topic}/groups/${groupId}/offset`;
|
|
681
|
+
const existingOffset = await client.get(offsetPath);
|
|
682
|
+
if (existingOffset != null) {
|
|
683
|
+
this.log.info(`Skipping bootstrap for ${topic} (group '${groupId}' has offset, subscribe will resume)`);
|
|
684
|
+
return 0;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
573
688
|
let cursor: string | undefined;
|
|
574
689
|
let applied = 0;
|
|
575
690
|
const filter = opts?.filter;
|
|
@@ -577,7 +692,7 @@ export class ReplicationEngine {
|
|
|
577
692
|
this.db.setReplaying(true);
|
|
578
693
|
try {
|
|
579
694
|
do {
|
|
580
|
-
const page = await client.streamMaterialize(
|
|
695
|
+
const page = await client.streamMaterialize(topic, { keepKey: 'path', batchSize: BOOTSTRAP_BATCH_SIZE, cursor });
|
|
581
696
|
if (page.data) {
|
|
582
697
|
for (const [, event] of Object.entries(page.data)) {
|
|
583
698
|
const ev = event as ReplEvent;
|
|
@@ -137,7 +137,7 @@ export class StorageEngine {
|
|
|
137
137
|
}
|
|
138
138
|
|
|
139
139
|
/** Get immediate children of a path (one level deep). Returns { key, isLeaf, value? }[] */
|
|
140
|
-
getShallow(path?: string): Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number }> {
|
|
140
|
+
getShallow(path?: string): Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number; count?: number }> {
|
|
141
141
|
const prefix = path ? path + '/' : '';
|
|
142
142
|
const end = prefix + '\uffff';
|
|
143
143
|
const rows = (prefix
|
|
@@ -145,7 +145,7 @@ export class StorageEngine {
|
|
|
145
145
|
: this.db.prepare('SELECT path, value, expires_at FROM nodes ORDER BY path').all()
|
|
146
146
|
) as Array<{ path: string; value: string; expires_at: number | null }>;
|
|
147
147
|
|
|
148
|
-
const children: Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number }> = [];
|
|
148
|
+
const children: Array<{ key: string; isLeaf: boolean; value?: unknown; ttl?: number; count?: number }> = [];
|
|
149
149
|
const seen = new Set<string>();
|
|
150
150
|
for (const row of rows) {
|
|
151
151
|
const rest = row.path.slice(prefix.length);
|
|
@@ -158,10 +158,18 @@ export class StorageEngine {
|
|
|
158
158
|
if (row.expires_at) entry.ttl = Math.max(0, row.expires_at - Math.floor(Date.now() / 1000));
|
|
159
159
|
children.push(entry);
|
|
160
160
|
} else {
|
|
161
|
-
// Check if any child in this branch has TTL
|
|
161
|
+
// Check if any child in this branch has TTL; count direct children
|
|
162
162
|
const branchPrefix = prefix + key + '/';
|
|
163
163
|
const hasTTL = rows.some(r => r.path.startsWith(branchPrefix) && r.expires_at);
|
|
164
|
-
|
|
164
|
+
const directChildren = new Set<string>();
|
|
165
|
+
for (const r of rows) {
|
|
166
|
+
if (!r.path.startsWith(branchPrefix)) continue;
|
|
167
|
+
const seg = r.path.slice(branchPrefix.length).split('/')[0];
|
|
168
|
+
if (seg) directChildren.add(seg);
|
|
169
|
+
}
|
|
170
|
+
const entry: { key: string; isLeaf: boolean; ttl?: number; count?: number } = { key, isLeaf: false, count: directChildren.size };
|
|
171
|
+
if (hasTTL) entry.ttl = -1;
|
|
172
|
+
children.push(entry);
|
|
165
173
|
}
|
|
166
174
|
}
|
|
167
175
|
return children;
|
|
@@ -128,9 +128,9 @@ export class StreamEngine {
|
|
|
128
128
|
}
|
|
129
129
|
});
|
|
130
130
|
|
|
131
|
-
// 2. Fetch replay batch
|
|
131
|
+
// 2. Fetch replay batch (use large limit to ensure all missed events are replayed for catch-up)
|
|
132
132
|
const offset = this.getOffset(topic, groupId);
|
|
133
|
-
const replayBatch = this.storage.queryAfterKey(topic, offset,
|
|
133
|
+
const replayBatch = this.storage.queryAfterKey(topic, offset, Number.MAX_SAFE_INTEGER);
|
|
134
134
|
|
|
135
135
|
// 3. Send replay batch
|
|
136
136
|
if (replayBatch.length > 0) {
|
package/src/server/Transport.ts
CHANGED
|
@@ -30,6 +30,8 @@ export class TransportOptions {
|
|
|
30
30
|
keyAuth?: KeyAuthEngine;
|
|
31
31
|
/** Map URL paths to local file paths, e.g. { '/admin': './admin/ui.html' } */
|
|
32
32
|
staticRoutes?: Record<string, string>;
|
|
33
|
+
/** Map URL path prefixes to local directories, e.g. { '/assets': './admin/dist/assets' } */
|
|
34
|
+
staticDirs?: Record<string, string>;
|
|
33
35
|
/** Custom REST routes checked before 404. Return null to fall through. */
|
|
34
36
|
extraRoutes?: Record<string, (req: Request, url: URL) => Response | Promise<Response> | null>;
|
|
35
37
|
/** Maximum concurrent WebSocket connections (default 10000) */
|
|
@@ -52,6 +54,11 @@ export class Transport {
|
|
|
52
54
|
private _log: ComponentLogger;
|
|
53
55
|
get clientCount(): number { return this._clients.size; }
|
|
54
56
|
|
|
57
|
+
/** Returns true if any WS client is subscribed to _admin/stats or _admin. */
|
|
58
|
+
get hasStatsSubscribers(): boolean {
|
|
59
|
+
return !!(this._valueGroups.get('_admin/stats')?.size || this._valueGroups.get('_admin')?.size);
|
|
60
|
+
}
|
|
61
|
+
|
|
55
62
|
/** Send pre-serialized stats directly to subscribed WS clients, bypassing SubscriptionEngine. */
|
|
56
63
|
broadcastStats(data: unknown, updatedAt: number): void {
|
|
57
64
|
// Check both exact path and ancestor subscribers
|
|
@@ -253,6 +260,49 @@ export class Transport {
|
|
|
253
260
|
})();
|
|
254
261
|
}
|
|
255
262
|
|
|
263
|
+
// Replication REST routes
|
|
264
|
+
if (url.pathname.startsWith('/replication')) {
|
|
265
|
+
return (async () => {
|
|
266
|
+
const repl = this.db.replication;
|
|
267
|
+
if (!repl) {
|
|
268
|
+
if (req.method === 'GET' && url.pathname === '/replication') {
|
|
269
|
+
return Response.json({ ok: true, role: 'primary', started: false, seq: 0, topology: null, sources: [], synced: {} });
|
|
270
|
+
}
|
|
271
|
+
return Response.json({ ok: false, error: 'Replication not configured' }, { status: 503 });
|
|
272
|
+
}
|
|
273
|
+
if (req.method === 'GET' && url.pathname === '/replication') {
|
|
274
|
+
const s = repl.stats();
|
|
275
|
+
// Build synced snapshot: read local copies of each source's paths
|
|
276
|
+
const synced: Record<string, unknown> = {};
|
|
277
|
+
for (const src of (s.sources ?? [])) {
|
|
278
|
+
for (const p of (src.paths ?? [])) {
|
|
279
|
+
try { synced[p] = this.db.get(p); } catch {}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return Response.json({ ok: true, ...s, synced });
|
|
283
|
+
}
|
|
284
|
+
if (req.method === 'POST' && url.pathname === '/replication/source-write') {
|
|
285
|
+
try {
|
|
286
|
+
const body = await req.json() as { path: string; value: unknown };
|
|
287
|
+
await repl.proxyWrite({ op: 'set', path: body.path, value: body.value });
|
|
288
|
+
return Response.json({ ok: true });
|
|
289
|
+
} catch (e: any) {
|
|
290
|
+
return Response.json({ ok: false, error: e.message }, { status: 500 });
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (req.method === 'DELETE' && url.pathname.startsWith('/replication/source-delete/')) {
|
|
294
|
+
const path = url.pathname.slice('/replication/source-delete/'.length);
|
|
295
|
+
try {
|
|
296
|
+
await repl.proxyWrite({ op: 'delete', path });
|
|
297
|
+
return Response.json({ ok: true });
|
|
298
|
+
} catch (e: any) {
|
|
299
|
+
return Response.json({ ok: false, error: e.message }, { status: 500 });
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return Response.json({ ok: false, error: 'Not found' }, { status: 404 });
|
|
303
|
+
})();
|
|
304
|
+
}
|
|
305
|
+
|
|
256
306
|
// VFS REST routes
|
|
257
307
|
if (this.db.vfs && url.pathname.startsWith('/files/')) {
|
|
258
308
|
const vfsPath = normalizePath(url.pathname.slice(7));
|
|
@@ -335,6 +385,16 @@ export class Transport {
|
|
|
335
385
|
if (filePath) return new Response(Bun.file(filePath));
|
|
336
386
|
}
|
|
337
387
|
|
|
388
|
+
// Static dirs (prefix-based, for SPA assets)
|
|
389
|
+
if (this.options.staticDirs) {
|
|
390
|
+
for (const [prefix, dir] of Object.entries(this.options.staticDirs)) {
|
|
391
|
+
if (url.pathname.startsWith(prefix)) {
|
|
392
|
+
const rel = url.pathname.slice(prefix.length);
|
|
393
|
+
return new Response(Bun.file(`${dir}${rel}`));
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
338
398
|
// Extra routes (custom REST handlers)
|
|
339
399
|
if (this.options.extraRoutes) {
|
|
340
400
|
for (const [pattern, handler] of Object.entries(this.options.extraRoutes)) {
|
|
@@ -276,7 +276,7 @@ describe('ReplicationEngine', () => {
|
|
|
276
276
|
|
|
277
277
|
// --- Source feed subscription tests ---
|
|
278
278
|
|
|
279
|
-
function createSourceLocal(sources: Array<{ url: string; paths: string[]; localPrefix?: string; excludePrefixes?: string[] }>) {
|
|
279
|
+
function createSourceLocal(sources: Array<{ url: string; paths: string[]; localPrefix?: string; excludePrefixes?: string[]; scopedStream?: boolean }>) {
|
|
280
280
|
const port = getPort();
|
|
281
281
|
const db = new BodDB({
|
|
282
282
|
path: ':memory:',
|
|
@@ -393,6 +393,167 @@ describe('ReplicationEngine', () => {
|
|
|
393
393
|
expect(local.get('src/users/u1')).toBeNull();
|
|
394
394
|
});
|
|
395
395
|
|
|
396
|
+
// --- Scoped replication stream tests ---
|
|
397
|
+
|
|
398
|
+
function createScopedPrimary(scopedPrefixes: string[]) {
|
|
399
|
+
const port = getPort();
|
|
400
|
+
const db = new BodDB({
|
|
401
|
+
path: ':memory:',
|
|
402
|
+
sweepInterval: 0,
|
|
403
|
+
replication: { role: 'primary', scopedPrefixes },
|
|
404
|
+
});
|
|
405
|
+
db.replication!.start();
|
|
406
|
+
db.serve({ port });
|
|
407
|
+
instances.push(db);
|
|
408
|
+
return { db, port };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
it('scoped primary dual-emits to _repl and _repl_scoped/<prefix>', async () => {
|
|
412
|
+
const { db } = createScopedPrimary(['users']);
|
|
413
|
+
db.set('users/u1', { name: 'Eli' });
|
|
414
|
+
db.set('orders/o1', { total: 100 });
|
|
415
|
+
await tick();
|
|
416
|
+
|
|
417
|
+
// Both events in _repl
|
|
418
|
+
const replData = db.get('_repl') as Record<string, any>;
|
|
419
|
+
const replPaths = Object.values(replData).map((e: any) => e.path);
|
|
420
|
+
expect(replPaths).toContain('users/u1');
|
|
421
|
+
expect(replPaths).toContain('orders/o1');
|
|
422
|
+
|
|
423
|
+
// Only users event in scoped stream
|
|
424
|
+
const scopedData = db.get('_repl_scoped/users') as Record<string, any>;
|
|
425
|
+
const scopedPaths = Object.values(scopedData).map((e: any) => e.path);
|
|
426
|
+
expect(scopedPaths).toContain('users/u1');
|
|
427
|
+
expect(scopedPaths).not.toContain('orders/o1');
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
it('source with scopedStream uses _repl_scoped instead of filtering _repl', async () => {
|
|
431
|
+
const { db: remote, port: rPort } = createScopedPrimary(['users']);
|
|
432
|
+
remote.set('users/u1', { name: 'Eli' });
|
|
433
|
+
remote.set('orders/o1', { total: 100 });
|
|
434
|
+
|
|
435
|
+
const { db: local } = createSourceLocal([
|
|
436
|
+
{ url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true },
|
|
437
|
+
]);
|
|
438
|
+
await local.replication!.start();
|
|
439
|
+
await wait(500);
|
|
440
|
+
|
|
441
|
+
expect(local.get('users/u1')).toEqual({ name: 'Eli' });
|
|
442
|
+
expect(local.get('orders/o1')).toBeNull();
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
it('source with scopedStream receives ongoing events', async () => {
|
|
446
|
+
const { db: remote, port: rPort } = createScopedPrimary(['users']);
|
|
447
|
+
remote.set('users/u1', { name: 'Eli' });
|
|
448
|
+
|
|
449
|
+
const { db: local } = createSourceLocal([
|
|
450
|
+
{ url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true },
|
|
451
|
+
]);
|
|
452
|
+
await local.replication!.start();
|
|
453
|
+
await wait(500);
|
|
454
|
+
|
|
455
|
+
expect(local.get('users/u1')).toEqual({ name: 'Eli' });
|
|
456
|
+
|
|
457
|
+
remote.set('users/u2', { name: 'Dan' });
|
|
458
|
+
await wait(500);
|
|
459
|
+
expect(local.get('users/u2')).toEqual({ name: 'Dan' });
|
|
460
|
+
|
|
461
|
+
// Non-matching writes should not appear
|
|
462
|
+
remote.set('orders/o1', { total: 50 });
|
|
463
|
+
await wait(500);
|
|
464
|
+
expect(local.get('orders/o1')).toBeNull();
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
it('whole-DB replica still works unchanged on _repl', async () => {
|
|
468
|
+
const { db: primary, port: pPort } = createScopedPrimary(['users']);
|
|
469
|
+
primary.set('users/u1', { name: 'Eli' });
|
|
470
|
+
primary.set('orders/o1', { total: 100 });
|
|
471
|
+
|
|
472
|
+
const { db: replica } = createReplica(pPort);
|
|
473
|
+
await replica.replication!.start();
|
|
474
|
+
await wait(500);
|
|
475
|
+
|
|
476
|
+
expect(replica.get('users/u1')).toEqual({ name: 'Eli' });
|
|
477
|
+
expect(replica.get('orders/o1')).toEqual({ total: 100 });
|
|
478
|
+
});
|
|
479
|
+
|
|
480
|
+
it('scoped compaction runs on _repl_scoped streams', async () => {
|
|
481
|
+
const { db } = createScopedPrimary(['users']);
|
|
482
|
+
// Write enough to trigger auto-compact threshold
|
|
483
|
+
for (let i = 0; i < 5; i++) db.set(`users/u${i}`, { n: i });
|
|
484
|
+
await tick();
|
|
485
|
+
|
|
486
|
+
const scopedData = db.get('_repl_scoped/users') as Record<string, any>;
|
|
487
|
+
expect(Object.keys(scopedData).length).toBeGreaterThan(0);
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it('per-path topology replica uses scoped streams', async () => {
|
|
491
|
+
const pPort = getPort();
|
|
492
|
+
const primary = new BodDB({
|
|
493
|
+
path: ':memory:',
|
|
494
|
+
sweepInterval: 0,
|
|
495
|
+
replication: { role: 'primary', scopedPrefixes: ['users', 'orders'] },
|
|
496
|
+
});
|
|
497
|
+
primary.replication!.start();
|
|
498
|
+
primary.serve({ port: pPort });
|
|
499
|
+
instances.push(primary);
|
|
500
|
+
|
|
501
|
+
primary.set('users/u1', { name: 'Eli' });
|
|
502
|
+
primary.set('orders/o1', { total: 100 });
|
|
503
|
+
primary.set('config/theme', 'dark');
|
|
504
|
+
|
|
505
|
+
const rPort = getPort();
|
|
506
|
+
const replica = new BodDB({
|
|
507
|
+
path: ':memory:',
|
|
508
|
+
sweepInterval: 0,
|
|
509
|
+
replication: {
|
|
510
|
+
role: 'replica',
|
|
511
|
+
primaryUrl: `ws://localhost:${pPort}`,
|
|
512
|
+
replicaId: `test-scoped-topo-${rPort}`,
|
|
513
|
+
paths: [{ path: 'users', mode: 'replica' as const }],
|
|
514
|
+
scopedPrefixes: ['users'],
|
|
515
|
+
},
|
|
516
|
+
});
|
|
517
|
+
replica.serve({ port: rPort });
|
|
518
|
+
instances.push(replica);
|
|
519
|
+
await replica.replication!.start();
|
|
520
|
+
await wait(500);
|
|
521
|
+
|
|
522
|
+
expect(replica.get('users/u1')).toEqual({ name: 'Eli' });
|
|
523
|
+
// orders not in topology — should not be replicated
|
|
524
|
+
expect(replica.get('orders/o1')).toBeNull();
|
|
525
|
+
});
|
|
526
|
+
|
|
527
|
+
it('multi-path source with scopedStream falls back to filtered _repl', async () => {
|
|
528
|
+
const { db: remote, port: rPort } = createScopedPrimary(['users', 'orders']);
|
|
529
|
+
remote.set('users/u1', { name: 'Eli' });
|
|
530
|
+
remote.set('orders/o1', { total: 100 });
|
|
531
|
+
|
|
532
|
+
const { db: local } = createSourceLocal([
|
|
533
|
+
{ url: `ws://localhost:${rPort}`, paths: ['users', 'orders'], scopedStream: true },
|
|
534
|
+
]);
|
|
535
|
+
await local.replication!.start();
|
|
536
|
+
await wait(500);
|
|
537
|
+
|
|
538
|
+
// Should still work via _repl fallback
|
|
539
|
+
expect(local.get('users/u1')).toEqual({ name: 'Eli' });
|
|
540
|
+
expect(local.get('orders/o1')).toEqual({ total: 100 });
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
it('source with scopedStream and localPrefix remaps correctly', async () => {
|
|
544
|
+
const { db: remote, port: rPort } = createScopedPrimary(['users']);
|
|
545
|
+
remote.set('users/u1', { name: 'Eli' });
|
|
546
|
+
|
|
547
|
+
const { db: local } = createSourceLocal([
|
|
548
|
+
{ url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true, localPrefix: 'ext' },
|
|
549
|
+
]);
|
|
550
|
+
await local.replication!.start();
|
|
551
|
+
await wait(500);
|
|
552
|
+
|
|
553
|
+
expect(local.get('ext/users/u1')).toEqual({ name: 'Eli' });
|
|
554
|
+
expect(local.get('users/u1')).toBeNull();
|
|
555
|
+
});
|
|
556
|
+
|
|
396
557
|
it('stop() disconnects source clients', async () => {
|
|
397
558
|
const { db: remote, port: rPort } = createPrimary();
|
|
398
559
|
remote.set('users/u1', { name: 'Eli' });
|