@bod.ee/db 0.12.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/config-file.md +6 -0
- package/.claude/skills/deploying-bod-db.md +21 -0
- package/.claude/skills/using-bod-db.md +30 -0
- package/CLAUDE.md +2 -1
- package/README.md +14 -0
- package/docs/para-chat-integration.md +198 -0
- package/docs/repl-stream-bloat-spec.md +48 -0
- package/package.json +1 -1
- package/src/server/BodDB.ts +3 -3
- package/src/server/ReplicationEngine.ts +254 -18
- package/src/server/Transport.ts +57 -29
- package/src/server/VFSEngine.ts +13 -2
- package/src/shared/protocol.ts +1 -0
- package/tests/repl-stream-bloat.test.ts +270 -0
- package/tests/replication-topology.test.ts +835 -0
|
@@ -2,6 +2,7 @@ import type { BodDB } from './BodDB.ts';
|
|
|
2
2
|
import { BodClient } from '../client/BodClient.ts';
|
|
3
3
|
import type { ClientMessage } from '../shared/protocol.ts';
|
|
4
4
|
import type { CompactOptions } from './StreamEngine.ts';
|
|
5
|
+
import type { ComponentLogger } from '../shared/logger.ts';
|
|
5
6
|
|
|
6
7
|
export interface ReplEvent {
|
|
7
8
|
op: 'set' | 'delete' | 'push';
|
|
@@ -20,6 +21,94 @@ export interface WriteEvent {
|
|
|
20
21
|
ttl?: number;
|
|
21
22
|
}
|
|
22
23
|
|
|
24
|
+
// --- Per-path topology types ---
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Per-path replication mode:
|
|
28
|
+
* - `primary`: local-authoritative, emits to _repl, not pulled from remote
|
|
29
|
+
* - `replica`: remote-authoritative, pulled from remote, client writes proxied to primary
|
|
30
|
+
* - `sync`: emits + pulls (one-directional pull — remote must also configure a source pointing back for true bidirectional)
|
|
31
|
+
* - `readonly`: pulled from remote, client writes rejected
|
|
32
|
+
* - `writeonly`: emits only, not pulled from remote, client writes allowed locally
|
|
33
|
+
*/
|
|
34
|
+
export type PathMode = 'primary' | 'replica' | 'sync' | 'readonly' | 'writeonly';
|
|
35
|
+
|
|
36
|
+
export interface PathTopology {
|
|
37
|
+
path: string;
|
|
38
|
+
mode: PathMode;
|
|
39
|
+
/** Override write behavior: 'reject' rejects writes even on replica paths (default: proxy for replica, reject for readonly) */
|
|
40
|
+
writeProxy?: 'proxy' | 'optimistic' | 'reject';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Resolves per-path replication mode via longest-prefix match */
|
|
44
|
+
export class PathTopologyRouter {
|
|
45
|
+
private entries: PathTopology[];
|
|
46
|
+
private fallbackMode: PathMode;
|
|
47
|
+
|
|
48
|
+
constructor(paths: Array<string | PathTopology>, fallbackRole: 'primary' | 'replica') {
|
|
49
|
+
this.fallbackMode = fallbackRole === 'primary' ? 'primary' : 'replica';
|
|
50
|
+
this.entries = paths.map(p =>
|
|
51
|
+
typeof p === 'string' ? { path: p, mode: 'sync' as PathMode } : p
|
|
52
|
+
);
|
|
53
|
+
// Sort longest-first for prefix matching
|
|
54
|
+
this.entries.sort((a, b) => b.path.length - a.path.length);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Longest-prefix match, falls back to role-based mode */
|
|
58
|
+
resolve(path: string): PathTopology {
|
|
59
|
+
for (const e of this.entries) {
|
|
60
|
+
if (path === e.path || path.startsWith(e.path + '/')) {
|
|
61
|
+
return e;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return { path: '', mode: this.fallbackMode };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Should this path's writes be emitted to _repl? */
|
|
68
|
+
shouldEmit(path: string): boolean {
|
|
69
|
+
const { mode } = this.resolve(path);
|
|
70
|
+
return mode === 'primary' || mode === 'sync' || mode === 'writeonly';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Should incoming repl events for this path be applied? */
|
|
74
|
+
shouldApply(path: string): boolean {
|
|
75
|
+
const { mode } = this.resolve(path);
|
|
76
|
+
return mode === 'replica' || mode === 'sync' || mode === 'readonly';
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Should client writes to this path be proxied to primary? */
|
|
80
|
+
shouldProxy(path: string): boolean {
|
|
81
|
+
const { mode, writeProxy } = this.resolve(path);
|
|
82
|
+
if (writeProxy === 'reject') return false;
|
|
83
|
+
if (mode === 'readonly') return false; // readonly rejects, not proxies
|
|
84
|
+
return mode === 'replica';
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Should client writes to this path be rejected? */
|
|
88
|
+
shouldReject(path: string): boolean {
|
|
89
|
+
const { mode, writeProxy } = this.resolve(path);
|
|
90
|
+
if (writeProxy === 'reject') return true;
|
|
91
|
+
return mode === 'readonly';
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Get all configured entries */
|
|
95
|
+
getEntries(): readonly PathTopology[] { return this.entries; }
|
|
96
|
+
|
|
97
|
+
/** Get paths that need pulling from primary (ongoing stream subscription) */
|
|
98
|
+
getReplicaPaths(): string[] {
|
|
99
|
+
return this.entries
|
|
100
|
+
.filter(e => e.mode === 'replica' || e.mode === 'readonly' || e.mode === 'sync')
|
|
101
|
+
.map(e => e.path);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Get paths that need full bootstrap from primary (excludes sync — sync paths get ongoing events only, avoiding overwrite of local state) */
|
|
105
|
+
getBootstrapPaths(): string[] {
|
|
106
|
+
return this.entries
|
|
107
|
+
.filter(e => e.mode === 'replica' || e.mode === 'readonly')
|
|
108
|
+
.map(e => e.path);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
23
112
|
export class ReplicationSource {
|
|
24
113
|
url: string = '';
|
|
25
114
|
auth?: () => string | Promise<string>;
|
|
@@ -34,17 +123,21 @@ export class ReplicationOptions {
|
|
|
34
123
|
primaryUrl?: string;
|
|
35
124
|
primaryAuth?: () => string | Promise<string>;
|
|
36
125
|
replicaId?: string;
|
|
126
|
+
/** Paths excluded from replication. Note: when `paths` explicitly configures a prefix (e.g. `_auth`), it overrides this exclusion. */
|
|
37
127
|
excludePrefixes: string[] = ['_repl', '_streams', '_mq', '_admin', '_auth'];
|
|
38
128
|
/** Bootstrap replica from primary's full state before applying _repl stream */
|
|
39
129
|
fullBootstrap: boolean = true;
|
|
40
130
|
compact?: CompactOptions;
|
|
41
131
|
sources?: ReplicationSource[];
|
|
132
|
+
/** Per-path topology: strings default to 'sync', objects specify mode. When absent, role governs all paths. */
|
|
133
|
+
paths?: Array<string | PathTopology>;
|
|
42
134
|
}
|
|
43
135
|
|
|
44
136
|
type ProxyableMessage = Extract<ClientMessage, { op: 'set' | 'delete' | 'update' | 'push' | 'batch' }>;
|
|
45
137
|
|
|
46
138
|
export class ReplicationEngine {
|
|
47
139
|
readonly options: ReplicationOptions;
|
|
140
|
+
readonly router: PathTopologyRouter | null = null;
|
|
48
141
|
private client: BodClient | null = null;
|
|
49
142
|
private unsubWrite: (() => void) | null = null;
|
|
50
143
|
private unsubStream: (() => void) | null = null;
|
|
@@ -53,9 +146,15 @@ export class ReplicationEngine {
|
|
|
53
146
|
private _seq = 0;
|
|
54
147
|
private _emitting = false;
|
|
55
148
|
private _pendingReplEvents: WriteEvent[] | null = null;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
149
|
+
private log: ComponentLogger;
|
|
150
|
+
|
|
151
|
+
/** When no router, falls back to role-based check. With router, both return false — use emitsToRepl/pullsFromPrimary instead. */
|
|
152
|
+
get isReplica(): boolean { return !this.router && this.options.role === 'replica'; }
|
|
153
|
+
get isPrimary(): boolean { return !this.router && this.options.role === 'primary'; }
|
|
154
|
+
/** True if this node emits write events to _repl (primary, router with emitting paths, or any node with startPrimary called) */
|
|
155
|
+
get emitsToRepl(): boolean { return this.isPrimary || !!this.router; }
|
|
156
|
+
/** True if this node pulls events from a primary */
|
|
157
|
+
get pullsFromPrimary(): boolean { return this.isReplica || (!!this.router && this.router.getReplicaPaths().length > 0); }
|
|
59
158
|
get started(): boolean { return this._started; }
|
|
60
159
|
get seq(): number { return this._seq; }
|
|
61
160
|
|
|
@@ -65,6 +164,7 @@ export class ReplicationEngine {
|
|
|
65
164
|
role: this.options.role,
|
|
66
165
|
started: this._started,
|
|
67
166
|
seq: this._seq,
|
|
167
|
+
topology: this.router ? this.router.getEntries().map(e => ({ path: e.path, mode: e.mode })) : null,
|
|
68
168
|
sources: (this.options.sources ?? []).map((s, i) => {
|
|
69
169
|
const conn = this.sourceConns[i];
|
|
70
170
|
return {
|
|
@@ -84,9 +184,42 @@ export class ReplicationEngine {
|
|
|
84
184
|
options?: Partial<ReplicationOptions>,
|
|
85
185
|
) {
|
|
86
186
|
this.options = { ...new ReplicationOptions(), ...options };
|
|
187
|
+
this.log = db.log.forComponent('repl');
|
|
188
|
+
if (this.options.paths?.length) {
|
|
189
|
+
this.router = new PathTopologyRouter(this.options.paths, this.options.role);
|
|
190
|
+
}
|
|
87
191
|
if (!this.options.replicaId) {
|
|
88
|
-
|
|
192
|
+
// Derive stable ID from config when possible, fall back to random
|
|
193
|
+
if (this.options.primaryUrl) {
|
|
194
|
+
const seed = `${this.options.primaryUrl}:${(this.options.paths ?? []).map(p => typeof p === 'string' ? p : p.path).sort().join('+')}`;
|
|
195
|
+
// Simple hash for stability across restarts
|
|
196
|
+
let h = 0;
|
|
197
|
+
for (let i = 0; i < seed.length; i++) h = ((h << 5) - h + seed.charCodeAt(i)) | 0;
|
|
198
|
+
this.options.replicaId = `replica_${(h >>> 0).toString(36)}`;
|
|
199
|
+
} else {
|
|
200
|
+
this.options.replicaId = `replica_${Math.random().toString(36).slice(2, 10)}`;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Should client writes to this path be proxied to primary? */
|
|
206
|
+
shouldProxyPath(path: string): boolean {
|
|
207
|
+
if (this.router) {
|
|
208
|
+
// Only proxy if we have a client connection to proxy through
|
|
209
|
+
return this.router.shouldProxy(path) && !!this.client;
|
|
89
210
|
}
|
|
211
|
+
return this.options.role === 'replica';
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Should client writes to this path be rejected? */
|
|
215
|
+
shouldRejectPath(path: string): boolean {
|
|
216
|
+
if (this.router) {
|
|
217
|
+
const shouldProxy = this.router.shouldProxy(path);
|
|
218
|
+
// If path wants proxy but no client → reject instead of crashing
|
|
219
|
+
if (shouldProxy && !this.client) return true;
|
|
220
|
+
return this.router.shouldReject(path);
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
90
223
|
}
|
|
91
224
|
|
|
92
225
|
/** Start replication — primary listens for writes, replica connects and consumes */
|
|
@@ -94,9 +227,19 @@ export class ReplicationEngine {
|
|
|
94
227
|
if (this._started) return;
|
|
95
228
|
this._started = true;
|
|
96
229
|
|
|
97
|
-
|
|
230
|
+
// Always hook writes for emit (router or primary will filter)
|
|
231
|
+
if (this.router || this.isPrimary) {
|
|
98
232
|
this.startPrimary();
|
|
99
|
-
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// Connect to primary for replica/readonly/sync paths
|
|
236
|
+
if (this.router) {
|
|
237
|
+
const pullPaths = this.router.getReplicaPaths();
|
|
238
|
+
if (pullPaths.length) {
|
|
239
|
+
if (!this.options.primaryUrl) throw new Error('primaryUrl is required when paths include replica/readonly/sync modes');
|
|
240
|
+
await this.startReplicaForPaths(pullPaths);
|
|
241
|
+
}
|
|
242
|
+
} else if (this.options.role === 'replica') {
|
|
100
243
|
await this.startReplica();
|
|
101
244
|
}
|
|
102
245
|
|
|
@@ -108,6 +251,7 @@ export class ReplicationEngine {
|
|
|
108
251
|
/** Stop replication */
|
|
109
252
|
stop(): void {
|
|
110
253
|
this._started = false;
|
|
254
|
+
if (this._compactTimer) { clearInterval(this._compactTimer); this._compactTimer = null; }
|
|
111
255
|
this.unsubWrite?.();
|
|
112
256
|
this.unsubWrite = null;
|
|
113
257
|
this.unsubStream?.();
|
|
@@ -123,7 +267,7 @@ export class ReplicationEngine {
|
|
|
123
267
|
|
|
124
268
|
/** Proxy a write operation to the primary (replica mode) */
|
|
125
269
|
async proxyWrite(msg: ProxyableMessage): Promise<unknown> {
|
|
126
|
-
if (!this.client) throw new Error('Replica not connected to primary');
|
|
270
|
+
if (!this.client) throw new Error('Replica not connected to primary — ensure primaryUrl is set and start() has been called');
|
|
127
271
|
switch (msg.op) {
|
|
128
272
|
case 'set':
|
|
129
273
|
await this.client.set(msg.path, msg.value, msg.ttl ? { ttl: msg.ttl } : undefined);
|
|
@@ -145,20 +289,51 @@ export class ReplicationEngine {
|
|
|
145
289
|
|
|
146
290
|
// --- Primary mode ---
|
|
147
291
|
|
|
292
|
+
private _compactTimer: ReturnType<typeof setInterval> | null = null;
|
|
293
|
+
|
|
148
294
|
private startPrimary(): void {
|
|
149
295
|
this.unsubWrite = this.db.onWrite((ev: WriteEvent) => {
|
|
150
296
|
this.emit(ev);
|
|
151
297
|
});
|
|
298
|
+
|
|
299
|
+
// Auto-compact _repl stream to prevent unbounded growth
|
|
300
|
+
const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
|
|
301
|
+
if (compact.maxCount || compact.maxAge) {
|
|
302
|
+
// Compact on startup
|
|
303
|
+
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
304
|
+
// Then periodically (every 5 minutes)
|
|
305
|
+
this._compactTimer = setInterval(() => {
|
|
306
|
+
try { this.db.stream.compact('_repl', compact); } catch {}
|
|
307
|
+
}, 5 * 60_000);
|
|
308
|
+
}
|
|
152
309
|
}
|
|
153
310
|
|
|
154
311
|
private isExcluded(path: string): boolean {
|
|
155
|
-
|
|
312
|
+
if (this.options.excludePrefixes.some(p => path.startsWith(p))) {
|
|
313
|
+
// If router explicitly configures this path, don't exclude it
|
|
314
|
+
if (this.router) {
|
|
315
|
+
const resolved = this.router.resolve(path);
|
|
316
|
+
if (resolved.path !== '') return false; // explicitly configured → not excluded
|
|
317
|
+
}
|
|
318
|
+
return true;
|
|
319
|
+
}
|
|
320
|
+
return false;
|
|
156
321
|
}
|
|
157
322
|
|
|
158
323
|
/** Buffer replication events during transactions, emit immediately otherwise */
|
|
159
324
|
private emit(ev: WriteEvent): void {
|
|
160
325
|
if (this._emitting) return;
|
|
161
|
-
if (this.
|
|
326
|
+
if (this.router) {
|
|
327
|
+
// Single resolve: check exclusion override + shouldEmit together
|
|
328
|
+
const resolved = this.router.resolve(ev.path);
|
|
329
|
+
const isConfigured = resolved.path !== '';
|
|
330
|
+
// If in excludePrefixes but not explicitly configured, skip
|
|
331
|
+
if (!isConfigured && this.options.excludePrefixes.some(p => ev.path.startsWith(p))) return;
|
|
332
|
+
const mode = resolved.mode;
|
|
333
|
+
if (mode !== 'primary' && mode !== 'sync' && mode !== 'writeonly') return;
|
|
334
|
+
} else {
|
|
335
|
+
if (this.isExcluded(ev.path)) return;
|
|
336
|
+
}
|
|
162
337
|
|
|
163
338
|
// If buffering (transaction in progress), collect events
|
|
164
339
|
if (this._pendingReplEvents) {
|
|
@@ -202,6 +377,62 @@ export class ReplicationEngine {
|
|
|
202
377
|
|
|
203
378
|
// --- Replica mode ---
|
|
204
379
|
|
|
380
|
+
/** Start replica for specific path prefixes (router-based) */
|
|
381
|
+
private async startReplicaForPaths(pathPrefixes: string[]): Promise<void> {
|
|
382
|
+
if (!this.options.primaryUrl) throw new Error('primaryUrl required for per-path replication');
|
|
383
|
+
|
|
384
|
+
this.client = new BodClient({
|
|
385
|
+
url: this.options.primaryUrl,
|
|
386
|
+
auth: this.options.primaryAuth,
|
|
387
|
+
});
|
|
388
|
+
await this.client.connect();
|
|
389
|
+
this.log.info(`Connected to primary ${this.options.primaryUrl} (paths: ${pathPrefixes.join(', ')})`);
|
|
390
|
+
|
|
391
|
+
// Bootstrap only replica/readonly paths (sync paths get ongoing events only — avoids overwriting local state)
|
|
392
|
+
const bootstrapPaths = this.router!.getBootstrapPaths();
|
|
393
|
+
if (this.options.fullBootstrap && bootstrapPaths.length) {
|
|
394
|
+
await this.bootstrapFullState(bootstrapPaths);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Stream bootstrap filtered
|
|
398
|
+
const snapshot = await this.client!.streamMaterialize('_repl', { keepKey: 'path' });
|
|
399
|
+
if (snapshot) {
|
|
400
|
+
this.db.setReplaying(true);
|
|
401
|
+
try {
|
|
402
|
+
for (const [, event] of Object.entries(snapshot)) {
|
|
403
|
+
const ev = event as ReplEvent;
|
|
404
|
+
if (this.matchesPathPrefixes(ev.path, pathPrefixes)) {
|
|
405
|
+
this.applyEvent(ev);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
} finally {
|
|
409
|
+
this.db.setReplaying(false);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
// Subscribe to ongoing events, filter by paths
|
|
414
|
+
const groupId = this.options.replicaId!;
|
|
415
|
+
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
416
|
+
this.db.setReplaying(true);
|
|
417
|
+
try {
|
|
418
|
+
for (const e of events) {
|
|
419
|
+
const ev = e.val() as ReplEvent;
|
|
420
|
+
if (this.matchesPathPrefixes(ev.path, pathPrefixes)) {
|
|
421
|
+
this.applyEvent(ev);
|
|
422
|
+
}
|
|
423
|
+
this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
|
|
424
|
+
}
|
|
425
|
+
} finally {
|
|
426
|
+
this.db.setReplaying(false);
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Check if path matches any of the given prefixes */
|
|
432
|
+
private matchesPathPrefixes(path: string, prefixes: string[]): boolean {
|
|
433
|
+
return prefixes.some(p => path === p || path.startsWith(p + '/'));
|
|
434
|
+
}
|
|
435
|
+
|
|
205
436
|
private async startReplica(): Promise<void> {
|
|
206
437
|
if (!this.options.primaryUrl) throw new Error('primaryUrl required for replica mode');
|
|
207
438
|
|
|
@@ -211,7 +442,7 @@ export class ReplicationEngine {
|
|
|
211
442
|
});
|
|
212
443
|
|
|
213
444
|
await this.client.connect();
|
|
214
|
-
|
|
445
|
+
this.log.info(`Connected to primary ${this.options.primaryUrl}`);
|
|
215
446
|
|
|
216
447
|
// Full state bootstrap: fetch all data from primary (catches pre-replication data)
|
|
217
448
|
if (this.options.fullBootstrap) {
|
|
@@ -220,7 +451,7 @@ export class ReplicationEngine {
|
|
|
220
451
|
|
|
221
452
|
// Stream bootstrap: apply _repl events on top (catches recent writes, deduped by idempotent set)
|
|
222
453
|
const snapshot = await this.client!.streamMaterialize('_repl', { keepKey: 'path' });
|
|
223
|
-
|
|
454
|
+
this.log.info(`Stream bootstrap: ${snapshot ? Object.keys(snapshot).length + ' events' : 'no events'}`);
|
|
224
455
|
if (snapshot) {
|
|
225
456
|
this.db.setReplaying(true);
|
|
226
457
|
try {
|
|
@@ -235,9 +466,9 @@ export class ReplicationEngine {
|
|
|
235
466
|
|
|
236
467
|
// Subscribe to ongoing events
|
|
237
468
|
const groupId = this.options.replicaId!;
|
|
238
|
-
|
|
469
|
+
this.log.info(`Subscribing to stream as '${groupId}'`);
|
|
239
470
|
this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
|
|
240
|
-
|
|
471
|
+
this.log.debug(`Received ${events.length} events`);
|
|
241
472
|
this.db.setReplaying(true);
|
|
242
473
|
try {
|
|
243
474
|
for (const e of events) {
|
|
@@ -252,13 +483,16 @@ export class ReplicationEngine {
|
|
|
252
483
|
});
|
|
253
484
|
}
|
|
254
485
|
|
|
255
|
-
/** Fetch full DB state from primary and apply locally */
|
|
256
|
-
private async bootstrapFullState(): Promise<void> {
|
|
486
|
+
/** Fetch full DB state from primary and apply locally. Optional pathPrefixes filters which top-level keys to pull. */
|
|
487
|
+
private async bootstrapFullState(pathPrefixes?: string[]): Promise<void> {
|
|
257
488
|
const topLevel = await this.client!.getShallow();
|
|
258
|
-
|
|
489
|
+
let keys = topLevel
|
|
259
490
|
.map(e => e.key)
|
|
260
491
|
.filter(k => !this.isExcluded(k));
|
|
261
|
-
|
|
492
|
+
if (pathPrefixes) {
|
|
493
|
+
keys = keys.filter(k => this.matchesPathPrefixes(k, pathPrefixes));
|
|
494
|
+
}
|
|
495
|
+
this.log.info(`Full bootstrap: ${keys.length} top-level keys [${keys.join(', ')}]`);
|
|
262
496
|
if (keys.length === 0) return;
|
|
263
497
|
|
|
264
498
|
this.db.setReplaying(true);
|
|
@@ -283,7 +517,7 @@ export class ReplicationEngine {
|
|
|
283
517
|
for (let i = 0; i < results.length; i++) {
|
|
284
518
|
if (results[i].status === 'rejected') {
|
|
285
519
|
const src = this.options.sources![i];
|
|
286
|
-
|
|
520
|
+
this.log.error(`source ${src.url} paths=[${src.paths}] failed:`, (results[i] as PromiseRejectedResult).reason);
|
|
287
521
|
}
|
|
288
522
|
}
|
|
289
523
|
}
|
|
@@ -345,6 +579,8 @@ export class ReplicationEngine {
|
|
|
345
579
|
|
|
346
580
|
private applyEvent(ev: ReplEvent, source?: ReplicationSource): void {
|
|
347
581
|
const path = source ? this.remapPath(ev.path, source) : ev.path;
|
|
582
|
+
// Defense-in-depth: skip events for paths we shouldn't apply (primary/writeonly)
|
|
583
|
+
if (!source && this.router && !this.router.shouldApply(path)) return;
|
|
348
584
|
switch (ev.op) {
|
|
349
585
|
case 'set':
|
|
350
586
|
this.db.set(path, ev.value, ev.ttl ? { ttl: ev.ttl } : undefined);
|
package/src/server/Transport.ts
CHANGED
|
@@ -159,7 +159,14 @@ export class Transport {
|
|
|
159
159
|
if (this.options.keyAuth && path.startsWith(AUTH_PREFIX)) {
|
|
160
160
|
return Response.json({ ok: false, error: `Write to ${AUTH_PREFIX} is reserved`, code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
161
161
|
}
|
|
162
|
-
|
|
162
|
+
const auth = await this.extractAuth(req);
|
|
163
|
+
if (this.rules && !this.rules.check('write', path, auth)) {
|
|
164
|
+
return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
165
|
+
}
|
|
166
|
+
if (this.db.replication?.shouldRejectPath(path)) {
|
|
167
|
+
return Response.json({ ok: false, error: 'Path is readonly', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
168
|
+
}
|
|
169
|
+
if (this.db.replication?.shouldProxyPath(path)) {
|
|
163
170
|
try {
|
|
164
171
|
const body = await req.json();
|
|
165
172
|
await this.db.replication.proxyWrite({ op: 'set', path, value: body });
|
|
@@ -168,10 +175,6 @@ export class Transport {
|
|
|
168
175
|
return Response.json({ ok: false, error: e.message, code: Errors.INTERNAL }, { status: 500 });
|
|
169
176
|
}
|
|
170
177
|
}
|
|
171
|
-
const auth = await this.extractAuth(req);
|
|
172
|
-
if (this.rules && !this.rules.check('write', path, auth)) {
|
|
173
|
-
return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
174
|
-
}
|
|
175
178
|
const body = await req.json();
|
|
176
179
|
this.db.set(path, body);
|
|
177
180
|
return Response.json({ ok: true });
|
|
@@ -185,7 +188,14 @@ export class Transport {
|
|
|
185
188
|
if (this.options.keyAuth && path.startsWith(AUTH_PREFIX)) {
|
|
186
189
|
return Response.json({ ok: false, error: `Write to ${AUTH_PREFIX} is reserved`, code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
187
190
|
}
|
|
188
|
-
|
|
191
|
+
const auth = await this.extractAuth(req);
|
|
192
|
+
if (this.rules && !this.rules.check('write', path, auth)) {
|
|
193
|
+
return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
194
|
+
}
|
|
195
|
+
if (this.db.replication?.shouldRejectPath(path)) {
|
|
196
|
+
return Response.json({ ok: false, error: 'Path is readonly', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
197
|
+
}
|
|
198
|
+
if (this.db.replication?.shouldProxyPath(path)) {
|
|
189
199
|
try {
|
|
190
200
|
await this.db.replication.proxyWrite({ op: 'delete', path });
|
|
191
201
|
return Response.json({ ok: true });
|
|
@@ -193,10 +203,6 @@ export class Transport {
|
|
|
193
203
|
return Response.json({ ok: false, error: e.message, code: Errors.INTERNAL }, { status: 500 });
|
|
194
204
|
}
|
|
195
205
|
}
|
|
196
|
-
const auth = await this.extractAuth(req);
|
|
197
|
-
if (this.rules && !this.rules.check('write', path, auth)) {
|
|
198
|
-
return Response.json({ ok: false, error: 'Permission denied', code: Errors.PERMISSION_DENIED }, { status: 403 });
|
|
199
|
-
}
|
|
200
206
|
this.db.delete(path);
|
|
201
207
|
return Response.json({ ok: true });
|
|
202
208
|
})();
|
|
@@ -418,13 +424,14 @@ export class Transport {
|
|
|
418
424
|
|
|
419
425
|
case 'set': {
|
|
420
426
|
if (guardAuthPrefix(msg.path)) return;
|
|
421
|
-
if (self.db.replication?.isReplica) {
|
|
422
|
-
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
423
|
-
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
424
|
-
}
|
|
425
427
|
if (self.rules && !self.rules.check('write', msg.path, ws.data.auth, self.db.get(msg.path), msg.value)) {
|
|
426
428
|
return error('Permission denied', Errors.PERMISSION_DENIED);
|
|
427
429
|
}
|
|
430
|
+
if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
|
|
431
|
+
if (self.db.replication?.shouldProxyPath(msg.path)) {
|
|
432
|
+
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
433
|
+
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
434
|
+
}
|
|
428
435
|
self.db.set(msg.path, msg.value, msg.ttl ? { ttl: msg.ttl } : undefined);
|
|
429
436
|
return reply(null);
|
|
430
437
|
}
|
|
@@ -433,28 +440,34 @@ export class Transport {
|
|
|
433
440
|
for (const path of Object.keys(msg.updates)) {
|
|
434
441
|
if (guardAuthPrefix(path)) return;
|
|
435
442
|
}
|
|
436
|
-
if (self.db.replication?.isReplica) {
|
|
437
|
-
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
438
|
-
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
439
|
-
}
|
|
440
443
|
for (const path of Object.keys(msg.updates)) {
|
|
441
444
|
if (self.rules && !self.rules.check('write', path, ws.data.auth)) {
|
|
442
445
|
return error(`Permission denied for ${path}`, Errors.PERMISSION_DENIED);
|
|
443
446
|
}
|
|
444
447
|
}
|
|
448
|
+
// Per-path: if ANY path needs reject, reject; if ANY needs proxy, proxy all (known limitation: greedy)
|
|
449
|
+
if (self.db.replication) {
|
|
450
|
+
const paths = Object.keys(msg.updates);
|
|
451
|
+
if (paths.some(p => self.db.replication!.shouldRejectPath(p))) return error('Path is readonly', Errors.PERMISSION_DENIED);
|
|
452
|
+
if (paths.some(p => self.db.replication!.shouldProxyPath(p))) {
|
|
453
|
+
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
454
|
+
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
455
|
+
}
|
|
456
|
+
}
|
|
445
457
|
self.db.update(msg.updates);
|
|
446
458
|
return reply(null);
|
|
447
459
|
}
|
|
448
460
|
|
|
449
461
|
case 'delete': {
|
|
450
462
|
if (guardAuthPrefix(msg.path)) return;
|
|
451
|
-
if (self.db.replication?.isReplica) {
|
|
452
|
-
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
453
|
-
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
454
|
-
}
|
|
455
463
|
if (self.rules && !self.rules.check('write', msg.path, ws.data.auth)) {
|
|
456
464
|
return error('Permission denied', Errors.PERMISSION_DENIED);
|
|
457
465
|
}
|
|
466
|
+
if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
|
|
467
|
+
if (self.db.replication?.shouldProxyPath(msg.path)) {
|
|
468
|
+
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
469
|
+
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
470
|
+
}
|
|
458
471
|
self.db.delete(msg.path);
|
|
459
472
|
return reply(null);
|
|
460
473
|
}
|
|
@@ -502,9 +515,23 @@ export class Transport {
|
|
|
502
515
|
}
|
|
503
516
|
|
|
504
517
|
case 'batch': {
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
518
|
+
// Upfront rules check before proxy (defense-in-depth)
|
|
519
|
+
for (const batchOp of msg.operations) {
|
|
520
|
+
const opPaths = batchOp.op === 'update' ? Object.keys(batchOp.updates) : [batchOp.path];
|
|
521
|
+
for (const p of opPaths) {
|
|
522
|
+
if (self.rules && !self.rules.check('write', p, ws.data.auth)) {
|
|
523
|
+
return error(`Permission denied for ${p}`, Errors.PERMISSION_DENIED);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
// Per-path topology: if ANY path needs reject, reject; if ANY needs proxy, proxy all (known limitation: greedy)
|
|
528
|
+
if (self.db.replication) {
|
|
529
|
+
const batchPaths = msg.operations.map((o: any) => o.path != null ? [o.path] : (o.updates ? Object.keys(o.updates) : [])).flat();
|
|
530
|
+
if (batchPaths.some((p: string) => self.db.replication!.shouldRejectPath(p))) return error('Path is readonly', Errors.PERMISSION_DENIED);
|
|
531
|
+
if (batchPaths.some((p: string) => self.db.replication!.shouldProxyPath(p))) {
|
|
532
|
+
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
533
|
+
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
534
|
+
}
|
|
508
535
|
}
|
|
509
536
|
const results: unknown[] = [];
|
|
510
537
|
self.db.transaction((tx) => {
|
|
@@ -546,13 +573,14 @@ export class Transport {
|
|
|
546
573
|
|
|
547
574
|
case 'push': {
|
|
548
575
|
if (guardAuthPrefix(msg.path)) return;
|
|
549
|
-
if (self.db.replication?.isReplica) {
|
|
550
|
-
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
551
|
-
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
552
|
-
}
|
|
553
576
|
if (self.rules && !self.rules.check('write', msg.path, ws.data.auth)) {
|
|
554
577
|
return error('Permission denied', Errors.PERMISSION_DENIED);
|
|
555
578
|
}
|
|
579
|
+
if (self.db.replication?.shouldRejectPath(msg.path)) return error('Path is readonly', Errors.PERMISSION_DENIED);
|
|
580
|
+
if (self.db.replication?.shouldProxyPath(msg.path)) {
|
|
581
|
+
try { return reply(await self.db.replication.proxyWrite(msg)); }
|
|
582
|
+
catch (e: any) { return error(e.message, Errors.INTERNAL); }
|
|
583
|
+
}
|
|
556
584
|
const key = self.db.push(msg.path, msg.value, msg.idempotencyKey ? { idempotencyKey: msg.idempotencyKey } : undefined);
|
|
557
585
|
return reply(key);
|
|
558
586
|
}
|
package/src/server/VFSEngine.ts
CHANGED
|
@@ -133,9 +133,14 @@ export class VFSEngine {
|
|
|
133
133
|
|
|
134
134
|
async write(virtualPath: string, data: Uint8Array, mime?: string): Promise<FileStat> {
|
|
135
135
|
const vp = normalizePath(virtualPath);
|
|
136
|
-
const existing = this.db.get(this.metaPath(vp)) as
|
|
136
|
+
const existing = this.db.get(this.metaPath(vp)) as FileStat | null;
|
|
137
137
|
const fileId = this.options.pathAsFileId ? vp : ((existing?.fileId as string) || generatePushId());
|
|
138
138
|
|
|
139
|
+
const hash = await computeHash(data);
|
|
140
|
+
|
|
141
|
+
// Skip write entirely if content hasn't changed (prevents unnecessary replication events)
|
|
142
|
+
if (existing?.hash === hash) return existing;
|
|
143
|
+
|
|
139
144
|
await this.backend.write(fileId, data);
|
|
140
145
|
|
|
141
146
|
const name = vp.split('/').pop()!;
|
|
@@ -147,6 +152,7 @@ export class VFSEngine {
|
|
|
147
152
|
mtime: Date.now(),
|
|
148
153
|
fileId,
|
|
149
154
|
isDir: false,
|
|
155
|
+
hash,
|
|
150
156
|
};
|
|
151
157
|
this.db.set(this.metaPath(vp), stat);
|
|
152
158
|
return stat;
|
|
@@ -251,13 +257,18 @@ export class VFSEngine {
|
|
|
251
257
|
|
|
252
258
|
const newName = dstPath.split('/').pop()!;
|
|
253
259
|
const fileId = this.options.pathAsFileId ? dstPath : meta.fileId;
|
|
254
|
-
const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now() };
|
|
260
|
+
const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now(), hash: meta.hash };
|
|
255
261
|
this.db.set(this.metaPath(dstPath), updated);
|
|
256
262
|
this.db.delete(this.containerPath(srcPath));
|
|
257
263
|
return updated;
|
|
258
264
|
}
|
|
259
265
|
}
|
|
260
266
|
|
|
267
|
+
async function computeHash(data: Uint8Array): Promise<string> {
|
|
268
|
+
const digest = await crypto.subtle.digest('SHA-256', data);
|
|
269
|
+
return Array.from(new Uint8Array(digest)).map(b => b.toString(16).padStart(2, '0')).join('');
|
|
270
|
+
}
|
|
271
|
+
|
|
261
272
|
function guessMime(name: string): string {
|
|
262
273
|
const ext = name.split('.').pop()?.toLowerCase();
|
|
263
274
|
const mimes: Record<string, string> = {
|