@bod.ee/db 0.12.2 → 0.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/developing-bod-db.md +1 -1
- package/.claude/skills/using-bod-db.md +19 -1
- package/CLAUDE.md +1 -1
- package/admin/ui.html +7 -6
- package/docs/para-chat-integration.md +7 -6
- package/package.json +1 -1
- package/src/client/BodClient.ts +9 -5
- package/src/server/BodDB.ts +6 -1
- package/src/server/ReplicationEngine.ts +80 -62
- package/src/server/Transport.ts +7 -2
- package/src/shared/logger.ts +40 -0
- package/src/shared/protocol.ts +1 -1
- package/tests/optimization.test.ts +4 -2
- package/tests/repl-load.test.ts +372 -0
- package/tests/repl-stream-bloat.test.ts +65 -35
- package/tests/replication-topology.test.ts +49 -7
- package/tests/replication.test.ts +16 -7
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from 'bun:test';
|
|
2
|
+
import { BodDB } from '../src/server/BodDB.ts';
|
|
3
|
+
import { BodClient } from '../src/client/BodClient.ts';
|
|
4
|
+
|
|
5
|
+
const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
|
|
6
|
+
let nextPort = 27400 + Math.floor(Math.random() * 1000);
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Massive load tests — battle-test cursor-based bootstrap + threshold compact
|
|
10
|
+
* under realistic and extreme conditions.
|
|
11
|
+
*/
|
|
12
|
+
describe('repl load test', () => {
|
|
13
|
+
const instances: BodDB[] = [];
|
|
14
|
+
const clients: BodClient[] = [];
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
for (const c of clients) c.disconnect();
|
|
18
|
+
clients.length = 0;
|
|
19
|
+
for (const db of [...instances].reverse()) db.close();
|
|
20
|
+
instances.length = 0;
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
function port() { return nextPort++; }
|
|
24
|
+
|
|
25
|
+
function primary(opts?: { compact?: any; autoCompactThreshold?: number }) {
|
|
26
|
+
const p = port();
|
|
27
|
+
const db = new BodDB({
|
|
28
|
+
path: ':memory:',
|
|
29
|
+
sweepInterval: 0,
|
|
30
|
+
replication: { role: 'primary', compact: opts?.compact ?? {}, autoCompactThreshold: opts?.autoCompactThreshold ?? 0 },
|
|
31
|
+
});
|
|
32
|
+
db.replication!.start();
|
|
33
|
+
db.serve({ port: p });
|
|
34
|
+
instances.push(db);
|
|
35
|
+
return { db, port: p };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function replica(primaryPort: number, opts?: Partial<{ replicaId: string; fullBootstrap: boolean }>) {
|
|
39
|
+
const p = port();
|
|
40
|
+
const db = new BodDB({
|
|
41
|
+
path: ':memory:',
|
|
42
|
+
sweepInterval: 0,
|
|
43
|
+
replication: {
|
|
44
|
+
role: 'replica',
|
|
45
|
+
primaryUrl: `ws://localhost:${primaryPort}`,
|
|
46
|
+
replicaId: opts?.replicaId ?? `load-replica-${p}`,
|
|
47
|
+
fullBootstrap: opts?.fullBootstrap ?? true,
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
db.serve({ port: p });
|
|
51
|
+
instances.push(db);
|
|
52
|
+
return { db, port: p };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function connect(p: number, opts?: any) {
|
|
56
|
+
const c = new BodClient({ url: `ws://localhost:${p}`, ...opts });
|
|
57
|
+
clients.push(c);
|
|
58
|
+
return c;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ─── 1. 20k entries: cursor pagination end-to-end ───
|
|
62
|
+
|
|
63
|
+
it('20k entries: cursor pagination collects every key', async () => {
|
|
64
|
+
const { db, port: p } = primary();
|
|
65
|
+
for (let i = 0; i < 20_000; i++) {
|
|
66
|
+
db.set(`items/i${i}`, { v: i, ts: Date.now() });
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const client = connect(p);
|
|
70
|
+
await client.connect();
|
|
71
|
+
|
|
72
|
+
const keys = new Set<string>();
|
|
73
|
+
let cursor: string | undefined;
|
|
74
|
+
let pages = 0;
|
|
75
|
+
const t0 = Date.now();
|
|
76
|
+
do {
|
|
77
|
+
const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 500, cursor });
|
|
78
|
+
for (const k of Object.keys(page.data)) keys.add(k);
|
|
79
|
+
cursor = page.nextCursor;
|
|
80
|
+
pages++;
|
|
81
|
+
} while (cursor);
|
|
82
|
+
const elapsed = Date.now() - t0;
|
|
83
|
+
|
|
84
|
+
console.log(` 20k entries: ${pages} pages, ${keys.size} unique keys, ${elapsed}ms`);
|
|
85
|
+
expect(keys.size).toBe(20_000);
|
|
86
|
+
expect(pages).toBeGreaterThanOrEqual(40); // 20k / 500
|
|
87
|
+
}, 30_000);
|
|
88
|
+
|
|
89
|
+
// ─── 2. Cursor vs monolithic: per-page response stays small ───
|
|
90
|
+
|
|
91
|
+
it('cursor pages stay under 1MB each while monolithic is huge', async () => {
|
|
92
|
+
const { db, port: p } = primary();
|
|
93
|
+
const payload = 'z'.repeat(500);
|
|
94
|
+
for (let i = 0; i < 5000; i++) {
|
|
95
|
+
db.set(`big/p${i}`, { data: payload, i });
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const client = connect(p);
|
|
99
|
+
await client.connect();
|
|
100
|
+
|
|
101
|
+
// Monolithic
|
|
102
|
+
const mono = await client.streamMaterialize('_repl', { keepKey: 'path' });
|
|
103
|
+
const monoSize = JSON.stringify(mono).length;
|
|
104
|
+
|
|
105
|
+
// Cursor-based
|
|
106
|
+
let maxPageSize = 0;
|
|
107
|
+
let cursor: string | undefined;
|
|
108
|
+
do {
|
|
109
|
+
const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 200, cursor });
|
|
110
|
+
const sz = JSON.stringify(page.data).length;
|
|
111
|
+
if (sz > maxPageSize) maxPageSize = sz;
|
|
112
|
+
cursor = page.nextCursor;
|
|
113
|
+
} while (cursor);
|
|
114
|
+
|
|
115
|
+
console.log(` monolithic: ${(monoSize / 1024 / 1024).toFixed(2)}MB, max page: ${(maxPageSize / 1024).toFixed(0)}KB`);
|
|
116
|
+
expect(monoSize).toBeGreaterThan(2 * 1024 * 1024); // >2MB total
|
|
117
|
+
expect(maxPageSize).toBeLessThan(1024 * 1024); // each page <1MB
|
|
118
|
+
}, 15_000);
|
|
119
|
+
|
|
120
|
+
// ─── 3. Auto-compact under sustained write load ───
|
|
121
|
+
|
|
122
|
+
it('auto-compact keeps _repl bounded under sustained 10k writes', async () => {
|
|
123
|
+
const { db } = primary({ compact: { maxCount: 200, keepKey: 'path' }, autoCompactThreshold: 500 });
|
|
124
|
+
|
|
125
|
+
for (let i = 0; i < 10_000; i++) {
|
|
126
|
+
db.set(`stream/key${i % 300}`, { round: Math.floor(i / 300), i });
|
|
127
|
+
}
|
|
128
|
+
await new Promise(r => setTimeout(r, 50));
|
|
129
|
+
|
|
130
|
+
const repl = db.get('_repl') as Record<string, any>;
|
|
131
|
+
const count = repl ? Object.keys(repl).length : 0;
|
|
132
|
+
console.log(` 10k writes (300 unique paths), threshold=500, maxCount=200: ${count} _repl entries`);
|
|
133
|
+
// Without compact: 10k. With compact every 500 writes keeping 200: should be way under 1000.
|
|
134
|
+
expect(count).toBeLessThan(1000);
|
|
135
|
+
}, 15_000);
|
|
136
|
+
|
|
137
|
+
// ─── 4. Replica bootstrap with 10k entries via cursor ───
|
|
138
|
+
|
|
139
|
+
it('replica bootstraps 10k entries via cursor without timeout', async () => {
|
|
140
|
+
const { db: p, port: pp } = primary();
|
|
141
|
+
for (let i = 0; i < 10_000; i++) {
|
|
142
|
+
p.set(`data/node${i}`, { value: i, name: `node-${i}`, tags: ['a', 'b'] });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const { db: r } = replica(pp);
|
|
146
|
+
const t0 = Date.now();
|
|
147
|
+
await r.replication!.start();
|
|
148
|
+
const elapsed = Date.now() - t0;
|
|
149
|
+
|
|
150
|
+
console.log(` 10k entry replica bootstrap: ${elapsed}ms`);
|
|
151
|
+
|
|
152
|
+
await wait(500);
|
|
153
|
+
// Spot-check
|
|
154
|
+
for (const idx of [0, 999, 5000, 9999]) {
|
|
155
|
+
const val = r.get(`data/node${idx}`) as any;
|
|
156
|
+
expect(val?.value).toBe(idx);
|
|
157
|
+
}
|
|
158
|
+
}, 30_000);
|
|
159
|
+
|
|
160
|
+
// ─── 5. Multiple replicas bootstrap concurrently ───
|
|
161
|
+
|
|
162
|
+
it('3 replicas bootstrap concurrently from same primary (10k entries)', async () => {
|
|
163
|
+
const { db: p, port: pp } = primary();
|
|
164
|
+
for (let i = 0; i < 10_000; i++) {
|
|
165
|
+
p.set(`shared/item${i}`, { v: i });
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const replicas = [replica(pp), replica(pp), replica(pp)];
|
|
169
|
+
const t0 = Date.now();
|
|
170
|
+
await Promise.all(replicas.map(r => r.db.replication!.start()));
|
|
171
|
+
const elapsed = Date.now() - t0;
|
|
172
|
+
|
|
173
|
+
console.log(` 3 concurrent replica bootstraps (10k): ${elapsed}ms`);
|
|
174
|
+
|
|
175
|
+
await wait(500);
|
|
176
|
+
for (const r of replicas) {
|
|
177
|
+
const v0 = r.db.get('shared/item0') as any;
|
|
178
|
+
const v9999 = r.db.get('shared/item9999') as any;
|
|
179
|
+
expect(v0?.v).toBe(0);
|
|
180
|
+
expect(v9999?.v).toBe(9999);
|
|
181
|
+
}
|
|
182
|
+
}, 45_000);
|
|
183
|
+
|
|
184
|
+
// ─── 6. Writes during bootstrap: replica catches up via stream sub ───
|
|
185
|
+
|
|
186
|
+
it('writes during bootstrap are caught via ongoing stream subscription', async () => {
|
|
187
|
+
const { db: p, port: pp } = primary();
|
|
188
|
+
// Pre-fill
|
|
189
|
+
for (let i = 0; i < 5000; i++) {
|
|
190
|
+
p.set(`pre/item${i}`, { v: i });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const { db: r } = replica(pp);
|
|
194
|
+
// Start replica (bootstrap starts)
|
|
195
|
+
const startPromise = r.replication!.start();
|
|
196
|
+
|
|
197
|
+
// Write more to primary while bootstrap is in progress
|
|
198
|
+
for (let i = 0; i < 500; i++) {
|
|
199
|
+
p.set(`live/item${i}`, { v: i + 100_000 });
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
await startPromise;
|
|
203
|
+
// Give stream sub time to deliver live writes
|
|
204
|
+
await wait(1000);
|
|
205
|
+
|
|
206
|
+
// Pre-fill data should be there
|
|
207
|
+
const pre0 = r.get('pre/item0') as any;
|
|
208
|
+
expect(pre0?.v).toBe(0);
|
|
209
|
+
const pre4999 = r.get('pre/item4999') as any;
|
|
210
|
+
expect(pre4999?.v).toBe(4999);
|
|
211
|
+
|
|
212
|
+
// Live writes should eventually arrive
|
|
213
|
+
const live499 = r.get('live/item499') as any;
|
|
214
|
+
expect(live499?.v).toBe(100_499);
|
|
215
|
+
}, 30_000);
|
|
216
|
+
|
|
217
|
+
// ─── 7. Heavy overwrite scenario: same 50 paths written 1000× each ───
|
|
218
|
+
|
|
219
|
+
it('50 paths × 1000 overwrites: compact deduplicates correctly', async () => {
|
|
220
|
+
const { db } = primary({ compact: { maxCount: 100, keepKey: 'path' }, autoCompactThreshold: 1000 });
|
|
221
|
+
|
|
222
|
+
for (let round = 0; round < 1000; round++) {
|
|
223
|
+
for (let i = 0; i < 50; i++) {
|
|
224
|
+
db.set(`hot/key${i}`, { round, value: round * 50 + i });
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
await new Promise(r => setTimeout(r, 50));
|
|
228
|
+
|
|
229
|
+
const repl = db.get('_repl') as Record<string, any>;
|
|
230
|
+
const count = repl ? Object.keys(repl).length : 0;
|
|
231
|
+
console.log(` 50×1000 overwrites: ${count} _repl entries (expect ≤ ~1100)`);
|
|
232
|
+
// 50k total writes. Compact every 1k writes keeping 100. Should be bounded.
|
|
233
|
+
expect(count).toBeLessThan(1500);
|
|
234
|
+
|
|
235
|
+
// Verify latest values survived compaction
|
|
236
|
+
const materialized = db.stream.materialize('_repl', { keepKey: 'path' });
|
|
237
|
+
const paths = Object.keys(materialized);
|
|
238
|
+
expect(paths.length).toBe(50);
|
|
239
|
+
}, 30_000);
|
|
240
|
+
|
|
241
|
+
// ─── 8. Cursor pagination with snapshot (post-compact) ───
|
|
242
|
+
|
|
243
|
+
it('cursor pagination works correctly after compaction (snapshot + live events)', async () => {
|
|
244
|
+
const { db, port: p } = primary();
|
|
245
|
+
// Write 2000, compact to 200, write 800 more
|
|
246
|
+
for (let i = 0; i < 2000; i++) db.set(`a/item${i}`, { v: i });
|
|
247
|
+
db.stream.compact('_repl', { maxCount: 200, keepKey: 'path' });
|
|
248
|
+
for (let i = 2000; i < 2800; i++) db.set(`a/item${i}`, { v: i });
|
|
249
|
+
|
|
250
|
+
const client = connect(p);
|
|
251
|
+
await client.connect();
|
|
252
|
+
|
|
253
|
+
const keys = new Set<string>();
|
|
254
|
+
let cursor: string | undefined;
|
|
255
|
+
let pages = 0;
|
|
256
|
+
do {
|
|
257
|
+
const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 100, cursor });
|
|
258
|
+
for (const k of Object.keys(page.data)) keys.add(k);
|
|
259
|
+
cursor = page.nextCursor;
|
|
260
|
+
pages++;
|
|
261
|
+
} while (cursor);
|
|
262
|
+
|
|
263
|
+
console.log(` 2000 + compact + 800 more: ${keys.size} keys in ${pages} pages`);
|
|
264
|
+
// All 2800 unique paths should be present (snapshot has older ones, events have newer)
|
|
265
|
+
expect(keys.size).toBe(2800);
|
|
266
|
+
}, 15_000);
|
|
267
|
+
|
|
268
|
+
// ─── 9. Replica with auto-compact primary: data integrity ───
|
|
269
|
+
|
|
270
|
+
it('replica gets correct data when primary auto-compacts during heavy writes', async () => {
|
|
271
|
+
const { db: p, port: pp } = primary({ compact: { maxCount: 300, keepKey: 'path' }, autoCompactThreshold: 500 });
|
|
272
|
+
|
|
273
|
+
// 5000 writes — auto-compact fires multiple times
|
|
274
|
+
for (let i = 0; i < 5000; i++) {
|
|
275
|
+
p.set(`verified/item${i}`, { value: i * 7, tag: 'check' });
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
const { db: r } = replica(pp);
|
|
279
|
+
await r.replication!.start();
|
|
280
|
+
await wait(500);
|
|
281
|
+
|
|
282
|
+
// Exhaustive integrity check on a sample
|
|
283
|
+
const sample = [0, 100, 999, 2500, 4000, 4999];
|
|
284
|
+
for (const idx of sample) {
|
|
285
|
+
const val = r.get(`verified/item${idx}`) as any;
|
|
286
|
+
expect(val?.value).toBe(idx * 7);
|
|
287
|
+
expect(val?.tag).toBe('check');
|
|
288
|
+
}
|
|
289
|
+
}, 30_000);
|
|
290
|
+
|
|
291
|
+
// ─── 10. Mixed deletes + sets under load ───
|
|
292
|
+
|
|
293
|
+
it('deletes replicate correctly through cursor-based bootstrap (no fullBootstrap)', async () => {
|
|
294
|
+
const { db: p, port: pp } = primary();
|
|
295
|
+
|
|
296
|
+
// Create 1000, then delete half
|
|
297
|
+
for (let i = 0; i < 1000; i++) {
|
|
298
|
+
p.set(`mix/item${i}`, { v: i });
|
|
299
|
+
}
|
|
300
|
+
for (let i = 0; i < 1000; i += 2) {
|
|
301
|
+
p.delete(`mix/item${i}`);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// Disable fullBootstrap so only _repl stream materialize is used
|
|
305
|
+
const { db: r } = replica(pp, { fullBootstrap: false });
|
|
306
|
+
await r.replication!.start();
|
|
307
|
+
await wait(500);
|
|
308
|
+
|
|
309
|
+
// Even indices: _repl has set then delete — materialize with keepKey=path keeps last op (delete)
|
|
310
|
+
// But materialize folds by keepKey, and delete events have op:'delete' — they apply as db.delete()
|
|
311
|
+
// Odd indices should exist from set events
|
|
312
|
+
for (const i of [1, 3, 99, 999]) {
|
|
313
|
+
const val = r.get(`mix/item${i}`) as any;
|
|
314
|
+
expect(val?.v).toBe(i);
|
|
315
|
+
}
|
|
316
|
+
// Verify primary has them deleted
|
|
317
|
+
for (const i of [0, 2, 100, 998]) {
|
|
318
|
+
expect(p.get(`mix/item${i}`)).toBeNull();
|
|
319
|
+
}
|
|
320
|
+
}, 15_000);
|
|
321
|
+
|
|
322
|
+
// ─── 11. Rapid batchSize=1 pagination (worst case) ───
|
|
323
|
+
|
|
324
|
+
it('batchSize=1 pagination still completes for 500 entries', async () => {
|
|
325
|
+
const { db, port: p } = primary();
|
|
326
|
+
for (let i = 0; i < 500; i++) db.set(`tiny/k${i}`, { i });
|
|
327
|
+
|
|
328
|
+
const client = connect(p);
|
|
329
|
+
await client.connect();
|
|
330
|
+
|
|
331
|
+
const keys = new Set<string>();
|
|
332
|
+
let cursor: string | undefined;
|
|
333
|
+
let pages = 0;
|
|
334
|
+
const t0 = Date.now();
|
|
335
|
+
do {
|
|
336
|
+
const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 1, cursor });
|
|
337
|
+
for (const k of Object.keys(page.data)) keys.add(k);
|
|
338
|
+
cursor = page.nextCursor;
|
|
339
|
+
pages++;
|
|
340
|
+
} while (cursor);
|
|
341
|
+
const elapsed = Date.now() - t0;
|
|
342
|
+
|
|
343
|
+
console.log(` batchSize=1 over 500 entries: ${pages} pages, ${elapsed}ms`);
|
|
344
|
+
expect(keys.size).toBe(500);
|
|
345
|
+
expect(pages).toBeGreaterThanOrEqual(500);
|
|
346
|
+
}, 30_000);
|
|
347
|
+
|
|
348
|
+
// ─── 12. Throughput benchmark: writes/sec with auto-compact ───
|
|
349
|
+
|
|
350
|
+
it('write throughput with auto-compact enabled', () => {
|
|
351
|
+
const { db: dbCompact } = primary({ compact: { maxCount: 200, keepKey: 'path' }, autoCompactThreshold: 500 });
|
|
352
|
+
const { db: dbPlain } = primary();
|
|
353
|
+
|
|
354
|
+
const N = 10_000;
|
|
355
|
+
|
|
356
|
+
const t0 = Date.now();
|
|
357
|
+
for (let i = 0; i < N; i++) dbPlain.set(`bench/k${i}`, { i });
|
|
358
|
+
const plainMs = Date.now() - t0;
|
|
359
|
+
|
|
360
|
+
const t1 = Date.now();
|
|
361
|
+
for (let i = 0; i < N; i++) dbCompact.set(`bench/k${i}`, { i });
|
|
362
|
+
const compactMs = Date.now() - t1;
|
|
363
|
+
|
|
364
|
+
const plainWps = Math.round(N / (plainMs / 1000));
|
|
365
|
+
const compactWps = Math.round(N / (compactMs / 1000));
|
|
366
|
+
const overhead = ((compactMs - plainMs) / plainMs * 100).toFixed(1);
|
|
367
|
+
|
|
368
|
+
console.log(` ${N} writes — plain: ${plainMs}ms (${plainWps} w/s), compact: ${compactMs}ms (${compactWps} w/s), overhead: ${overhead}%`);
|
|
369
|
+
// Auto-compact overhead should be < 100% (compact is cheap relative to N writes)
|
|
370
|
+
expect(compactMs).toBeLessThan(plainMs * 3);
|
|
371
|
+
}, 30_000);
|
|
372
|
+
});
|
|
@@ -5,18 +5,6 @@ import { BodClient } from '../src/client/BodClient.ts';
|
|
|
5
5
|
const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
|
|
6
6
|
let nextPort = 26400 + Math.floor(Math.random() * 1000);
|
|
7
7
|
|
|
8
|
-
/**
|
|
9
|
-
* Stress tests for _repl stream bloat.
|
|
10
|
-
*
|
|
11
|
-
* The real-world issue: _repl grows unbounded → streamMaterialize produces
|
|
12
|
-
* a massive WS response → client requestTimeout (30s) fires or WS chokes.
|
|
13
|
-
*
|
|
14
|
-
* Locally we can't easily reproduce network latency, but we CAN:
|
|
15
|
-
* 1. Push the entry count + payload size to stress serialization/parsing
|
|
16
|
-
* 2. Use short requestTimeout on the replica client to simulate the real failure
|
|
17
|
-
* 3. Measure materialization time scaling (O(n) proof)
|
|
18
|
-
* 4. Verify compaction actually fixes it
|
|
19
|
-
*/
|
|
20
8
|
describe('_repl stream bloat', () => {
|
|
21
9
|
const instances: BodDB[] = [];
|
|
22
10
|
const clients: BodClient[] = [];
|
|
@@ -31,12 +19,12 @@ describe('_repl stream bloat', () => {
|
|
|
31
19
|
function getPort() { return nextPort++; }
|
|
32
20
|
|
|
33
21
|
/** Primary with auto-compact DISABLED so _repl grows unbounded */
|
|
34
|
-
function createPrimary(opts?: { maxMessageSize?: number }) {
|
|
22
|
+
function createPrimary(opts?: { maxMessageSize?: number; compact?: any; autoCompactThreshold?: number }) {
|
|
35
23
|
const port = getPort();
|
|
36
24
|
const db = new BodDB({
|
|
37
25
|
path: ':memory:',
|
|
38
26
|
sweepInterval: 0,
|
|
39
|
-
replication: { role: 'primary', compact: {} },
|
|
27
|
+
replication: { role: 'primary', compact: opts?.compact ?? {}, autoCompactThreshold: opts?.autoCompactThreshold ?? 0 },
|
|
40
28
|
});
|
|
41
29
|
db.replication!.start();
|
|
42
30
|
db.serve({ port, maxMessageSize: opts?.maxMessageSize });
|
|
@@ -78,9 +66,10 @@ describe('_repl stream bloat', () => {
|
|
|
78
66
|
|
|
79
67
|
// --- Accumulation ---
|
|
80
68
|
|
|
81
|
-
it('_repl grows unbounded without compaction', () => {
|
|
69
|
+
it('_repl grows unbounded without compaction', async () => {
|
|
82
70
|
const { db } = createPrimary();
|
|
83
71
|
fillRepl(db, 5000, 300);
|
|
72
|
+
await new Promise(r => setTimeout(r, 50));
|
|
84
73
|
const repl = db.get('_repl') as Record<string, any>;
|
|
85
74
|
expect(Object.keys(repl).length).toBe(5000);
|
|
86
75
|
});
|
|
@@ -117,17 +106,62 @@ describe('_repl stream bloat', () => {
|
|
|
117
106
|
expect(ratio).toBeGreaterThan(5); // 5000/500 = 10x, allow some dedup
|
|
118
107
|
});
|
|
119
108
|
|
|
109
|
+
// --- Cursor-based materialize ---
|
|
110
|
+
|
|
111
|
+
it('cursor-based materialize pages correctly over large _repl', async () => {
|
|
112
|
+
const { db, port } = createPrimary();
|
|
113
|
+
fillRepl(db, 1000, 200);
|
|
114
|
+
|
|
115
|
+
const client = new BodClient({ url: `ws://localhost:${port}` });
|
|
116
|
+
clients.push(client);
|
|
117
|
+
await client.connect();
|
|
118
|
+
|
|
119
|
+
// Page through with batchSize=200
|
|
120
|
+
const allKeys = new Set<string>();
|
|
121
|
+
let cursor: string | undefined;
|
|
122
|
+
let pages = 0;
|
|
123
|
+
do {
|
|
124
|
+
const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 200, cursor });
|
|
125
|
+
if (page.data) {
|
|
126
|
+
for (const key of Object.keys(page.data)) allKeys.add(key);
|
|
127
|
+
}
|
|
128
|
+
cursor = page.nextCursor;
|
|
129
|
+
pages++;
|
|
130
|
+
} while (cursor);
|
|
131
|
+
|
|
132
|
+
console.log(` 1000 entries paged in ${pages} pages, got ${allKeys.size} unique keys`);
|
|
133
|
+
expect(allKeys.size).toBe(1000);
|
|
134
|
+
expect(pages).toBeGreaterThanOrEqual(5); // 1000/200 = 5 pages minimum
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
// --- Auto-compact on write threshold ---
|
|
138
|
+
|
|
139
|
+
it('auto-compact triggers after N writes', async () => {
|
|
140
|
+
const { db } = createPrimary({ compact: { maxCount: 50, keepKey: 'path' }, autoCompactThreshold: 100 });
|
|
141
|
+
|
|
142
|
+
// Write 250 entries — compact triggers at 100, 200; maxCount=50 keeps only 50 each time
|
|
143
|
+
for (let i = 0; i < 250; i++) {
|
|
144
|
+
db.set(`data/item${i}`, { value: i });
|
|
145
|
+
}
|
|
146
|
+
await new Promise(r => setTimeout(r, 50));
|
|
147
|
+
|
|
148
|
+
const repl = db.get('_repl') as Record<string, any>;
|
|
149
|
+
const count = repl ? Object.keys(repl).length : 0;
|
|
150
|
+
console.log(` 250 writes with threshold=100, maxCount=50: ${count} _repl entries`);
|
|
151
|
+
// After compact at 200 (keeps 50), then 50 more → ~100. Way less than 250.
|
|
152
|
+
expect(count).toBeLessThan(150);
|
|
153
|
+
});
|
|
154
|
+
|
|
120
155
|
// --- Timeout reproduction: short requestTimeout simulates real-world failure ---
|
|
121
156
|
|
|
122
157
|
it('short requestTimeout causes streamMaterialize to fail on bloated _repl', async () => {
|
|
123
158
|
const { db: primary, port: primaryPort } = createPrimary();
|
|
124
159
|
fillRepl(primary, 10000, 1000); // 10k entries × 1KB = ~10MB response
|
|
160
|
+
await new Promise(r => setTimeout(r, 50));
|
|
125
161
|
|
|
126
162
|
const replCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
|
|
127
163
|
expect(replCount).toBe(10000);
|
|
128
164
|
|
|
129
|
-
// Direct client with 1ms timeout — guaranteed to fail, proving the
|
|
130
|
-
// timeout path exists and that materialize has no retry/fallback
|
|
131
165
|
const client = new BodClient({
|
|
132
166
|
url: `ws://localhost:${primaryPort}`,
|
|
133
167
|
requestTimeout: 1,
|
|
@@ -161,16 +195,14 @@ describe('_repl stream bloat', () => {
|
|
|
161
195
|
const responseSize = JSON.stringify(result).length;
|
|
162
196
|
|
|
163
197
|
console.log(` 10k entries × 1KB: ${(responseSize / 1024 / 1024).toFixed(1)}MB response in ${elapsed}ms`);
|
|
164
|
-
// On real networks with latency, this 10MB+ response would easily exceed 30s timeout
|
|
165
198
|
expect(responseSize).toBeGreaterThan(5 * 1024 * 1024); // >5MB
|
|
166
199
|
});
|
|
167
200
|
|
|
168
|
-
// --- Payload size bomb
|
|
201
|
+
// --- Payload size bomb ---
|
|
169
202
|
|
|
170
203
|
it('large payloads per entry amplify the problem', async () => {
|
|
171
204
|
const { db, port } = createPrimary();
|
|
172
205
|
|
|
173
|
-
// 1000 entries but 2KB each → ~2MB+ materialize response
|
|
174
206
|
const bigPadding = 'y'.repeat(2000);
|
|
175
207
|
for (let i = 0; i < 1000; i++) {
|
|
176
208
|
db.set(`data/big${i}`, {
|
|
@@ -190,7 +222,7 @@ describe('_repl stream bloat', () => {
|
|
|
190
222
|
const responseSize = JSON.stringify(result).length;
|
|
191
223
|
|
|
192
224
|
console.log(` 1000 entries × 2KB = ${(responseSize / 1024 / 1024).toFixed(2)}MB, ${elapsed}ms`);
|
|
193
|
-
expect(responseSize).toBeGreaterThan(2 * 1024 * 1024);
|
|
225
|
+
expect(responseSize).toBeGreaterThan(2 * 1024 * 1024);
|
|
194
226
|
});
|
|
195
227
|
|
|
196
228
|
// --- Compaction fixes it ---
|
|
@@ -198,18 +230,17 @@ describe('_repl stream bloat', () => {
|
|
|
198
230
|
it('compaction reduces _repl and speeds up bootstrap', async () => {
|
|
199
231
|
const { db: primary, port: primaryPort } = createPrimary();
|
|
200
232
|
fillRepl(primary, 5000, 500);
|
|
233
|
+
await new Promise(r => setTimeout(r, 50));
|
|
201
234
|
|
|
202
235
|
const beforeCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
|
|
203
236
|
expect(beforeCount).toBe(5000);
|
|
204
237
|
|
|
205
|
-
// Compact down to 500
|
|
206
238
|
primary.stream.compact('_repl', { maxCount: 500, keepKey: 'path' });
|
|
207
239
|
const afterRepl = primary.get('_repl') as Record<string, any>;
|
|
208
240
|
const afterCount = afterRepl ? Object.keys(afterRepl).length : 0;
|
|
209
241
|
console.log(` Compacted: ${beforeCount} → ${afterCount} entries`);
|
|
210
242
|
expect(afterCount).toBeLessThanOrEqual(500);
|
|
211
243
|
|
|
212
|
-
// Bootstrap should now work fast with compacted stream
|
|
213
244
|
const { db: replica } = createReplica(primaryPort);
|
|
214
245
|
const start = Date.now();
|
|
215
246
|
await replica.replication!.start();
|
|
@@ -219,52 +250,51 @@ describe('_repl stream bloat', () => {
|
|
|
219
250
|
expect(elapsed).toBeLessThan(3000);
|
|
220
251
|
|
|
221
252
|
await wait(300);
|
|
222
|
-
// Verify latest writes are present (compaction keeps newest by keepKey)
|
|
223
253
|
const val = replica.get('vfs/files/project/src/deep/nested/path/module4999/component.tsx') as any;
|
|
224
254
|
expect(val?.size).toBe(1024 + 4999);
|
|
225
255
|
});
|
|
226
256
|
|
|
227
|
-
// --- Repeated writes
|
|
257
|
+
// --- Repeated writes ---
|
|
228
258
|
|
|
229
|
-
it('repeated writes to same paths bloat _repl with duplicates', () => {
|
|
259
|
+
it('repeated writes to same paths bloat _repl with duplicates', async () => {
|
|
230
260
|
const { db } = createPrimary();
|
|
231
261
|
|
|
232
|
-
// 100 paths × 50 writes each = 5000 _repl entries, but only 100 unique paths
|
|
233
262
|
for (let round = 0; round < 50; round++) {
|
|
234
263
|
for (let i = 0; i < 100; i++) {
|
|
235
264
|
db.set(`config/setting${i}`, { value: round, updated: Date.now() });
|
|
236
265
|
}
|
|
237
266
|
}
|
|
267
|
+
await new Promise(r => setTimeout(r, 50));
|
|
238
268
|
|
|
239
269
|
const repl = db.get('_repl') as Record<string, any>;
|
|
240
270
|
const totalEntries = Object.keys(repl).length;
|
|
241
271
|
console.log(` 100 paths × 50 writes = ${totalEntries} _repl entries`);
|
|
242
272
|
expect(totalEntries).toBe(5000);
|
|
243
273
|
|
|
244
|
-
// Compact with keepKey deduplicates to 100
|
|
245
274
|
db.stream.compact('_repl', { keepKey: 'path' });
|
|
246
275
|
const after = db.get('_repl') as Record<string, any>;
|
|
247
276
|
const afterCount = after ? Object.keys(after).length : 0;
|
|
248
|
-
// snapshot (1) + remaining entries
|
|
249
277
|
console.log(` After compact: ${afterCount} entries (expect ~100 unique paths)`);
|
|
250
278
|
expect(afterCount).toBeLessThanOrEqual(150);
|
|
251
279
|
});
|
|
252
280
|
|
|
253
|
-
// ---
|
|
281
|
+
// --- Cursor-based bootstrap works for replica ---
|
|
254
282
|
|
|
255
|
-
it('replica
|
|
283
|
+
it('replica bootstrap uses cursor-based pagination (no timeout on large _repl)', async () => {
|
|
256
284
|
const { db: primary, port: primaryPort } = createPrimary();
|
|
257
285
|
fillRepl(primary, 3000, 300);
|
|
258
286
|
|
|
259
287
|
const { db: replica } = createReplica(primaryPort);
|
|
260
288
|
|
|
261
|
-
// Measure: start() blocks until materialize completes — no internal timeout
|
|
262
289
|
const start = Date.now();
|
|
263
290
|
await replica.replication!.start();
|
|
264
291
|
const elapsed = Date.now() - start;
|
|
265
292
|
|
|
266
|
-
console.log(` replica
|
|
267
|
-
|
|
268
|
-
|
|
293
|
+
console.log(` Cursor-based replica bootstrap: ${elapsed}ms for 3000 entries`);
|
|
294
|
+
|
|
295
|
+
await wait(300);
|
|
296
|
+
// Verify data arrived
|
|
297
|
+
const val = replica.get('vfs/files/project/src/deep/nested/path/module2999/component.tsx') as any;
|
|
298
|
+
expect(val?.size).toBe(1024 + 2999);
|
|
269
299
|
});
|
|
270
300
|
});
|