@bod.ee/db 0.10.2 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +5 -1
- package/.claude/skills/config-file.md +3 -1
- package/.claude/skills/developing-bod-db.md +10 -1
- package/.claude/skills/using-bod-db.md +18 -1
- package/CLAUDE.md +12 -3
- package/admin/admin.ts +9 -0
- package/admin/demo.config.ts +1 -0
- package/admin/ui.html +67 -37
- package/docs/keyauth-integration.md +141 -0
- package/index.ts +2 -0
- package/package.json +1 -1
- package/src/client/BodClient.ts +30 -6
- package/src/client/BodClientCached.ts +91 -3
- package/src/server/BodDB.ts +113 -42
- package/src/server/KeyAuthEngine.ts +32 -7
- package/src/server/ReplicationEngine.ts +31 -1
- package/src/server/StorageEngine.ts +117 -6
- package/src/server/StreamEngine.ts +20 -2
- package/src/server/SubscriptionEngine.ts +122 -35
- package/src/server/Transport.ts +125 -15
- package/src/server/VFSEngine.ts +27 -0
- package/src/shared/keyAuth.ts +4 -4
- package/src/shared/logger.ts +68 -0
- package/src/shared/protocol.ts +4 -1
- package/tests/optimization.test.ts +392 -0
package/src/client/BodClient.ts
CHANGED
|
@@ -15,6 +15,8 @@ export class BodClientOptions {
|
|
|
15
15
|
reconnect: boolean = true;
|
|
16
16
|
reconnectInterval: number = 1000;
|
|
17
17
|
maxReconnectInterval: number = 30000;
|
|
18
|
+
/** Timeout for individual requests in ms (0 = no timeout) */
|
|
19
|
+
requestTimeout: number = 30000;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
/** Interface for persisting device keys. Default: localStorage (browser) or in-memory (Node). */
|
|
@@ -115,11 +117,18 @@ export class BodClient {
|
|
|
115
117
|
const token = await this.options.auth();
|
|
116
118
|
await this.send('auth', { token });
|
|
117
119
|
}
|
|
118
|
-
// Re-subscribe all active subscriptions
|
|
119
|
-
|
|
120
|
-
const [
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
// Re-subscribe all active subscriptions (batch)
|
|
121
|
+
if (this.activeSubs.size > 0) {
|
|
122
|
+
const subs = [...this.activeSubs].map(key => {
|
|
123
|
+
const [event, ...pathParts] = key.split(':');
|
|
124
|
+
return { path: pathParts.join(':'), event };
|
|
125
|
+
});
|
|
126
|
+
try {
|
|
127
|
+
await this.send('batch-sub', { subscriptions: subs });
|
|
128
|
+
} catch {
|
|
129
|
+
// Fallback to individual subs if server doesn't support batch-sub
|
|
130
|
+
for (const sub of subs) await this.send('sub', sub);
|
|
131
|
+
}
|
|
123
132
|
}
|
|
124
133
|
// Re-subscribe stream subs
|
|
125
134
|
for (const key of this.activeStreamSubs) {
|
|
@@ -239,7 +248,18 @@ export class BodClient {
|
|
|
239
248
|
return reject(new Error('Not connected'));
|
|
240
249
|
}
|
|
241
250
|
const id = this.nextId();
|
|
242
|
-
|
|
251
|
+
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
252
|
+
const cleanup = () => { if (timer) { clearTimeout(timer); timer = null; } };
|
|
253
|
+
this.pending.set(id, {
|
|
254
|
+
resolve: (v: unknown) => { cleanup(); resolve(v as Record<string, unknown>); },
|
|
255
|
+
reject: (e: Error) => { cleanup(); reject(e); },
|
|
256
|
+
});
|
|
257
|
+
if (this.options.requestTimeout > 0) {
|
|
258
|
+
timer = setTimeout(() => {
|
|
259
|
+
this.pending.delete(id);
|
|
260
|
+
reject(new Error(`Request timeout after ${this.options.requestTimeout}ms`));
|
|
261
|
+
}, this.options.requestTimeout);
|
|
262
|
+
}
|
|
243
263
|
this.ws.send(JSON.stringify({ id, op, ...params }));
|
|
244
264
|
});
|
|
245
265
|
}
|
|
@@ -809,6 +829,10 @@ export class VFSClient {
|
|
|
809
829
|
return await this.client._send('vfs-list', { path }) as FileStat[];
|
|
810
830
|
}
|
|
811
831
|
|
|
832
|
+
async tree(path: string, opts?: { hiddenPaths?: string[]; hideDotfiles?: boolean }): Promise<any[]> {
|
|
833
|
+
return await this.client._send('vfs-tree', { path, ...opts }) as any[];
|
|
834
|
+
}
|
|
835
|
+
|
|
812
836
|
async delete(path: string): Promise<void> {
|
|
813
837
|
await this.client._send('vfs-delete', { path });
|
|
814
838
|
}
|
|
@@ -182,7 +182,7 @@ export class BodClientCached {
|
|
|
182
182
|
|
|
183
183
|
vfs(): CachedVFSClient {
|
|
184
184
|
if (!this._vfs) {
|
|
185
|
-
this._vfs = new CachedVFSClient(this.client.vfs(), this.options.maxAge);
|
|
185
|
+
this._vfs = new CachedVFSClient(this.client.vfs(), this.options.maxAge, this.idb);
|
|
186
186
|
this._vfsUnsub = this.client.onChild('_vfs', () => {
|
|
187
187
|
// _vfs events are coarse (top-level key only), clear all VFS caches
|
|
188
188
|
this._vfs?.clear();
|
|
@@ -252,18 +252,34 @@ export class CachedVFSClient {
|
|
|
252
252
|
constructor(
|
|
253
253
|
private raw: VFSClient,
|
|
254
254
|
private maxAge: number,
|
|
255
|
+
private idb: IDBDatabase | null = null,
|
|
255
256
|
) {}
|
|
256
257
|
|
|
257
258
|
async stat(path: string): Promise<FileStat | null> {
|
|
258
259
|
const cached = this.statCache.get(path);
|
|
259
260
|
if (cached && Date.now() - cached.cachedAt < this.maxAge) {
|
|
260
261
|
this._stats.hits++;
|
|
261
|
-
this.raw.stat(path).then(r =>
|
|
262
|
+
this.raw.stat(path).then(r => {
|
|
263
|
+
this.statCache.set(path, { data: r, cachedAt: Date.now() });
|
|
264
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: r, cachedAt: Date.now() });
|
|
265
|
+
}).catch(() => {});
|
|
262
266
|
return cached.data;
|
|
263
267
|
}
|
|
268
|
+
// Check IDB before network
|
|
269
|
+
const idbEntry = await this.idbGetVfs(`vfs:stat:${path}`);
|
|
270
|
+
if (idbEntry && Date.now() - idbEntry.cachedAt < this.maxAge) {
|
|
271
|
+
this._stats.hits++;
|
|
272
|
+
this.statCache.set(path, idbEntry);
|
|
273
|
+
this.raw.stat(path).then(r => {
|
|
274
|
+
this.statCache.set(path, { data: r, cachedAt: Date.now() });
|
|
275
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: r, cachedAt: Date.now() });
|
|
276
|
+
}).catch(() => {});
|
|
277
|
+
return idbEntry.data as FileStat | null;
|
|
278
|
+
}
|
|
264
279
|
this._stats.misses++;
|
|
265
280
|
const result = await this.raw.stat(path);
|
|
266
281
|
this.statCache.set(path, { data: result, cachedAt: Date.now() });
|
|
282
|
+
this.idbSetVfs(`vfs:stat:${path}`, { data: result, cachedAt: Date.now() });
|
|
267
283
|
return result;
|
|
268
284
|
}
|
|
269
285
|
|
|
@@ -271,15 +287,34 @@ export class CachedVFSClient {
|
|
|
271
287
|
const cached = this.listCache.get(path);
|
|
272
288
|
if (cached && Date.now() - cached.cachedAt < this.maxAge) {
|
|
273
289
|
this._stats.hits++;
|
|
274
|
-
this.raw.list(path).then(r =>
|
|
290
|
+
this.raw.list(path).then(r => {
|
|
291
|
+
this.listCache.set(path, { data: r, cachedAt: Date.now() });
|
|
292
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: r, cachedAt: Date.now() });
|
|
293
|
+
}).catch(() => {});
|
|
275
294
|
return cached.data;
|
|
276
295
|
}
|
|
296
|
+
// Check IDB before network
|
|
297
|
+
const idbEntry = await this.idbGetVfs(`vfs:list:${path}`);
|
|
298
|
+
if (idbEntry && Date.now() - idbEntry.cachedAt < this.maxAge) {
|
|
299
|
+
this._stats.hits++;
|
|
300
|
+
this.listCache.set(path, idbEntry as { data: FileStat[]; cachedAt: number });
|
|
301
|
+
this.raw.list(path).then(r => {
|
|
302
|
+
this.listCache.set(path, { data: r, cachedAt: Date.now() });
|
|
303
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: r, cachedAt: Date.now() });
|
|
304
|
+
}).catch(() => {});
|
|
305
|
+
return idbEntry.data as FileStat[];
|
|
306
|
+
}
|
|
277
307
|
this._stats.misses++;
|
|
278
308
|
const result = await this.raw.list(path);
|
|
279
309
|
this.listCache.set(path, { data: result, cachedAt: Date.now() });
|
|
310
|
+
this.idbSetVfs(`vfs:list:${path}`, { data: result, cachedAt: Date.now() });
|
|
280
311
|
return result;
|
|
281
312
|
}
|
|
282
313
|
|
|
314
|
+
async tree(path: string, opts?: { hiddenPaths?: string[]; hideDotfiles?: boolean }): Promise<any[]> {
|
|
315
|
+
return this.raw.tree(path, opts);
|
|
316
|
+
}
|
|
317
|
+
|
|
283
318
|
async upload(path: string, data: Uint8Array, mime?: string): Promise<FileStat> {
|
|
284
319
|
const r = await this.raw.upload(path, data, mime);
|
|
285
320
|
this.invalidatePath(path);
|
|
@@ -296,6 +331,7 @@ export class CachedVFSClient {
|
|
|
296
331
|
this.invalidatePath(path);
|
|
297
332
|
// Also invalidate the dir's own list (was empty/nonexistent before)
|
|
298
333
|
this.listCache.delete(path);
|
|
334
|
+
this.idbDeleteVfs(`vfs:list:${path}`);
|
|
299
335
|
return r;
|
|
300
336
|
}
|
|
301
337
|
|
|
@@ -313,14 +349,17 @@ export class CachedVFSClient {
|
|
|
313
349
|
invalidatePath(filePath: string) {
|
|
314
350
|
this._stats.invalidations++;
|
|
315
351
|
this.statCache.delete(filePath);
|
|
352
|
+
this.idbDeleteVfs(`vfs:stat:${filePath}`);
|
|
316
353
|
const parent = filePath.includes('/') ? filePath.slice(0, filePath.lastIndexOf('/')) : '';
|
|
317
354
|
this.listCache.delete(parent);
|
|
355
|
+
this.idbDeleteVfs(`vfs:list:${parent}`);
|
|
318
356
|
}
|
|
319
357
|
|
|
320
358
|
clear() {
|
|
321
359
|
this._stats.pushClears++;
|
|
322
360
|
this.listCache.clear();
|
|
323
361
|
this.statCache.clear();
|
|
362
|
+
this.idbClearVfs();
|
|
324
363
|
}
|
|
325
364
|
|
|
326
365
|
/** Cache stats for diagnostics. Use in browser: `__bodCache.vfs().stats` */
|
|
@@ -334,4 +373,53 @@ export class CachedVFSClient {
|
|
|
334
373
|
statCacheSize: this.statCache.size,
|
|
335
374
|
};
|
|
336
375
|
}
|
|
376
|
+
|
|
377
|
+
// --- IDB helpers for VFS cache ---
|
|
378
|
+
|
|
379
|
+
private idbGetVfs(key: string): Promise<{ data: unknown; cachedAt: number } | null> {
|
|
380
|
+
if (!this.idb) return Promise.resolve(null);
|
|
381
|
+
try {
|
|
382
|
+
const tx = this.idb.transaction('entries', 'readonly');
|
|
383
|
+
const req = tx.objectStore('entries').get(key);
|
|
384
|
+
return new Promise(resolve => {
|
|
385
|
+
req.onsuccess = () => {
|
|
386
|
+
const entry = req.result;
|
|
387
|
+
resolve(entry ? { data: entry.data, cachedAt: entry.cachedAt } : null);
|
|
388
|
+
};
|
|
389
|
+
req.onerror = () => resolve(null);
|
|
390
|
+
});
|
|
391
|
+
} catch { return Promise.resolve(null); }
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
private idbSetVfs(key: string, value: { data: unknown; cachedAt: number }): void {
|
|
395
|
+
if (!this.idb) return;
|
|
396
|
+
try {
|
|
397
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
398
|
+
tx.objectStore('entries').put({ path: key, data: value.data, cachedAt: value.cachedAt, updatedAt: Date.now() });
|
|
399
|
+
} catch {}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
private idbDeleteVfs(key: string): void {
|
|
403
|
+
if (!this.idb) return;
|
|
404
|
+
try {
|
|
405
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
406
|
+
tx.objectStore('entries').delete(key);
|
|
407
|
+
} catch {}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
private idbClearVfs(): void {
|
|
411
|
+
if (!this.idb) return;
|
|
412
|
+
try {
|
|
413
|
+
// Delete all vfs: prefixed entries
|
|
414
|
+
const tx = this.idb.transaction('entries', 'readwrite');
|
|
415
|
+
const store = tx.objectStore('entries');
|
|
416
|
+
const req = store.openCursor();
|
|
417
|
+
req.onsuccess = () => {
|
|
418
|
+
const cursor = req.result;
|
|
419
|
+
if (!cursor) return;
|
|
420
|
+
if (typeof cursor.key === 'string' && cursor.key.startsWith('vfs:')) cursor.delete();
|
|
421
|
+
cursor.continue();
|
|
422
|
+
};
|
|
423
|
+
} catch {}
|
|
424
|
+
}
|
|
337
425
|
}
|
package/src/server/BodDB.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { ReplicationEngine, type ReplicationOptions, type WriteEvent } from './R
|
|
|
11
11
|
import { VFSEngine, type VFSEngineOptions } from './VFSEngine.ts';
|
|
12
12
|
import { KeyAuthEngine, type KeyAuthEngineOptions } from './KeyAuthEngine.ts';
|
|
13
13
|
import { validatePath } from '../shared/pathUtils.ts';
|
|
14
|
+
import { Logger, type LogConfig } from '../shared/logger.ts';
|
|
14
15
|
|
|
15
16
|
export interface TransactionProxy {
|
|
16
17
|
get(path: string): unknown;
|
|
@@ -46,6 +47,8 @@ export class BodDBOptions {
|
|
|
46
47
|
port?: number;
|
|
47
48
|
auth?: TransportOptions['auth'];
|
|
48
49
|
transport?: Partial<TransportOptions>;
|
|
50
|
+
/** Logging config — disabled by default */
|
|
51
|
+
log?: LogConfig;
|
|
49
52
|
}
|
|
50
53
|
|
|
51
54
|
export class BodDB {
|
|
@@ -59,6 +62,7 @@ export class BodDB {
|
|
|
59
62
|
readonly vfs: VFSEngine | null = null;
|
|
60
63
|
readonly keyAuth: KeyAuthEngine | null = null;
|
|
61
64
|
readonly options: BodDBOptions;
|
|
65
|
+
readonly log: Logger;
|
|
62
66
|
replication: ReplicationEngine | null = null;
|
|
63
67
|
private _replaying = false;
|
|
64
68
|
private _onWriteHooks: Array<(ev: WriteEvent) => void> = [];
|
|
@@ -66,11 +70,16 @@ export class BodDB {
|
|
|
66
70
|
get transport(): Transport | null { return this._transport; }
|
|
67
71
|
private sweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
68
72
|
private _statsInterval: ReturnType<typeof setInterval> | null = null;
|
|
73
|
+
private _statsStmtCount: any = null;
|
|
69
74
|
private _lastCpuUsage = process.cpuUsage();
|
|
70
75
|
private _lastCpuTime = performance.now();
|
|
76
|
+
private _lastStats: Record<string, unknown> | null = null;
|
|
71
77
|
|
|
72
78
|
constructor(options?: Partial<BodDBOptions>) {
|
|
73
79
|
this.options = { ...new BodDBOptions(), ...options };
|
|
80
|
+
this.log = new Logger(this.options.log);
|
|
81
|
+
const _log = this.log.forComponent('db');
|
|
82
|
+
_log.info(`Initializing BodDB (path: ${this.options.path})`);
|
|
74
83
|
this.storage = new StorageEngine({ path: this.options.path });
|
|
75
84
|
this.subs = new SubscriptionEngine();
|
|
76
85
|
this.stream = new StreamEngine(this.storage, this.subs, { compact: this.options.compact });
|
|
@@ -88,6 +97,7 @@ export class BodDB {
|
|
|
88
97
|
? BodDB.loadRulesFileSync(rulesConfig)
|
|
89
98
|
: (rulesConfig ?? {});
|
|
90
99
|
this.rules = new RulesEngine({ rules: resolvedRules, defaultDeny: this.options.defaultDeny });
|
|
100
|
+
_log.debug(`Rules loaded (${Object.keys(resolvedRules).length} paths, defaultDeny: ${!!this.options.defaultDeny})`);
|
|
91
101
|
|
|
92
102
|
// Auto-create indexes from config
|
|
93
103
|
if (this.options.indexes) {
|
|
@@ -96,21 +106,25 @@ export class BodDB {
|
|
|
96
106
|
this.storage.createIndex(basePath, field);
|
|
97
107
|
}
|
|
98
108
|
}
|
|
109
|
+
_log.debug(`Indexes created: ${JSON.stringify(this.options.indexes)}`);
|
|
99
110
|
}
|
|
100
111
|
|
|
101
112
|
// Initialize FTS if configured
|
|
102
113
|
if (this.options.fts) {
|
|
103
114
|
this.fts = new FTSEngine(this.storage.db, this.options.fts);
|
|
115
|
+
_log.info('FTS5 engine enabled');
|
|
104
116
|
}
|
|
105
117
|
|
|
106
118
|
// Initialize vectors if configured
|
|
107
119
|
if (this.options.vectors) {
|
|
108
120
|
this.vectors = new VectorEngine(this.storage.db, this.options.vectors);
|
|
121
|
+
_log.info(`Vector engine enabled (dimensions: ${this.options.vectors.dimensions})`);
|
|
109
122
|
}
|
|
110
123
|
|
|
111
124
|
// Initialize VFS if configured
|
|
112
125
|
if (this.options.vfs) {
|
|
113
126
|
(this as { vfs: VFSEngine }).vfs = new VFSEngine(this, this.options.vfs);
|
|
127
|
+
_log.info(`VFS enabled (root: ${this.options.vfs.storageRoot})`);
|
|
114
128
|
}
|
|
115
129
|
|
|
116
130
|
// Initialize KeyAuth if configured
|
|
@@ -119,11 +133,13 @@ export class BodDB {
|
|
|
119
133
|
if (this.options.keyAuth.rootPublicKey) {
|
|
120
134
|
this.keyAuth!.initRoot(this.options.keyAuth.rootPublicKey);
|
|
121
135
|
}
|
|
136
|
+
_log.info('KeyAuth engine enabled');
|
|
122
137
|
}
|
|
123
138
|
|
|
124
139
|
// Start TTL sweep
|
|
125
140
|
if (this.options.sweepInterval > 0) {
|
|
126
141
|
this.sweepTimer = setInterval(() => this.sweep(), this.options.sweepInterval);
|
|
142
|
+
_log.debug(`TTL sweep interval: ${this.options.sweepInterval}ms`);
|
|
127
143
|
}
|
|
128
144
|
|
|
129
145
|
// Init replication
|
|
@@ -134,7 +150,10 @@ export class BodDB {
|
|
|
134
150
|
const compactOpts = this.options.replication.compact ?? { keepKey: 'path', maxCount: 10000 };
|
|
135
151
|
this.stream.options.compact = { ...this.stream.options.compact, _repl: compactOpts };
|
|
136
152
|
}
|
|
153
|
+
_log.info(`Replication enabled (role: ${this.replication.isPrimary ? 'primary' : 'replica'})`);
|
|
137
154
|
}
|
|
155
|
+
|
|
156
|
+
_log.info('BodDB ready');
|
|
138
157
|
}
|
|
139
158
|
|
|
140
159
|
/** Load rules from a JSON or TS file synchronously */
|
|
@@ -195,6 +214,7 @@ export class BodDB {
|
|
|
195
214
|
}
|
|
196
215
|
|
|
197
216
|
get(path: string): unknown {
|
|
217
|
+
if (path === '_admin/stats' && this._lastStats) return this._lastStats;
|
|
198
218
|
return this.storage.get(path);
|
|
199
219
|
}
|
|
200
220
|
|
|
@@ -204,7 +224,7 @@ export class BodDB {
|
|
|
204
224
|
|
|
205
225
|
set(path: string, value: unknown, options?: { ttl?: number }): void {
|
|
206
226
|
const p = validatePath(path);
|
|
207
|
-
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) :
|
|
227
|
+
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
|
|
208
228
|
const changed = this.storage.set(path, value);
|
|
209
229
|
if (options?.ttl) {
|
|
210
230
|
this.storage.setExpiry(path, options.ttl);
|
|
@@ -218,12 +238,15 @@ export class BodDB {
|
|
|
218
238
|
/** Manually trigger TTL sweep + stream auto-compact, returns expired paths */
|
|
219
239
|
sweep(): string[] {
|
|
220
240
|
const expired = this.storage.sweep();
|
|
221
|
-
if (expired.length > 0
|
|
222
|
-
this.subs.
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
241
|
+
if (expired.length > 0) {
|
|
242
|
+
if (this.subs.hasSubscriptions) {
|
|
243
|
+
this.subs.notify(expired, (p) => this.storage.get(p));
|
|
244
|
+
}
|
|
245
|
+
// Fire delete events for expired paths (replication)
|
|
246
|
+
for (const p of expired) {
|
|
247
|
+
this._fireWrite({ op: 'delete', path: p });
|
|
248
|
+
}
|
|
249
|
+
this.log.forComponent('storage').debug(`Sweep: ${expired.length} expired paths removed`);
|
|
227
250
|
}
|
|
228
251
|
this.stream.autoCompact();
|
|
229
252
|
this.mq.sweep();
|
|
@@ -250,7 +273,7 @@ export class BodDB {
|
|
|
250
273
|
|
|
251
274
|
delete(path: string): void {
|
|
252
275
|
const p = validatePath(path);
|
|
253
|
-
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) :
|
|
276
|
+
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
|
|
254
277
|
this.storage.delete(path);
|
|
255
278
|
if (this.subs.hasSubscriptions) {
|
|
256
279
|
this.subs.notify([p], (pp) => this.storage.get(pp), existedBefore);
|
|
@@ -261,7 +284,7 @@ export class BodDB {
|
|
|
261
284
|
/** Push a value with auto-generated time-sortable key (stored as single JSON row, not flattened) */
|
|
262
285
|
push(path: string, value: unknown, opts?: { idempotencyKey?: string }): string {
|
|
263
286
|
const p = validatePath(path);
|
|
264
|
-
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) :
|
|
287
|
+
const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
|
|
265
288
|
const { key, changedPaths, duplicate } = this.storage.push(path, value, opts);
|
|
266
289
|
if (!duplicate && this.subs.hasSubscriptions) {
|
|
267
290
|
this.subs.notify(changedPaths, (pp) => this.storage.get(pp), existedBefore);
|
|
@@ -350,13 +373,21 @@ export class BodDB {
|
|
|
350
373
|
},
|
|
351
374
|
};
|
|
352
375
|
|
|
376
|
+
this.replication?.beginBatch();
|
|
353
377
|
const sqliteTx = this.storage.db.transaction(() => fn(proxy));
|
|
354
|
-
|
|
378
|
+
let result: T;
|
|
379
|
+
try {
|
|
380
|
+
result = sqliteTx();
|
|
381
|
+
} catch (e) {
|
|
382
|
+
this.replication?.discardBatch();
|
|
383
|
+
throw e;
|
|
384
|
+
}
|
|
355
385
|
|
|
356
386
|
if (this.subs.hasSubscriptions && allChanged.length > 0) {
|
|
357
387
|
this.subs.notify(allChanged, (p) => this.storage.get(p), allExistedBefore);
|
|
358
388
|
}
|
|
359
389
|
for (const ev of txWriteEvents) this._fireWrite(ev);
|
|
390
|
+
this.replication?.flushBatch();
|
|
360
391
|
return result;
|
|
361
392
|
}
|
|
362
393
|
|
|
@@ -366,53 +397,95 @@ export class BodDB {
|
|
|
366
397
|
const { statSync } = require('fs');
|
|
367
398
|
const { cpus, totalmem } = require('os');
|
|
368
399
|
let lastOsCpus = cpus();
|
|
400
|
+
// Clean up old flattened _admin/stats/* leaf rows (migrated to single JSON row)
|
|
401
|
+
this.storage.db.prepare("DELETE FROM nodes WHERE path LIKE '_admin/stats/%'").run();
|
|
402
|
+
|
|
403
|
+
// Reuse prepared statement across start/stop cycles
|
|
404
|
+
if (!this._statsStmtCount) {
|
|
405
|
+
this._statsStmtCount = this.storage.db.prepare('SELECT COUNT(*) as n FROM nodes WHERE mq_status IS NULL');
|
|
406
|
+
}
|
|
407
|
+
const stmtCount = this._statsStmtCount;
|
|
408
|
+
const statsPath = '_admin/stats';
|
|
409
|
+
|
|
410
|
+
// Reuse a single stats object to minimize allocations
|
|
411
|
+
const statsData: Record<string, unknown> = {
|
|
412
|
+
process: {}, db: {}, system: {},
|
|
413
|
+
subs: 0, clients: 0, repl: null, ts: 0,
|
|
414
|
+
};
|
|
415
|
+
const proc = statsData.process as Record<string, number>;
|
|
416
|
+
const dbStats = statsData.db as Record<string, number>;
|
|
417
|
+
const sys = statsData.system as Record<string, number>;
|
|
418
|
+
|
|
419
|
+
// Heavy calls (cpus, statSync, stmtCount) run every 5s; lightweight calls every 1s
|
|
420
|
+
let tick = 0;
|
|
421
|
+
const MB = 1 / (1024 * 1024);
|
|
422
|
+
sys.cpuCores = cpus().length;
|
|
423
|
+
sys.totalMemMb = Math.round(totalmem() * MB);
|
|
369
424
|
|
|
370
425
|
this._statsInterval = setInterval(() => {
|
|
371
|
-
if (!this.
|
|
426
|
+
if (!this._transport) return;
|
|
427
|
+
tick++;
|
|
372
428
|
|
|
373
429
|
const now = performance.now();
|
|
374
430
|
const cpu = process.cpuUsage();
|
|
375
431
|
const elapsedUs = (now - this._lastCpuTime) * 1000;
|
|
376
|
-
|
|
432
|
+
proc.cpuPercent = Math.round((cpu.user - this._lastCpuUsage.user + cpu.system - this._lastCpuUsage.system) / elapsedUs * 1000) / 10;
|
|
377
433
|
this._lastCpuUsage = cpu;
|
|
378
434
|
this._lastCpuTime = now;
|
|
379
435
|
|
|
380
436
|
const mem = process.memoryUsage();
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
437
|
+
proc.heapUsedMb = Math.round(mem.heapUsed * MB * 100) / 100;
|
|
438
|
+
proc.rssMb = Math.round(mem.rss * MB * 100) / 100;
|
|
439
|
+
proc.uptimeSec = Math.floor(process.uptime());
|
|
440
|
+
|
|
441
|
+
// Expensive calls: only every 5 ticks
|
|
442
|
+
if (tick % 5 === 0) {
|
|
443
|
+
try { dbStats.nodeCount = (stmtCount.get() as any).n; } catch {}
|
|
444
|
+
try { dbStats.sizeMb = Math.round(statSync(this.options.path).size * MB * 100) / 100; } catch {}
|
|
445
|
+
const cur = cpus();
|
|
446
|
+
let idleDelta = 0, totalDelta = 0;
|
|
447
|
+
for (let i = 0; i < cur.length; i++) {
|
|
448
|
+
const prev = lastOsCpus[i]?.times ?? cur[i].times;
|
|
449
|
+
const c = cur[i].times;
|
|
450
|
+
idleDelta += c.idle - prev.idle;
|
|
451
|
+
totalDelta += (c.user + c.nice + c.sys + c.irq + c.idle) - (prev.user + prev.nice + prev.sys + prev.irq + prev.idle);
|
|
452
|
+
}
|
|
453
|
+
lastOsCpus = cur;
|
|
454
|
+
sys.cpuPercent = totalDelta > 0 ? Math.round((1 - idleDelta / totalDelta) * 1000) / 10 : 0;
|
|
394
455
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
this.
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
ts: Date.now(),
|
|
406
|
-
});
|
|
456
|
+
|
|
457
|
+
statsData.subs = this.subs.subscriberCount();
|
|
458
|
+
statsData.clients = this._transport?.clientCount ?? 0;
|
|
459
|
+
statsData.repl = this.replication?.stats() ?? null;
|
|
460
|
+
statsData.ts = Date.now();
|
|
461
|
+
|
|
462
|
+
// Push directly to WS clients — bypass SubscriptionEngine entirely
|
|
463
|
+
this._lastStats = statsData;
|
|
464
|
+
if (this._transport) this._transport.broadcastStats(statsData, Date.now());
|
|
465
|
+
|
|
407
466
|
}, 1000);
|
|
467
|
+
this.log.forComponent('stats').info('Stats publisher started');
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/** Stop the stats publisher. */
|
|
471
|
+
stopStatsPublisher(): void {
|
|
472
|
+
if (!this._statsInterval) return;
|
|
473
|
+
clearInterval(this._statsInterval);
|
|
474
|
+
this._statsInterval = null;
|
|
475
|
+
this._lastStats = null;
|
|
476
|
+
this.subs.pushValue('_admin/stats', null);
|
|
477
|
+
this.log.forComponent('stats').info('Stats publisher stopped');
|
|
408
478
|
}
|
|
409
479
|
|
|
480
|
+
get statsEnabled(): boolean { return this._statsInterval !== null; }
|
|
481
|
+
|
|
410
482
|
/** Start standalone WebSocket + REST server on its own port */
|
|
411
483
|
serve(options?: Partial<TransportOptions>) {
|
|
412
484
|
const port = options?.port ?? this.options.port;
|
|
413
485
|
const auth = options?.auth ?? this.options.auth ?? (this.keyAuth ? this.keyAuth.authCallback() : undefined);
|
|
414
486
|
this._transport = new Transport(this, this.rules, { ...this.options.transport, ...options, port, auth, keyAuth: this.keyAuth ?? undefined });
|
|
415
487
|
this.startStatsPublisher();
|
|
488
|
+
this.log.forComponent('transport').info(`Server starting on port ${port}`);
|
|
416
489
|
return this._transport.start();
|
|
417
490
|
}
|
|
418
491
|
|
|
@@ -467,13 +540,11 @@ export class BodDB {
|
|
|
467
540
|
}
|
|
468
541
|
|
|
469
542
|
private snapshotExisting(path: string): Set<string> {
|
|
470
|
-
const
|
|
471
|
-
if (this.storage.exists(path)) existing.add(path);
|
|
543
|
+
const paths = [path];
|
|
472
544
|
const parts = path.split('/');
|
|
473
545
|
for (let i = 1; i < parts.length; i++) {
|
|
474
|
-
|
|
475
|
-
if (this.storage.exists(childPath)) existing.add(childPath);
|
|
546
|
+
paths.push(parts.slice(0, i + 1).join('/'));
|
|
476
547
|
}
|
|
477
|
-
return
|
|
548
|
+
return this.storage.existsMany(paths);
|
|
478
549
|
}
|
|
479
550
|
}
|
|
@@ -19,6 +19,12 @@ export class KeyAuthEngineOptions {
|
|
|
19
19
|
clockSkew: number = 30;
|
|
20
20
|
/** Allow unauthenticated device registration (default true) */
|
|
21
21
|
allowOpenRegistration: boolean = true;
|
|
22
|
+
/** PBKDF2 iterations for password-based key derivation (default 100000) */
|
|
23
|
+
pbkdf2Iterations: number = 100_000;
|
|
24
|
+
/** Max failed auth attempts before lockout (0 = disabled) */
|
|
25
|
+
maxAuthFailures: number = 0;
|
|
26
|
+
/** Lockout duration in seconds after max failures */
|
|
27
|
+
authLockoutSeconds: number = 60;
|
|
22
28
|
}
|
|
23
29
|
|
|
24
30
|
/** Reserved prefix — _auth/ writes blocked for external clients */
|
|
@@ -80,7 +86,7 @@ export class KeyAuthEngine {
|
|
|
80
86
|
* If this is the first account ever created, it is auto-elevated to root. */
|
|
81
87
|
createAccount(password: string, roles: string[] = [], displayName?: string, devicePublicKey?: string): { publicKey: string; fingerprint: string; isRoot: boolean; deviceFingerprint?: string } {
|
|
82
88
|
const kp = generateEd25519KeyPair();
|
|
83
|
-
const enc = encryptPrivateKey(kp.privateKey, password);
|
|
89
|
+
const enc = encryptPrivateKey(kp.privateKey, password, this.options.pbkdf2Iterations);
|
|
84
90
|
const fp = fingerprint(kp.publicKeyBase64);
|
|
85
91
|
|
|
86
92
|
// First account becomes root automatically
|
|
@@ -145,6 +151,14 @@ export class KeyAuthEngine {
|
|
|
145
151
|
|
|
146
152
|
/** Verify a signed nonce. Returns token + expiry or null. */
|
|
147
153
|
verify(publicKeyBase64: string, signatureBase64: string, nonce: string): { token: string; expiresAt: number } | null {
|
|
154
|
+
const fp = fingerprint(publicKeyBase64);
|
|
155
|
+
|
|
156
|
+
// Rate limit check
|
|
157
|
+
if (this.options.maxAuthFailures > 0) {
|
|
158
|
+
const rlData = this.db.get(`_auth/rateLimit/${fp}`) as { count: number } | null;
|
|
159
|
+
if (rlData && rlData.count >= this.options.maxAuthFailures) return null;
|
|
160
|
+
}
|
|
161
|
+
|
|
148
162
|
// Check nonce exists (one-time use)
|
|
149
163
|
const nonceData = this.db.get(`_auth/nonces/${nonce}`);
|
|
150
164
|
if (!nonceData) return null;
|
|
@@ -154,9 +168,10 @@ export class KeyAuthEngine {
|
|
|
154
168
|
// Verify signature
|
|
155
169
|
const pubKeyDer = Buffer.from(publicKeyBase64, 'base64');
|
|
156
170
|
const sig = Buffer.from(signatureBase64, 'base64');
|
|
157
|
-
if (!verifySignature(Buffer.from(nonce), sig, new Uint8Array(pubKeyDer)))
|
|
158
|
-
|
|
159
|
-
|
|
171
|
+
if (!verifySignature(Buffer.from(nonce), sig, new Uint8Array(pubKeyDer))) {
|
|
172
|
+
this._recordAuthFailure(fp);
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
160
175
|
|
|
161
176
|
// Check if root
|
|
162
177
|
const root = this.db.get('_auth/root') as { fingerprint: string } | null;
|
|
@@ -200,6 +215,8 @@ export class KeyAuthEngine {
|
|
|
200
215
|
sid, fp, accountFp, roles, root: isRoot, exp: expiresAt,
|
|
201
216
|
};
|
|
202
217
|
const token = encodeToken(payload, this.serverPrivateKey);
|
|
218
|
+
// Clear rate limit on success
|
|
219
|
+
if (this.options.maxAuthFailures > 0) this.db.delete(`_auth/rateLimit/${fp}`);
|
|
203
220
|
return { token, expiresAt };
|
|
204
221
|
}
|
|
205
222
|
|
|
@@ -213,7 +230,7 @@ export class KeyAuthEngine {
|
|
|
213
230
|
let accountPrivateKey: Uint8Array;
|
|
214
231
|
try {
|
|
215
232
|
accountPrivateKey = decryptPrivateKey(
|
|
216
|
-
account.encryptedPrivateKey, account.salt, account.iv, account.authTag, password,
|
|
233
|
+
account.encryptedPrivateKey, account.salt, account.iv, account.authTag, password, this.options.pbkdf2Iterations,
|
|
217
234
|
);
|
|
218
235
|
} catch {
|
|
219
236
|
return null; // Wrong password
|
|
@@ -269,12 +286,12 @@ export class KeyAuthEngine {
|
|
|
269
286
|
|
|
270
287
|
let privateKey: Uint8Array;
|
|
271
288
|
try {
|
|
272
|
-
privateKey = decryptPrivateKey(account.encryptedPrivateKey, account.salt, account.iv, account.authTag, oldPassword);
|
|
289
|
+
privateKey = decryptPrivateKey(account.encryptedPrivateKey, account.salt, account.iv, account.authTag, oldPassword, this.options.pbkdf2Iterations);
|
|
273
290
|
} catch {
|
|
274
291
|
return false;
|
|
275
292
|
}
|
|
276
293
|
|
|
277
|
-
const enc = encryptPrivateKey(privateKey, newPassword);
|
|
294
|
+
const enc = encryptPrivateKey(privateKey, newPassword, this.options.pbkdf2Iterations);
|
|
278
295
|
privateKey.fill(0); // Wipe
|
|
279
296
|
|
|
280
297
|
// Atomic: single update call writes all fields together
|
|
@@ -460,6 +477,14 @@ export class KeyAuthEngine {
|
|
|
460
477
|
return { accountFp, roles: account.roles ?? [] };
|
|
461
478
|
}
|
|
462
479
|
|
|
480
|
+
/** Record a failed auth attempt for rate limiting */
|
|
481
|
+
private _recordAuthFailure(fp: string): void {
|
|
482
|
+
if (this.options.maxAuthFailures <= 0) return;
|
|
483
|
+
const rlData = this.db.get(`_auth/rateLimit/${fp}`) as { count: number } | null;
|
|
484
|
+
const count = (rlData?.count ?? 0) + 1;
|
|
485
|
+
this.db.set(`_auth/rateLimit/${fp}`, { count }, { ttl: this.options.authLockoutSeconds });
|
|
486
|
+
}
|
|
487
|
+
|
|
463
488
|
/** Invalidate all sessions for a given device fingerprint */
|
|
464
489
|
private _invalidateDeviceSessions(deviceFp: string): void {
|
|
465
490
|
const sessions = this.db.getShallow('_auth/sessions');
|