@dxos/log-store-idb 0.8.4-main.fdfb99ef29 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/log-store-idb",
3
- "version": "0.8.4-main.fdfb99ef29",
3
+ "version": "0.9.0",
4
4
  "description": "IndexedDB-backed log store for @dxos/log.",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -8,7 +8,7 @@
8
8
  "type": "git",
9
9
  "url": "https://github.com/dxos/dxos"
10
10
  },
11
- "license": "MIT",
11
+ "license": "FSL-1.1-Apache-2.0",
12
12
  "author": "DXOS.org",
13
13
  "sideEffects": false,
14
14
  "type": "module",
@@ -29,7 +29,7 @@
29
29
  "src"
30
30
  ],
31
31
  "dependencies": {
32
- "@dxos/log": "0.8.4-main.fdfb99ef29"
32
+ "@dxos/log": "0.9.0"
33
33
  },
34
34
  "devDependencies": {},
35
35
  "publishConfig": {
@@ -34,6 +34,50 @@ const drop = (dbName: string): Promise<void> =>
34
34
  req.onblocked = () => resolve();
35
35
  });
36
36
 
37
+ /** Count raw records in the store (chunks, not lines). */
38
+ const countRecords = (dbName: string): Promise<number> =>
39
+ new Promise((resolve, reject) => {
40
+ const openReq = indexedDB.open(dbName);
41
+ openReq.onsuccess = () => {
42
+ const db = openReq.result;
43
+ const countReq = db.transaction('logs', 'readonly').objectStore('logs').count();
44
+ countReq.onsuccess = () => {
45
+ db.close();
46
+ resolve(countReq.result);
47
+ };
48
+ countReq.onerror = () => {
49
+ db.close();
50
+ reject(countReq.error);
51
+ };
52
+ };
53
+ openReq.onerror = () => reject(openReq.error);
54
+ });
55
+
56
+ /** Create a database with the v1 schema (one row per line, autoincrement keyPath). */
57
+ const createV1Database = (dbName: string, lines: string[]): Promise<void> =>
58
+ new Promise((resolve, reject) => {
59
+ const openReq = indexedDB.open(dbName, 1);
60
+ openReq.onupgradeneeded = () => {
61
+ openReq.result.createObjectStore('logs', { keyPath: 'seq', autoIncrement: true });
62
+ };
63
+ openReq.onsuccess = () => {
64
+ const db = openReq.result;
65
+ const tx = db.transaction('logs', 'readwrite');
66
+ for (const line of lines) {
67
+ tx.objectStore('logs').add({ line });
68
+ }
69
+ tx.oncomplete = () => {
70
+ db.close();
71
+ resolve();
72
+ };
73
+ tx.onerror = () => {
74
+ db.close();
75
+ reject(tx.error);
76
+ };
77
+ };
78
+ openReq.onerror = () => reject(openReq.error);
79
+ });
80
+
37
81
  describe('IdbLogStore', () => {
38
82
  let store: IdbLogStore | undefined;
39
83
  let dbName = '';
@@ -151,6 +195,22 @@ describe('IdbLogStore', () => {
151
195
  expect(jsonl).toContain('persisted');
152
196
  });
153
197
 
198
+ test('each flush writes a single chunk record', async ({ expect }) => {
199
+ store = new IdbLogStore({ dbName, flushInterval: 60_000 });
200
+ for (let i = 0; i < 5; i++) {
201
+ store.processor(fakeConfig, makeEntry(LogLevel.INFO, `first-${i}`));
202
+ }
203
+ await store.flush();
204
+ for (let i = 0; i < 3; i++) {
205
+ store.processor(fakeConfig, makeEntry(LogLevel.INFO, `second-${i}`));
206
+ }
207
+ await store.flush();
208
+
209
+ expect(await countRecords(dbName)).toBe(2);
210
+ const jsonl = await store.export();
211
+ expect(jsonl.split('\n')).toHaveLength(8);
212
+ });
213
+
154
214
  test('eviction trims down to maxRecords', async ({ expect }) => {
155
215
  store = new IdbLogStore({
156
216
  dbName,
@@ -158,10 +218,11 @@ describe('IdbLogStore', () => {
158
218
  maxRecords: 5,
159
219
  evictionInterval: 0, // disable timer; trigger manually
160
220
  });
221
+ // Flush per entry so eviction (whole chunks) can trim at line granularity.
161
222
  for (let i = 0; i < 12; i++) {
162
223
  store.processor(fakeConfig, makeEntry(LogLevel.INFO, `msg-${i}`));
224
+ await store.flush();
163
225
  }
164
- await store.flush();
165
226
  await store.evictNow();
166
227
  const jsonl = await store.export();
167
228
  const lines = jsonl.split('\n').filter((line) => line.length > 0);
@@ -170,6 +231,46 @@ describe('IdbLogStore', () => {
170
231
  expect(JSON.parse(lines[lines.length - 1]).m).toBe('msg-11');
171
232
  });
172
233
 
234
+ test('eviction drops whole chunks and always keeps the newest chunk', async ({ expect }) => {
235
+ store = new IdbLogStore({
236
+ dbName,
237
+ flushInterval: 10,
238
+ maxRecords: 5,
239
+ evictionInterval: 0,
240
+ });
241
+ // Older chunk of 3 lines, then a newest chunk of 12 lines (exceeds the budget alone).
242
+ for (let i = 0; i < 3; i++) {
243
+ store.processor(fakeConfig, makeEntry(LogLevel.INFO, `old-${i}`));
244
+ }
245
+ await store.flush();
246
+ for (let i = 0; i < 12; i++) {
247
+ store.processor(fakeConfig, makeEntry(LogLevel.INFO, `new-${i}`));
248
+ }
249
+ await store.flush();
250
+
251
+ await store.evictNow();
252
+ const messages = (await store.export())
253
+ .split('\n')
254
+ .filter(Boolean)
255
+ .map((line) => JSON.parse(line).m);
256
+ // The newest chunk is never split or dropped; the older chunk is evicted.
257
+ expect(messages).toHaveLength(12);
258
+ expect(messages[0]).toBe('new-0');
259
+ expect(messages[11]).toBe('new-11');
260
+ });
261
+
262
+ test('upgrades a v1 database, discarding old single-line rows', async ({ expect }) => {
263
+ await createV1Database(dbName, ['{"m":"legacy"}']);
264
+
265
+ store = new IdbLogStore({ dbName, flushInterval: 10 });
266
+ store.processor(fakeConfig, makeEntry(LogLevel.INFO, 'fresh'));
267
+ await store.flush();
268
+
269
+ const jsonl = await store.export();
270
+ expect(jsonl).not.toContain('legacy');
271
+ expect(jsonl).toContain('fresh');
272
+ });
273
+
173
274
  test('multiple stores against the same db preserve all writes', async ({ expect }) => {
174
275
  const sharedDb = makeDbName();
175
276
  try {
@@ -24,15 +24,26 @@ const DEFAULT_FLUSH_BATCH_SIZE = 500;
24
24
  const DEFAULT_MAX_RECORDS = 150_000;
25
25
  const DEFAULT_EVICTION_INTERVAL = 30_000;
26
26
  const EVICTION_LOCK_NAME = '@dxos/log-store-idb:evictor';
27
+ // v2 introduced chunked rows with out-of-line array keys (v1 stored one row per line
28
+ // under an autoincrement keyPath).
29
+ const DB_VERSION = 2;
27
30
 
28
31
  /**
29
- * Stored log row. The line is pre-encoded JSONL (no trailing newline).
32
+ * Chunk key: `[epochMs, writerId, writerSeq, lineCount]`.
33
+ *
34
+ * Keys order chunks by flush time; `writerId`/`writerSeq` keep keys from concurrent
35
+ * writers unique. `lineCount` rides in the key so eviction can compute the retention
36
+ * budget from `getAllKeys()` alone, without deserializing chunk values.
37
+ */
38
+ type ChunkKey = [epochMs: number, writerId: string, writerSeq: number, lineCount: number];
39
+
40
+ /**
41
+ * Stored row: one flush batch of pre-encoded JSONL lines joined with `\n` (no trailing
42
+ * newline). Storing the whole batch as a single record keeps the per-flush cost at one
43
+ * structured clone and one backend record regardless of batch size.
30
44
  */
31
- type LogRow = {
32
- /** Auto-incremented sequence (commit order). */
33
- seq?: number;
34
- /** Encoded JSONL line. */
35
- line: string;
45
+ type LogChunk = {
46
+ lines: string;
36
47
  };
37
48
 
38
49
  export type IdbLogStoreOptions = {
@@ -45,9 +56,10 @@ export type IdbLogStoreOptions = {
45
56
  /** Auto-flush when in-memory queue reaches this size. Default `500`. */
46
57
  flushBatchSize?: number;
47
58
  /**
48
- * Soft cap on retained records. Older rows are evicted past this. Default `150_000`,
49
- * sized for roughly 50 MB on disk at typical JSONL line sizes (~350 bytes average
50
- * observed in the Composer app).
59
+ * Soft cap on retained log records (JSONL lines). Eviction drops whole flush chunks,
60
+ * oldest first; the newest chunk is always retained. Default `150_000`, sized for
61
+ * roughly 50 MB on disk at typical JSONL line sizes (~350 bytes average observed in
62
+ * the Composer app).
51
63
  */
52
64
  maxRecords?: number;
53
65
  /** Eviction sweep interval in milliseconds. Default `30_000`. */
@@ -67,13 +79,14 @@ export type IdbLogStoreOptions = {
67
79
  /**
68
80
  * IndexedDB-backed log store.
69
81
  *
70
- * Logs are batched in memory and flushed to IndexedDB as pre-encoded JSONL strings.
71
- * Multiple browsing contexts can write to the same database concurrently — IDB
72
- * serializes transactions, so no extra coordination is needed for writes.
73
- * Eviction is deduplicated across tabs via the Web Locks API (`ifAvailable`).
82
+ * Logs are batched in memory and each flush is written as a single chunk record of
83
+ * pre-encoded JSONL. Multiple browsing contexts can write to the same database
84
+ * concurrently — IDB serializes transactions, so no extra coordination is needed for
85
+ * writes. Eviction is deduplicated across tabs via the Web Locks API (`ifAvailable`).
74
86
  *
75
87
  * Flushes are triggered by:
76
- * - the configured interval timer,
88
+ * - a deferred task after `flushInterval` (background priority where the Scheduler API
89
+ * is available, so flushes don't compete with input handling and rendering),
77
90
  * - the queue exceeding `flushBatchSize`,
78
91
  * - `visibilitychange` to `hidden` and `pagehide` events.
79
92
  */
@@ -86,9 +99,12 @@ export class IdbLogStore {
86
99
  readonly #evictionInterval: number;
87
100
  readonly #tabId: string;
88
101
  readonly #filters: LogFilter[];
102
+ /** Distinguishes chunk keys written by concurrent contexts against the same database. */
103
+ readonly #writerId = Math.random().toString(36).slice(2);
89
104
 
90
105
  #queue: string[] = [];
91
- #flushTimer: ReturnType<typeof setTimeout> | undefined;
106
+ #writerSeq = 0;
107
+ #flushTask: ScheduledTask | undefined;
92
108
  #evictionTimer: ReturnType<typeof setInterval> | undefined;
93
109
  #pendingFlush: Promise<void> | undefined;
94
110
  #db: IDBDatabase | undefined;
@@ -129,9 +145,9 @@ export class IdbLogStore {
129
145
 
130
146
  if (this.#queue.length >= this.#flushBatchSize) {
131
147
  void this.flush();
132
- } else if (this.#flushTimer === undefined) {
133
- this.#flushTimer = setTimeout(() => {
134
- this.#flushTimer = undefined;
148
+ } else if (this.#flushTask === undefined) {
149
+ this.#flushTask = scheduleBackgroundTask(() => {
150
+ this.#flushTask = undefined;
135
151
  void this.flush();
136
152
  }, this.#flushInterval);
137
153
  }
@@ -142,9 +158,9 @@ export class IdbLogStore {
142
158
  * Concurrent calls share a single in-flight flush.
143
159
  */
144
160
  async flush(): Promise<void> {
145
- if (this.#flushTimer !== undefined) {
146
- clearTimeout(this.#flushTimer);
147
- this.#flushTimer = undefined;
161
+ if (this.#flushTask !== undefined) {
162
+ this.#flushTask.cancel();
163
+ this.#flushTask = undefined;
148
164
  }
149
165
  if (this.#pendingFlush) {
150
166
  return this.#pendingFlush;
@@ -171,11 +187,13 @@ export class IdbLogStore {
171
187
  async export(options: { maxSize?: number } = {}): Promise<string> {
172
188
  await this.flush();
173
189
  const db = await this.#open();
174
- const lines = await readAllLines(db, this.#storeName);
190
+ const chunks = await readAllChunks(db, this.#storeName);
175
191
  if (options.maxSize !== undefined && options.maxSize >= 0) {
192
+ // Trim on individual lines so the size cap never under-fills by a whole chunk.
193
+ const lines = chunks.flatMap((chunk) => chunk.split('\n'));
176
194
  return trimJsonlToSize(lines, options.maxSize);
177
195
  }
178
- return lines.join('\n');
196
+ return chunks.join('\n');
179
197
  }
180
198
 
181
199
  /**
@@ -183,9 +201,9 @@ export class IdbLogStore {
183
201
  */
184
202
  async clear(): Promise<void> {
185
203
  this.#queue = [];
186
- if (this.#flushTimer !== undefined) {
187
- clearTimeout(this.#flushTimer);
188
- this.#flushTimer = undefined;
204
+ if (this.#flushTask !== undefined) {
205
+ this.#flushTask.cancel();
206
+ this.#flushTask = undefined;
189
207
  }
190
208
  const db = await this.#open();
191
209
  await runTransaction(db, this.#storeName, 'readwrite', (store) => {
@@ -244,11 +262,10 @@ export class IdbLogStore {
244
262
  }
245
263
 
246
264
  try {
265
+ const chunk: LogChunk = { lines: batch.join('\n') };
266
+ const key: ChunkKey = [Date.now(), this.#writerId, this.#writerSeq++, batch.length];
247
267
  await runTransaction(db, this.#storeName, 'readwrite', (store) => {
248
- for (const line of batch) {
249
- const row: LogRow = { line };
250
- store.add(row);
251
- }
268
+ store.add(chunk, key);
252
269
  });
253
270
  } catch {
254
271
  // Ignore write errors.
@@ -315,14 +332,12 @@ export class IdbLogStore {
315
332
  return;
316
333
  }
317
334
 
318
- const count = await runTransaction(db, this.#storeName, 'readonly', (store) => promisifyRequest(store.count()));
319
- if (count <= this.#maxRecords) {
320
- return;
321
- }
322
-
323
- const toDelete = count - this.#maxRecords;
324
335
  await runTransaction(db, this.#storeName, 'readwrite', async (store) => {
325
- const cutoff = await findCutoffKey(store, toDelete);
336
+ // Keys carry per-chunk line counts, so the retention budget is computed from
337
+ // getAllKeys() alone — no count() pre-pass and no chunk values are read.
338
+ // Cast: IDB types keys as IDBValidKey[]; this store only ever receives ChunkKey keys.
339
+ const keys = (await promisifyRequest(store.getAllKeys())) as ChunkKey[];
340
+ const cutoff = findEvictionCutoff(keys, this.#maxRecords);
326
341
  if (cutoff !== undefined) {
327
342
  store.delete(IDBKeyRange.upperBound(cutoff));
328
343
  }
@@ -362,12 +377,16 @@ export class IdbLogStore {
362
377
 
363
378
  const openDatabase = (name: string, storeName: string): Promise<IDBDatabase> => {
364
379
  return new Promise((resolve, reject) => {
365
- const request = indexedDB.open(name, 1);
380
+ const request = indexedDB.open(name, DB_VERSION);
366
381
  request.onupgradeneeded = () => {
367
382
  const db = request.result;
368
- if (!db.objectStoreNames.contains(storeName)) {
369
- db.createObjectStore(storeName, { keyPath: 'seq', autoIncrement: true });
383
+ // The v1 schema stored one row per line under an autoincrement keyPath; chunked
384
+ // storage requires out-of-line keys, so the store is recreated and v1 data is
385
+ // discarded (logs are expendable diagnostics).
386
+ if (db.objectStoreNames.contains(storeName)) {
387
+ db.deleteObjectStore(storeName);
370
388
  }
389
+ db.createObjectStore(storeName);
371
390
  };
372
391
  request.onsuccess = () => resolve(request.result);
373
392
  request.onerror = () => reject(request.error);
@@ -439,37 +458,54 @@ const runTransaction = <T>(
439
458
  });
440
459
  };
441
460
 
442
- const readAllLines = async (db: IDBDatabase, storeName: string): Promise<string[]> => {
461
+ const readAllChunks = async (db: IDBDatabase, storeName: string): Promise<string[]> => {
443
462
  return runTransaction(db, storeName, 'readonly', (store) =>
444
- promisifyRequest(store.getAll()).then((rows) => (rows as LogRow[]).map((row) => row.line)),
463
+ promisifyRequest(store.getAll()).then((rows) => (rows as LogChunk[]).map((row) => row.lines)),
445
464
  );
446
465
  };
447
466
 
448
467
  /**
449
- * Walk the store in key order and return the key of the `count`-th entry.
450
- * Returns `undefined` if the store has fewer than `count` entries.
468
+ * Given all chunk keys in ascending (key) order, return the key of the newest chunk that
469
+ * must be evicted — delete it and everything older — so the retained line total stays
470
+ * within `maxRecords`. The newest chunk is always retained, even if it alone exceeds the
471
+ * budget. Returns `undefined` when everything fits.
451
472
  */
452
- const findCutoffKey = (store: IDBObjectStore, count: number): Promise<IDBValidKey | undefined> => {
453
- return new Promise((resolve, reject) => {
454
- if (count <= 0) {
455
- resolve(undefined);
456
- return;
473
+ const findEvictionCutoff = (keys: readonly ChunkKey[], maxRecords: number): ChunkKey | undefined => {
474
+ let retained = 0;
475
+ for (let index = keys.length - 1; index >= 0; index--) {
476
+ const [, , , lineCount] = keys[index];
477
+ retained += lineCount;
478
+ if (retained > maxRecords && index < keys.length - 1) {
479
+ return keys[index];
457
480
  }
458
- const request = store.openKeyCursor();
459
- let advanced = false;
460
- request.onsuccess = () => {
461
- const cursor = request.result;
462
- if (!cursor) {
463
- resolve(undefined);
464
- return;
465
- }
466
- if (!advanced && count > 1) {
467
- advanced = true;
468
- cursor.advance(count - 1);
469
- return;
470
- }
471
- resolve(cursor.key);
472
- };
473
- request.onerror = () => reject(request.error);
474
- });
481
+ }
482
+ return undefined;
483
+ };
484
+
485
+ type ScheduledTask = { cancel: () => void };
486
+
487
+ type SchedulerLike = {
488
+ postTask: (
489
+ callback: () => void,
490
+ options?: { priority?: 'background' | 'user-visible' | 'user-blocking'; delay?: number; signal?: AbortSignal },
491
+ ) => Promise<unknown>;
492
+ };
493
+
494
+ /**
495
+ * Run `callback` after `delay` ms at background priority where the Scheduler API is
496
+ * available, so deferred work yields to input handling and rendering; falls back to a
497
+ * plain timeout elsewhere.
498
+ */
499
+ const scheduleBackgroundTask = (callback: () => void, delay: number): ScheduledTask => {
500
+ // The Scheduler API is not in the TS DOM lib yet.
501
+ const scheduler = (globalThis as { scheduler?: SchedulerLike }).scheduler;
502
+ if (typeof scheduler?.postTask === 'function') {
503
+ const controller = new AbortController();
504
+ scheduler.postTask(callback, { priority: 'background', delay, signal: controller.signal }).catch(() => {
505
+ // Rejects on abort; the callback never ran.
506
+ });
507
+ return { cancel: () => controller.abort() };
508
+ }
509
+ const timer = setTimeout(callback, delay);
510
+ return { cancel: () => clearTimeout(timer) };
475
511
  };