mindoodb 0.0.19 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +200 -0
- package/README-server.md +14 -14
- package/README.md +12 -10
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.d.ts +6 -0
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.d.ts.map +1 -1
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.js +519 -373
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.js.map +1 -1
- package/dist/browser/cache/IndexedDBLocalCacheStore.d.ts +12 -0
- package/dist/browser/cache/IndexedDBLocalCacheStore.d.ts.map +1 -1
- package/dist/browser/cache/IndexedDBLocalCacheStore.js +152 -65
- package/dist/browser/cache/IndexedDBLocalCacheStore.js.map +1 -1
- package/dist/browser/index.d.ts +1 -1
- package/dist/browser/index.d.ts.map +1 -1
- package/dist/browser/index.js +2 -1
- package/dist/browser/index.js.map +1 -1
- package/dist/browser/index.mjs +14 -0
- package/dist/core/BaseMindooDB.d.ts +594 -6
- package/dist/core/BaseMindooDB.d.ts.map +1 -1
- package/dist/core/BaseMindooDB.js +2478 -191
- package/dist/core/BaseMindooDB.js.map +1 -1
- package/dist/core/BaseMindooTenant.d.ts +65 -1
- package/dist/core/BaseMindooTenant.d.ts.map +1 -1
- package/dist/core/BaseMindooTenant.js +250 -35
- package/dist/core/BaseMindooTenant.js.map +1 -1
- package/dist/core/BaseMindooTenantFactory.d.ts.map +1 -1
- package/dist/core/BaseMindooTenantFactory.js +47 -18
- package/dist/core/BaseMindooTenantFactory.js.map +1 -1
- package/dist/core/DocumentConflictAnalysis.d.ts +22 -0
- package/dist/core/DocumentConflictAnalysis.d.ts.map +1 -0
- package/dist/core/DocumentConflictAnalysis.js +101 -0
- package/dist/core/DocumentConflictAnalysis.js.map +1 -0
- package/dist/core/MindooDBServerAdmin.d.ts +26 -1
- package/dist/core/MindooDBServerAdmin.d.ts.map +1 -1
- package/dist/core/MindooDBServerAdmin.js +19 -3
- package/dist/core/MindooDBServerAdmin.js.map +1 -1
- package/dist/core/appendonlystores/types.d.ts +9 -0
- package/dist/core/appendonlystores/types.d.ts.map +1 -1
- package/dist/core/automerge-adapter.d.ts.map +1 -1
- package/dist/core/automerge-adapter.js +26 -14
- package/dist/core/automerge-adapter.js.map +1 -1
- package/dist/core/cache/EncryptedLocalCacheStore.d.ts +51 -1
- package/dist/core/cache/EncryptedLocalCacheStore.d.ts.map +1 -1
- package/dist/core/cache/EncryptedLocalCacheStore.js +48 -9
- package/dist/core/cache/EncryptedLocalCacheStore.js.map +1 -1
- package/dist/core/cache/LocalCacheStore.d.ts +34 -0
- package/dist/core/cache/LocalCacheStore.d.ts.map +1 -1
- package/dist/core/cache/LocalCacheStore.js +16 -0
- package/dist/core/cache/LocalCacheStore.js.map +1 -1
- package/dist/core/databaseIdValidation.d.ts +13 -0
- package/dist/core/databaseIdValidation.d.ts.map +1 -0
- package/dist/core/databaseIdValidation.js +38 -0
- package/dist/core/databaseIdValidation.js.map +1 -0
- package/dist/core/index.d.ts +2 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +2 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.js +5 -1
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualView.d.ts +14 -0
- package/dist/core/indexing/virtualviews/VirtualView.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualView.js +38 -6
- package/dist/core/indexing/virtualviews/VirtualView.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewDataChange.d.ts +9 -2
- package/dist/core/indexing/virtualviews/VirtualViewDataChange.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewDataChange.js +10 -3
- package/dist/core/indexing/virtualviews/VirtualViewDataChange.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.d.ts +23 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.js +17 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.js.map +1 -1
- package/dist/core/indexing/virtualviews/types.d.ts +8 -0
- package/dist/core/indexing/virtualviews/types.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/types.js.map +1 -1
- package/dist/core/keys/KeyBag.d.ts +244 -7
- package/dist/core/keys/KeyBag.d.ts.map +1 -1
- package/dist/core/keys/KeyBag.js +299 -82
- package/dist/core/keys/KeyBag.js.map +1 -1
- package/dist/core/tenantIdValidation.d.ts +19 -0
- package/dist/core/tenantIdValidation.d.ts.map +1 -0
- package/dist/core/tenantIdValidation.js +43 -0
- package/dist/core/tenantIdValidation.js.map +1 -0
- package/dist/core/types.d.ts +703 -12
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/node/cache/FileSystemLocalCacheStore.d.ts +2 -1
- package/dist/node/cache/FileSystemLocalCacheStore.d.ts.map +1 -1
- package/dist/node/cache/FileSystemLocalCacheStore.js +4 -0
- package/dist/node/cache/FileSystemLocalCacheStore.js.map +1 -1
- package/dist/node/server/ServerSync.d.ts.map +1 -1
- package/dist/node/server/ServerSync.js +3 -1
- package/dist/node/server/ServerSync.js.map +1 -1
- package/dist/node/server/TenantManager.d.ts +1 -0
- package/dist/node/server/TenantManager.d.ts.map +1 -1
- package/dist/node/server/TenantManager.js +11 -3
- package/dist/node/server/TenantManager.js.map +1 -1
- package/dist/node/server/validation.d.ts +2 -6
- package/dist/node/server/validation.d.ts.map +1 -1
- package/dist/node/server/validation.js +14 -24
- package/dist/node/server/validation.js.map +1 -1
- package/dist/reactnative/cache/MMKVLocalCacheStore.d.ts +1 -0
- package/dist/reactnative/cache/MMKVLocalCacheStore.d.ts.map +1 -1
- package/dist/reactnative/cache/MMKVLocalCacheStore.js +14 -0
- package/dist/reactnative/cache/MMKVLocalCacheStore.js.map +1 -1
- package/package.json +25 -23
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
20
|
exports.IndexedDBContentAddressedStore = void 0;
|
|
21
|
+
exports.isIndexedDBConnectionLostError = isIndexedDBConnectionLostError;
|
|
21
22
|
exports.readContentAddressedStoreBytes = readContentAddressedStoreBytes;
|
|
22
23
|
const types_1 = require("../../core/appendonlystores/types");
|
|
23
24
|
const AttachmentReadPlanner_1 = require("../../core/appendonlystores/AttachmentReadPlanner");
|
|
@@ -42,6 +43,7 @@ const BLOOM_CACHE_KEY = "current";
|
|
|
42
43
|
const RECEIPT_ORDER_COUNTER_KEY = "receiptOrderCounter";
|
|
43
44
|
const TOTAL_CONTENT_BYTES_KEY = "total_content_bytes";
|
|
44
45
|
const TOTAL_METADATA_BYTES_KEY = "total_metadata_bytes";
|
|
46
|
+
const RECONNECT_RETRY_DELAYS_MS = [50, 250, 1000];
|
|
45
47
|
function qualifyIndexedDbNamespace(namespace, storeKind) {
|
|
46
48
|
const separatorIndex = namespace.indexOf("-");
|
|
47
49
|
if (separatorIndex === -1) {
|
|
@@ -49,6 +51,22 @@ function qualifyIndexedDbNamespace(namespace, storeKind) {
|
|
|
49
51
|
}
|
|
50
52
|
return `${namespace.slice(0, separatorIndex)}_${storeKind}${namespace.slice(separatorIndex)}`;
|
|
51
53
|
}
|
|
54
|
+
function isIndexedDBConnectionLostError(error) {
|
|
55
|
+
const name = error instanceof DOMException || error instanceof Error ? error.name : "";
|
|
56
|
+
const message = error instanceof DOMException || error instanceof Error
|
|
57
|
+
? error.message
|
|
58
|
+
: String(error ?? "");
|
|
59
|
+
const normalized = message.toLowerCase();
|
|
60
|
+
return name === "InvalidStateError"
|
|
61
|
+
|| name === "UnknownError"
|
|
62
|
+
|| normalized.includes("connection to indexed database server lost")
|
|
63
|
+
|| normalized.includes("connection to the database lost")
|
|
64
|
+
|| normalized.includes("database connection is closing")
|
|
65
|
+
|| normalized.includes("indexeddb transaction aborted");
|
|
66
|
+
}
|
|
67
|
+
function wait(ms) {
|
|
68
|
+
return new Promise((resolve) => window.setTimeout(resolve, ms));
|
|
69
|
+
}
|
|
52
70
|
// ---------------------------------------------------------------------------
|
|
53
71
|
// Browser-compatible bloom filter helpers (no Buffer dependency)
|
|
54
72
|
// ---------------------------------------------------------------------------
|
|
@@ -354,16 +372,16 @@ class IndexedDBContentAddressedStore {
|
|
|
354
372
|
this.db = request.result;
|
|
355
373
|
this.db.onversionchange = () => {
|
|
356
374
|
this.logger.warn("IndexedDB version change detected, closing connection");
|
|
357
|
-
this.
|
|
358
|
-
|
|
359
|
-
|
|
375
|
+
this.resetConnection();
|
|
376
|
+
};
|
|
377
|
+
this.db.onclose = () => {
|
|
378
|
+
this.logger.warn(`IndexedDB connection closed for ${this.idbName}; future operations will reopen it`);
|
|
379
|
+
this.resetConnection();
|
|
360
380
|
};
|
|
361
381
|
void this.backfillStoredBytesIfNeeded(this.db)
|
|
362
382
|
.then(() => resolve(this.db))
|
|
363
383
|
.catch((error) => {
|
|
364
|
-
this.
|
|
365
|
-
this.db = null;
|
|
366
|
-
this.openPromise = null;
|
|
384
|
+
this.resetConnection();
|
|
367
385
|
reject(error);
|
|
368
386
|
});
|
|
369
387
|
};
|
|
@@ -389,6 +407,36 @@ class IndexedDBContentAddressedStore {
|
|
|
389
407
|
getCacheIdentity() {
|
|
390
408
|
return `idb:${this.idbName}`;
|
|
391
409
|
}
|
|
410
|
+
resetConnection() {
|
|
411
|
+
const db = this.db;
|
|
412
|
+
this.db = null;
|
|
413
|
+
this.openPromise = null;
|
|
414
|
+
try {
|
|
415
|
+
db?.close();
|
|
416
|
+
}
|
|
417
|
+
catch {
|
|
418
|
+
// Ignore close errors from already-invalid WebKit IDB handles.
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
async runWithReconnect(opName, operation) {
|
|
422
|
+
let lastError;
|
|
423
|
+
for (let attempt = 0; attempt <= RECONNECT_RETRY_DELAYS_MS.length; attempt++) {
|
|
424
|
+
try {
|
|
425
|
+
return await operation();
|
|
426
|
+
}
|
|
427
|
+
catch (error) {
|
|
428
|
+
lastError = error;
|
|
429
|
+
if (!isIndexedDBConnectionLostError(error) || attempt >= RECONNECT_RETRY_DELAYS_MS.length) {
|
|
430
|
+
throw error;
|
|
431
|
+
}
|
|
432
|
+
this.logger.warn(`[idb-recovery] ${opName} failed for ${this.idbName}; reopening IndexedDB connection (attempt ${attempt + 1})`, error);
|
|
433
|
+
this.resetConnection();
|
|
434
|
+
await wait(RECONNECT_RETRY_DELAYS_MS[attempt]);
|
|
435
|
+
await this.ensureOpen();
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
throw lastError;
|
|
439
|
+
}
|
|
392
440
|
/**
|
|
393
441
|
* Returns approximate payload bytes currently stored in this browser-backed
|
|
394
442
|
* content-addressed store.
|
|
@@ -400,131 +448,139 @@ class IndexedDBContentAddressedStore {
|
|
|
400
448
|
* increase `refCount`.
|
|
401
449
|
*/
|
|
402
450
|
async getStoredBytes() {
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
451
|
+
return this.runWithReconnect("getStoredBytes", async () => {
|
|
452
|
+
const db = await this.ensureOpen();
|
|
453
|
+
const tx = db.transaction(META_STORE, "readonly");
|
|
454
|
+
const metaOS = tx.objectStore(META_STORE);
|
|
455
|
+
const contentBytes = (await readNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY)) ?? 0;
|
|
456
|
+
const metadataBytes = (await readNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY)) ?? 0;
|
|
457
|
+
return {
|
|
458
|
+
contentBytes,
|
|
459
|
+
metadataBytes,
|
|
460
|
+
totalBytes: contentBytes + metadataBytes,
|
|
461
|
+
};
|
|
462
|
+
});
|
|
413
463
|
}
|
|
414
464
|
async putEntries(entries) {
|
|
415
465
|
if (entries.length === 0)
|
|
416
466
|
return;
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
existingContent
|
|
448
|
-
|
|
449
|
-
|
|
467
|
+
return this.runWithReconnect("putEntries", async () => {
|
|
468
|
+
const db = await this.ensureOpen();
|
|
469
|
+
const tx = db.transaction([ENTRIES_STORE, CONTENT_STORE, BLOOM_CACHE_STORE, META_STORE], "readwrite");
|
|
470
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
471
|
+
const contentOS = tx.objectStore(CONTENT_STORE);
|
|
472
|
+
const bloomOS = tx.objectStore(BLOOM_CACHE_STORE);
|
|
473
|
+
const metaOS = tx.objectStore(META_STORE);
|
|
474
|
+
const insertedIds = [];
|
|
475
|
+
const counterRecord = (await reqToPromise(metaOS.get(RECEIPT_ORDER_COUNTER_KEY)));
|
|
476
|
+
let nextReceiptOrder = typeof counterRecord?.value === "number" ? counterRecord.value + 1 : 1;
|
|
477
|
+
let totalContentBytes = (await readNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY)) ?? 0;
|
|
478
|
+
let totalMetadataBytes = (await readNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY)) ?? 0;
|
|
479
|
+
for (const entry of entries) {
|
|
480
|
+
// Check if entry already exists (skip duplicate)
|
|
481
|
+
const existing = await reqToPromise(entriesOS.get(entry.id));
|
|
482
|
+
if (existing) {
|
|
483
|
+
this.logger.debug(`Entry ${entry.id} already exists, skipping`);
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
// Separate metadata from encrypted data
|
|
487
|
+
const { encryptedData, receiptOrder: _ignoredReceiptOrder, ...rest } = entry;
|
|
488
|
+
const metadata = {
|
|
489
|
+
...rest,
|
|
490
|
+
receiptOrder: nextReceiptOrder++,
|
|
491
|
+
};
|
|
492
|
+
totalMetadataBytes += getMetadataPayloadBytes(metadata);
|
|
493
|
+
// Store metadata
|
|
494
|
+
await reqToPromise(entriesOS.put(metadata));
|
|
495
|
+
// Deduplicate content
|
|
496
|
+
const existingContent = (await reqToPromise(contentOS.get(entry.contentHash)));
|
|
497
|
+
if (existingContent) {
|
|
498
|
+
existingContent.refCount += 1;
|
|
499
|
+
await reqToPromise(contentOS.put(existingContent));
|
|
500
|
+
this.logger.debug(`Content ${entry.contentHash.substring(0, 8)}... already exists (deduplicated)`);
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
await reqToPromise(contentOS.put({
|
|
504
|
+
contentHash: entry.contentHash,
|
|
505
|
+
data: encryptedData,
|
|
506
|
+
refCount: 1,
|
|
507
|
+
}));
|
|
508
|
+
totalContentBytes += encryptedData.byteLength;
|
|
509
|
+
this.logger.debug(`Stored content for hash ${entry.contentHash.substring(0, 8)}...`);
|
|
510
|
+
}
|
|
511
|
+
insertedIds.push(entry.id);
|
|
512
|
+
this.logger.debug(`Stored entry ${entry.id} for doc ${entry.docId}`);
|
|
450
513
|
}
|
|
451
|
-
|
|
452
|
-
await
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
refCount: 1,
|
|
456
|
-
}));
|
|
457
|
-
totalContentBytes += encryptedData.byteLength;
|
|
458
|
-
this.logger.debug(`Stored content for hash ${entry.contentHash.substring(0, 8)}...`);
|
|
514
|
+
if (insertedIds.length > 0) {
|
|
515
|
+
await writeNumericMetaValue(metaOS, RECEIPT_ORDER_COUNTER_KEY, nextReceiptOrder - 1);
|
|
516
|
+
await writeNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY, totalContentBytes);
|
|
517
|
+
await writeNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY, totalMetadataBytes);
|
|
459
518
|
}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
key: BLOOM_CACHE_KEY,
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
}));
|
|
480
|
-
this.logger.debug(`Incrementally updated bloom cache (+${insertedIds.length} IDs, total ${updated.totalIds})`);
|
|
519
|
+
// Incrementally update bloom cache with newly inserted IDs
|
|
520
|
+
if (insertedIds.length > 0) {
|
|
521
|
+
const cached = await reqToPromise(bloomOS.get(BLOOM_CACHE_KEY));
|
|
522
|
+
if (cached && !cached.dirty && cached.summary) {
|
|
523
|
+
// Try incremental update (returns null if rebuild is needed)
|
|
524
|
+
const updated = incrementalBloomUpdate(cached.summary, insertedIds);
|
|
525
|
+
if (updated) {
|
|
526
|
+
await reqToPromise(bloomOS.put({
|
|
527
|
+
key: BLOOM_CACHE_KEY,
|
|
528
|
+
summary: updated,
|
|
529
|
+
dirty: false,
|
|
530
|
+
}));
|
|
531
|
+
this.logger.debug(`Incrementally updated bloom cache (+${insertedIds.length} IDs, total ${updated.totalIds})`);
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
// Filter has grown past rebuild threshold; mark dirty for full recompute
|
|
535
|
+
await reqToPromise(bloomOS.put({ key: BLOOM_CACHE_KEY, summary: null, dirty: true }));
|
|
536
|
+
this.logger.debug("Bloom cache exceeded rebuild threshold, marked dirty");
|
|
537
|
+
}
|
|
481
538
|
}
|
|
482
539
|
else {
|
|
483
|
-
//
|
|
540
|
+
// No valid cache exists; mark dirty so next getIdBloomSummary() rebuilds
|
|
484
541
|
await reqToPromise(bloomOS.put({ key: BLOOM_CACHE_KEY, summary: null, dirty: true }));
|
|
485
|
-
this.logger.debug("Bloom cache exceeded rebuild threshold, marked dirty");
|
|
486
542
|
}
|
|
487
543
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
await reqToPromise(bloomOS.put({ key: BLOOM_CACHE_KEY, summary: null, dirty: true }));
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
await txToPromise(tx);
|
|
544
|
+
await txToPromise(tx);
|
|
545
|
+
});
|
|
494
546
|
}
|
|
495
547
|
async getEntries(ids) {
|
|
496
548
|
if (ids.length === 0)
|
|
497
549
|
return [];
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
const
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
550
|
+
return this.runWithReconnect("getEntries", async () => {
|
|
551
|
+
const db = await this.ensureOpen();
|
|
552
|
+
const tx = db.transaction([ENTRIES_STORE, CONTENT_STORE], "readonly");
|
|
553
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
554
|
+
const contentOS = tx.objectStore(CONTENT_STORE);
|
|
555
|
+
const result = [];
|
|
556
|
+
for (const id of ids) {
|
|
557
|
+
const metadata = (await reqToPromise(entriesOS.get(id)));
|
|
558
|
+
if (!metadata) {
|
|
559
|
+
this.logger.warn(`Entry ${id} not found`);
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
const contentRecord = await reqToPromise(contentOS.get(metadata.contentHash));
|
|
563
|
+
if (!contentRecord) {
|
|
564
|
+
this.logger.warn(`Content ${metadata.contentHash} not found for entry ${id}`);
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
result.push({
|
|
568
|
+
...metadata,
|
|
569
|
+
encryptedData: new Uint8Array(contentRecord.data),
|
|
570
|
+
});
|
|
513
571
|
}
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
});
|
|
518
|
-
}
|
|
519
|
-
this.logger.debug(`Retrieved ${result.length} entries out of ${ids.length} requested`);
|
|
520
|
-
return result;
|
|
572
|
+
this.logger.debug(`Retrieved ${result.length} entries out of ${ids.length} requested`);
|
|
573
|
+
return result;
|
|
574
|
+
});
|
|
521
575
|
}
|
|
522
576
|
async getEntryMetadata(id) {
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
577
|
+
return this.runWithReconnect("getEntryMetadata", async () => {
|
|
578
|
+
const db = await this.ensureOpen();
|
|
579
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
580
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
581
|
+
const metadata = (await reqToPromise(entriesOS.get(id)));
|
|
582
|
+
return metadata ?? null;
|
|
583
|
+
});
|
|
528
584
|
}
|
|
529
585
|
async planAttachmentReadByWalkingMetadata(lastChunkId, attachmentSize, options) {
|
|
530
586
|
return (0, AttachmentReadPlanner_1.planAttachmentReadByWalkingMetadata)(this, lastChunkId, attachmentSize, options);
|
|
@@ -532,304 +588,339 @@ class IndexedDBContentAddressedStore {
|
|
|
532
588
|
async hasEntries(ids) {
|
|
533
589
|
if (ids.length === 0)
|
|
534
590
|
return [];
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
const
|
|
541
|
-
|
|
542
|
-
|
|
591
|
+
return this.runWithReconnect("hasEntries", async () => {
|
|
592
|
+
const db = await this.ensureOpen();
|
|
593
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
594
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
595
|
+
const existing = [];
|
|
596
|
+
for (const id of ids) {
|
|
597
|
+
const key = await reqToPromise(entriesOS.getKey(id));
|
|
598
|
+
if (key !== undefined) {
|
|
599
|
+
existing.push(id);
|
|
600
|
+
}
|
|
543
601
|
}
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
602
|
+
this.logger.debug(`Found ${existing.length} existing entries out of ${ids.length} checked`);
|
|
603
|
+
return existing;
|
|
604
|
+
});
|
|
547
605
|
}
|
|
548
606
|
async findNewEntries(knownIds) {
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
607
|
+
return this.runWithReconnect("findNewEntries", async () => {
|
|
608
|
+
const knownSet = new Set(knownIds);
|
|
609
|
+
const db = await this.ensureOpen();
|
|
610
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
611
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
612
|
+
const newEntries = [];
|
|
613
|
+
return new Promise((resolve, reject) => {
|
|
614
|
+
const cursorReq = entriesOS.openCursor();
|
|
615
|
+
cursorReq.onsuccess = () => {
|
|
616
|
+
const cursor = cursorReq.result;
|
|
617
|
+
if (!cursor) {
|
|
618
|
+
this.logger.debug(`Found ${newEntries.length} new entries out of total`);
|
|
619
|
+
resolve(newEntries);
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
const metadata = cursor.value;
|
|
623
|
+
if (!knownSet.has(metadata.id)) {
|
|
624
|
+
newEntries.push(metadata);
|
|
625
|
+
}
|
|
626
|
+
cursor.continue();
|
|
627
|
+
};
|
|
628
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
629
|
+
});
|
|
570
630
|
});
|
|
571
631
|
}
|
|
572
632
|
async findNewEntriesForDoc(knownIds, docId) {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
633
|
+
return this.runWithReconnect("findNewEntriesForDoc", async () => {
|
|
634
|
+
const knownSet = new Set(knownIds);
|
|
635
|
+
const db = await this.ensureOpen();
|
|
636
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
637
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
638
|
+
const index = entriesOS.index(IDX_DOC_ID);
|
|
639
|
+
const newEntries = [];
|
|
640
|
+
return new Promise((resolve, reject) => {
|
|
641
|
+
const cursorReq = index.openCursor(IDBKeyRange.only(docId));
|
|
642
|
+
cursorReq.onsuccess = () => {
|
|
643
|
+
const cursor = cursorReq.result;
|
|
644
|
+
if (!cursor) {
|
|
645
|
+
this.logger.debug(`Found ${newEntries.length} new entries for doc ${docId}`);
|
|
646
|
+
resolve(newEntries);
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
const metadata = cursor.value;
|
|
650
|
+
if (!knownSet.has(metadata.id)) {
|
|
651
|
+
newEntries.push(metadata);
|
|
652
|
+
}
|
|
653
|
+
cursor.continue();
|
|
654
|
+
};
|
|
655
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
656
|
+
});
|
|
595
657
|
});
|
|
596
658
|
}
|
|
597
659
|
async findEntries(type, creationDateFrom, creationDateUntil) {
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
660
|
+
return this.runWithReconnect("findEntries", async () => {
|
|
661
|
+
const db = await this.ensureOpen();
|
|
662
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
663
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
664
|
+
const index = entriesOS.index(IDX_ENTRY_TYPE_CREATED_AT);
|
|
665
|
+
// Build key range on [entryType, createdAt]
|
|
666
|
+
const lower = [type, creationDateFrom ?? -Infinity];
|
|
667
|
+
const upper = [type, creationDateUntil ?? Infinity];
|
|
668
|
+
const range = IDBKeyRange.bound(lower, upper, false, // lowerOpen: inclusive
|
|
669
|
+
creationDateUntil !== null // upperOpen: exclusive when creationDateUntil is set
|
|
670
|
+
);
|
|
671
|
+
const results = [];
|
|
672
|
+
return new Promise((resolve, reject) => {
|
|
673
|
+
const cursorReq = index.openCursor(range);
|
|
674
|
+
cursorReq.onsuccess = () => {
|
|
675
|
+
const cursor = cursorReq.result;
|
|
676
|
+
if (!cursor) {
|
|
677
|
+
this.logger.debug(`Found ${results.length} entries of type ${type} in date range`);
|
|
678
|
+
resolve(results);
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
results.push(cursor.value);
|
|
682
|
+
cursor.continue();
|
|
683
|
+
};
|
|
684
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
685
|
+
});
|
|
622
686
|
});
|
|
623
687
|
}
|
|
624
688
|
async getAllIds() {
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
689
|
+
return this.runWithReconnect("getAllIds", async () => {
|
|
690
|
+
const db = await this.ensureOpen();
|
|
691
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
692
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
693
|
+
const keys = [];
|
|
694
|
+
return new Promise((resolve, reject) => {
|
|
695
|
+
const cursorReq = entriesOS.openKeyCursor();
|
|
696
|
+
cursorReq.onsuccess = () => {
|
|
697
|
+
const cursor = cursorReq.result;
|
|
698
|
+
if (!cursor) {
|
|
699
|
+
this.logger.debug(`Returning ${keys.length} entry IDs`);
|
|
700
|
+
resolve(keys);
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
703
|
+
keys.push(String(cursor.primaryKey));
|
|
704
|
+
cursor.continue();
|
|
705
|
+
};
|
|
706
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
707
|
+
});
|
|
642
708
|
});
|
|
643
709
|
}
|
|
644
710
|
async scanEntriesSince(cursor, limit = Number.MAX_SAFE_INTEGER, filters) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
711
|
+
return this.runWithReconnect("scanEntriesSince", async () => {
|
|
712
|
+
const db = await this.ensureOpen();
|
|
713
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
714
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
715
|
+
const index = entriesOS.index(IDX_RECEIPT_ORDER_ID);
|
|
716
|
+
// Build lower bound from cursor
|
|
717
|
+
let range = null;
|
|
718
|
+
if (cursor) {
|
|
719
|
+
range = IDBKeyRange.lowerBound([cursor.receiptOrder, cursor.id], true);
|
|
720
|
+
}
|
|
721
|
+
const page = [];
|
|
722
|
+
return new Promise((resolve, reject) => {
|
|
723
|
+
const cursorReq = index.openCursor(range);
|
|
724
|
+
let hasMore = false;
|
|
725
|
+
cursorReq.onsuccess = () => {
|
|
726
|
+
const idbCursor = cursorReq.result;
|
|
727
|
+
if (!idbCursor) {
|
|
728
|
+
const last = page.length > 0 ? page[page.length - 1] : null;
|
|
729
|
+
resolve({
|
|
730
|
+
entries: page,
|
|
731
|
+
nextCursor: last
|
|
732
|
+
? { receiptOrder: last.receiptOrder ?? 0, id: last.id }
|
|
733
|
+
: cursor,
|
|
734
|
+
hasMore,
|
|
735
|
+
});
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
const meta = idbCursor.value;
|
|
739
|
+
if (page.length >= limit) {
|
|
740
|
+
// We already have enough entries. Check if this one matches filters
|
|
741
|
+
// to determine hasMore.
|
|
742
|
+
if (this.matchesScanFilters(meta, filters)) {
|
|
743
|
+
hasMore = true;
|
|
744
|
+
}
|
|
745
|
+
const last = page[page.length - 1];
|
|
746
|
+
resolve({
|
|
747
|
+
entries: page,
|
|
748
|
+
nextCursor: { receiptOrder: last.receiptOrder ?? 0, id: last.id },
|
|
749
|
+
hasMore: true,
|
|
750
|
+
});
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
675
753
|
if (this.matchesScanFilters(meta, filters)) {
|
|
676
|
-
|
|
754
|
+
page.push(meta);
|
|
677
755
|
}
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
hasMore: true,
|
|
683
|
-
});
|
|
684
|
-
return;
|
|
685
|
-
}
|
|
686
|
-
if (this.matchesScanFilters(meta, filters)) {
|
|
687
|
-
page.push(meta);
|
|
688
|
-
}
|
|
689
|
-
idbCursor.continue();
|
|
690
|
-
};
|
|
691
|
-
cursorReq.onerror = () => reject(cursorReq.error);
|
|
756
|
+
idbCursor.continue();
|
|
757
|
+
};
|
|
758
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
759
|
+
});
|
|
692
760
|
});
|
|
693
761
|
}
|
|
694
762
|
async getIdBloomSummary() {
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
763
|
+
return this.runWithReconnect("getIdBloomSummary", async () => {
|
|
764
|
+
const db = await this.ensureOpen();
|
|
765
|
+
// Try to read from cache first
|
|
766
|
+
const readTx = db.transaction(BLOOM_CACHE_STORE, "readonly");
|
|
767
|
+
const bloomOS = readTx.objectStore(BLOOM_CACHE_STORE);
|
|
768
|
+
const cached = await reqToPromise(bloomOS.get(BLOOM_CACHE_KEY));
|
|
769
|
+
if (cached && !cached.dirty && cached.summary) {
|
|
770
|
+
this.logger.debug("Returning cached bloom summary");
|
|
771
|
+
return cached.summary;
|
|
772
|
+
}
|
|
773
|
+
// Recompute bloom summary
|
|
774
|
+
this.logger.debug("Recomputing bloom summary");
|
|
775
|
+
const ids = await this.getAllIds();
|
|
776
|
+
const summary = createIdBloomSummaryBrowser(ids);
|
|
777
|
+
// Persist to cache
|
|
778
|
+
const writeTx = db.transaction(BLOOM_CACHE_STORE, "readwrite");
|
|
779
|
+
const writeBloomOS = writeTx.objectStore(BLOOM_CACHE_STORE);
|
|
780
|
+
await reqToPromise(writeBloomOS.put({
|
|
781
|
+
key: BLOOM_CACHE_KEY,
|
|
782
|
+
summary,
|
|
783
|
+
dirty: false,
|
|
784
|
+
}));
|
|
785
|
+
await txToPromise(writeTx);
|
|
786
|
+
return summary;
|
|
787
|
+
});
|
|
718
788
|
}
|
|
719
789
|
async planDocumentMaterialization(docId, options) {
|
|
720
|
-
|
|
721
|
-
|
|
790
|
+
return this.runWithReconnect("planDocumentMaterialization", async () => {
|
|
791
|
+
const entries = await this.readEntriesForDoc(docId);
|
|
792
|
+
return (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, entries, options);
|
|
793
|
+
});
|
|
722
794
|
}
|
|
723
795
|
async planDocumentMaterializationBatch(docIds, options) {
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
796
|
+
return this.runWithReconnect("planDocumentMaterializationBatch", async () => {
|
|
797
|
+
const all = [];
|
|
798
|
+
// Batch doc-scoped index reads so browser planning stays proportional to
|
|
799
|
+
// the requested documents instead of loading the entire entries store.
|
|
800
|
+
const concurrency = 16;
|
|
801
|
+
for (let offset = 0; offset < docIds.length; offset += concurrency) {
|
|
802
|
+
const batch = docIds.slice(offset, offset + concurrency);
|
|
803
|
+
const batchResults = await Promise.all(batch.map((docId) => this.readEntriesForDoc(docId)));
|
|
804
|
+
for (const entries of batchResults) {
|
|
805
|
+
all.push(...entries);
|
|
806
|
+
}
|
|
733
807
|
}
|
|
734
|
-
|
|
735
|
-
|
|
808
|
+
return (0, MaterializationPlanner_1.computeBatchMaterializationPlan)(all, docIds, options);
|
|
809
|
+
});
|
|
736
810
|
}
|
|
737
811
|
async resolveDependencies(startId, options) {
|
|
738
812
|
const stopAtEntryType = options?.stopAtEntryType;
|
|
739
813
|
const maxDepth = options?.maxDepth;
|
|
740
814
|
const includeStart = options?.includeStart !== false;
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
815
|
+
return this.runWithReconnect("resolveDependencies", async () => {
|
|
816
|
+
const db = await this.ensureOpen();
|
|
817
|
+
const tx = db.transaction(ENTRIES_STORE, "readonly");
|
|
818
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
819
|
+
const result = [];
|
|
820
|
+
const visited = new Set();
|
|
821
|
+
const queue = [
|
|
822
|
+
{ id: startId, depth: 0 },
|
|
823
|
+
];
|
|
824
|
+
let queueIdx = 0;
|
|
825
|
+
while (queueIdx < queue.length) {
|
|
826
|
+
const { id, depth } = queue[queueIdx++];
|
|
827
|
+
if (visited.has(id))
|
|
828
|
+
continue;
|
|
829
|
+
visited.add(id);
|
|
830
|
+
if (maxDepth !== undefined && depth > maxDepth)
|
|
831
|
+
continue;
|
|
832
|
+
const entry = (await reqToPromise(entriesOS.get(id)));
|
|
833
|
+
if (!entry) {
|
|
834
|
+
this.logger.warn(`Entry ${id} not found during dependency resolution`);
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
if (stopAtEntryType && entry.entryType === stopAtEntryType && id !== startId) {
|
|
838
|
+
result.push(id);
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
if (id !== startId || includeStart) {
|
|
842
|
+
result.push(id);
|
|
843
|
+
}
|
|
844
|
+
for (const depId of entry.dependencyIds) {
|
|
845
|
+
if (!visited.has(depId)) {
|
|
846
|
+
queue.push({ id: depId, depth: depth + 1 });
|
|
847
|
+
}
|
|
772
848
|
}
|
|
773
849
|
}
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
850
|
+
result.reverse();
|
|
851
|
+
this.logger.debug(`Resolved ${result.length} dependencies for ${startId}`);
|
|
852
|
+
return result;
|
|
853
|
+
});
|
|
778
854
|
}
|
|
779
855
|
async purgeDocHistory(docId) {
|
|
780
856
|
this.logger.info(`Purging entry history for document: ${docId}`);
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
857
|
+
return this.runWithReconnect("purgeDocHistory", async () => {
|
|
858
|
+
const db = await this.ensureOpen();
|
|
859
|
+
const tx = db.transaction([ENTRIES_STORE, CONTENT_STORE, BLOOM_CACHE_STORE, META_STORE], "readwrite");
|
|
860
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
861
|
+
const contentOS = tx.objectStore(CONTENT_STORE);
|
|
862
|
+
const bloomOS = tx.objectStore(BLOOM_CACHE_STORE);
|
|
863
|
+
const metaOS = tx.objectStore(META_STORE);
|
|
864
|
+
const docIndex = entriesOS.index(IDX_DOC_ID);
|
|
865
|
+
let totalContentBytes = (await readNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY)) ?? 0;
|
|
866
|
+
let totalMetadataBytes = (await readNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY)) ?? 0;
|
|
867
|
+
// Collect all entries for this docId
|
|
868
|
+
const entriesToPurge = [];
|
|
869
|
+
await new Promise((resolve, reject) => {
|
|
870
|
+
const cursorReq = docIndex.openCursor(IDBKeyRange.only(docId));
|
|
871
|
+
cursorReq.onsuccess = () => {
|
|
872
|
+
const cursor = cursorReq.result;
|
|
873
|
+
if (!cursor) {
|
|
874
|
+
resolve();
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
877
|
+
entriesToPurge.push(cursor.value);
|
|
878
|
+
cursor.continue();
|
|
879
|
+
};
|
|
880
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
881
|
+
});
|
|
882
|
+
if (entriesToPurge.length === 0) {
|
|
883
|
+
this.logger.debug(`No entries found for document ${docId}, nothing to purge`);
|
|
884
|
+
return;
|
|
885
|
+
}
|
|
886
|
+
// Delete entries and decrement content ref counts
|
|
887
|
+
for (const entry of entriesToPurge) {
|
|
888
|
+
await reqToPromise(entriesOS.delete(entry.id));
|
|
889
|
+
totalMetadataBytes -= getMetadataPayloadBytes(entry);
|
|
890
|
+
const contentRecord = (await reqToPromise(contentOS.get(entry.contentHash)));
|
|
891
|
+
if (contentRecord) {
|
|
892
|
+
const newCount = contentRecord.refCount - 1;
|
|
893
|
+
if (newCount <= 0) {
|
|
894
|
+
await reqToPromise(contentOS.delete(entry.contentHash));
|
|
895
|
+
totalContentBytes -= contentRecord.data.byteLength;
|
|
896
|
+
this.logger.debug(`Cleaned up orphaned content ${entry.contentHash.substring(0, 8)}...`);
|
|
897
|
+
}
|
|
898
|
+
else {
|
|
899
|
+
contentRecord.refCount = newCount;
|
|
900
|
+
await reqToPromise(contentOS.put(contentRecord));
|
|
901
|
+
}
|
|
824
902
|
}
|
|
825
903
|
}
|
|
904
|
+
// Invalidate bloom cache
|
|
905
|
+
await reqToPromise(bloomOS.put({ key: BLOOM_CACHE_KEY, summary: null, dirty: true }));
|
|
906
|
+
await writeNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY, Math.max(0, totalContentBytes));
|
|
907
|
+
await writeNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY, Math.max(0, totalMetadataBytes));
|
|
908
|
+
await txToPromise(tx);
|
|
909
|
+
this.logger.info(`Purged ${entriesToPurge.length} entries for document ${docId}`);
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
async deleteEntriesForAttachment(docId, attachmentId) {
|
|
913
|
+
const prefix = `${docId}_a_${attachmentId}_`;
|
|
914
|
+
return this.deleteEntriesWhere(`deleteEntriesForAttachment:${attachmentId}`, (entry) => entry.docId === docId
|
|
915
|
+
&& entry.entryType === "attachment_chunk"
|
|
916
|
+
&& (entry.attachmentId === attachmentId || entry.id.startsWith(prefix)));
|
|
917
|
+
}
|
|
918
|
+
async deleteEntriesById(entryIds) {
|
|
919
|
+
if (entryIds.length === 0) {
|
|
920
|
+
return 0;
|
|
826
921
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
await writeNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY, Math.max(0, totalContentBytes));
|
|
830
|
-
await writeNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY, Math.max(0, totalMetadataBytes));
|
|
831
|
-
await txToPromise(tx);
|
|
832
|
-
this.logger.info(`Purged ${entriesToPurge.length} entries for document ${docId}`);
|
|
922
|
+
const ids = new Set(entryIds);
|
|
923
|
+
return this.deleteEntriesWhere("deleteEntriesById", (entry) => ids.has(entry.id));
|
|
833
924
|
}
|
|
834
925
|
async clearAllLocalData() {
|
|
835
926
|
this.logger.info(`Clearing all local data for store ${this.storeId}`);
|
|
@@ -888,6 +979,61 @@ class IndexedDBContentAddressedStore {
|
|
|
888
979
|
}
|
|
889
980
|
return true;
|
|
890
981
|
}
|
|
982
|
+
async deleteEntriesWhere(opName, predicate) {
|
|
983
|
+
return this.runWithReconnect(opName, async () => {
|
|
984
|
+
const db = await this.ensureOpen();
|
|
985
|
+
const tx = db.transaction([ENTRIES_STORE, CONTENT_STORE, BLOOM_CACHE_STORE, META_STORE], "readwrite");
|
|
986
|
+
const entriesOS = tx.objectStore(ENTRIES_STORE);
|
|
987
|
+
const contentOS = tx.objectStore(CONTENT_STORE);
|
|
988
|
+
const bloomOS = tx.objectStore(BLOOM_CACHE_STORE);
|
|
989
|
+
const metaOS = tx.objectStore(META_STORE);
|
|
990
|
+
let totalContentBytes = (await readNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY)) ?? 0;
|
|
991
|
+
let totalMetadataBytes = (await readNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY)) ?? 0;
|
|
992
|
+
let removedCount = 0;
|
|
993
|
+
await new Promise((resolve, reject) => {
|
|
994
|
+
const cursorReq = entriesOS.openCursor();
|
|
995
|
+
cursorReq.onsuccess = async () => {
|
|
996
|
+
const cursor = cursorReq.result;
|
|
997
|
+
if (!cursor) {
|
|
998
|
+
resolve();
|
|
999
|
+
return;
|
|
1000
|
+
}
|
|
1001
|
+
try {
|
|
1002
|
+
const entry = cursor.value;
|
|
1003
|
+
if (predicate(entry)) {
|
|
1004
|
+
await reqToPromise(cursor.delete());
|
|
1005
|
+
totalMetadataBytes -= getMetadataPayloadBytes(entry);
|
|
1006
|
+
const contentRecord = (await reqToPromise(contentOS.get(entry.contentHash)));
|
|
1007
|
+
if (contentRecord) {
|
|
1008
|
+
const nextRefCount = contentRecord.refCount - 1;
|
|
1009
|
+
if (nextRefCount <= 0) {
|
|
1010
|
+
await reqToPromise(contentOS.delete(entry.contentHash));
|
|
1011
|
+
totalContentBytes -= contentRecord.data.byteLength;
|
|
1012
|
+
}
|
|
1013
|
+
else {
|
|
1014
|
+
contentRecord.refCount = nextRefCount;
|
|
1015
|
+
await reqToPromise(contentOS.put(contentRecord));
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
removedCount++;
|
|
1019
|
+
}
|
|
1020
|
+
cursor.continue();
|
|
1021
|
+
}
|
|
1022
|
+
catch (error) {
|
|
1023
|
+
reject(error);
|
|
1024
|
+
}
|
|
1025
|
+
};
|
|
1026
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
1027
|
+
});
|
|
1028
|
+
if (removedCount > 0) {
|
|
1029
|
+
await reqToPromise(bloomOS.put({ key: BLOOM_CACHE_KEY, summary: null, dirty: true }));
|
|
1030
|
+
await writeNumericMetaValue(metaOS, TOTAL_CONTENT_BYTES_KEY, Math.max(0, totalContentBytes));
|
|
1031
|
+
await writeNumericMetaValue(metaOS, TOTAL_METADATA_BYTES_KEY, Math.max(0, totalMetadataBytes));
|
|
1032
|
+
}
|
|
1033
|
+
await txToPromise(tx);
|
|
1034
|
+
return removedCount;
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
891
1037
|
async readEntriesForDoc(docId) {
|
|
892
1038
|
const db = await this.ensureOpen();
|
|
893
1039
|
const tx = db.transaction(ENTRIES_STORE, "readonly");
|