mindoodb 0.0.38 → 0.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.d.ts.map +1 -1
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.js +22 -0
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.js.map +1 -1
- package/dist/core/BaseMindooDB.d.ts +85 -0
- package/dist/core/BaseMindooDB.d.ts.map +1 -1
- package/dist/core/BaseMindooDB.js +385 -21
- package/dist/core/BaseMindooDB.js.map +1 -1
- package/dist/core/BaseMindooTenant.d.ts +28 -0
- package/dist/core/BaseMindooTenant.d.ts.map +1 -1
- package/dist/core/BaseMindooTenant.js +59 -0
- package/dist/core/BaseMindooTenant.js.map +1 -1
- package/dist/core/BaseMindooTenantDirectory.d.ts +9 -0
- package/dist/core/BaseMindooTenantDirectory.d.ts.map +1 -1
- package/dist/core/BaseMindooTenantDirectory.js +34 -0
- package/dist/core/BaseMindooTenantDirectory.js.map +1 -1
- package/dist/core/types.d.ts +19 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.d.ts +27 -0
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.d.ts.map +1 -1
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.js +35 -0
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.js.map +1 -1
- package/package.json +1 -1
|
@@ -229,6 +229,27 @@ class BaseMindooDB {
|
|
|
229
229
|
// Cache for imported public keys (CryptoKey objects) to avoid re-importing the same key
|
|
230
230
|
// Map<publicKeyPEM, CryptoKey>
|
|
231
231
|
this.publicKeyCache = new Map();
|
|
232
|
+
/**
|
|
233
|
+
* Author signing keys whose entries were skipped during materialization
|
|
234
|
+
* because the key was not (yet) trusted by the tenant directory, mapped to
|
|
235
|
+
* the ids of the affected documents.
|
|
236
|
+
*
|
|
237
|
+
* This is the record that makes skipped entries recoverable: when the
|
|
238
|
+
* directory later learns about the author (a `grantaccess` document arrives
|
|
239
|
+
* on sync), {@link reconcileAuthorTrust} purges and force-re-emits exactly
|
|
240
|
+
* these documents so they re-materialize with the now-trusted entries and
|
|
241
|
+
* every `iterateChangesSince` consumer (views, summary, full-text) catches
|
|
242
|
+
* up. Persisted in the metadata checkpoint so the healing also works when
|
|
243
|
+
* the directory is synced in a later session while this DB is closed.
|
|
244
|
+
*/
|
|
245
|
+
this.pendingUntrustedAuthors = new Map();
|
|
246
|
+
/**
|
|
247
|
+
* Set while restoring a metadata checkpoint that predates the
|
|
248
|
+
* `pendingUntrustedAuthors` field. Triggers a one-time heal scan at open
|
|
249
|
+
* ({@link reconcileAuthorTrustAtOpen}) that detects store entries missing
|
|
250
|
+
* from the cached materialization and seeds the pending map from them.
|
|
251
|
+
*/
|
|
252
|
+
this.legacyCheckpointNeedsAuthorTrustScan = false;
|
|
232
253
|
// Local cache support
|
|
233
254
|
this.cacheManager = null;
|
|
234
255
|
this.cachePrefix = null;
|
|
@@ -326,6 +347,12 @@ class BaseMindooDB {
|
|
|
326
347
|
* the warmer settled can still display its outcome.
|
|
327
348
|
*/
|
|
328
349
|
this.warmerProgress = null;
|
|
350
|
+
// ---------------------------------------------------------------------------
|
|
351
|
+
// Author-trust reconciliation: heal documents whose entries were skipped
|
|
352
|
+
// because their author was not yet known to the tenant directory.
|
|
353
|
+
// ---------------------------------------------------------------------------
|
|
354
|
+
/** Serializes concurrent author-trust reconciles (per database). */
|
|
355
|
+
this.authorTrustReconcileChain = Promise.resolve();
|
|
329
356
|
(0, databaseIdValidation_1.validateDatabaseId)(store.getId(), "dbId");
|
|
330
357
|
(0, databaseIdValidation_1.validateDatabaseId)(attachmentStore.getId(), "dbId");
|
|
331
358
|
if (store.getId() !== attachmentStore.getId()) {
|
|
@@ -1191,6 +1218,18 @@ class BaseMindooDB {
|
|
|
1191
1218
|
if (this.lastReconciledKeyBagFingerprint !== null) {
|
|
1192
1219
|
checkpoint.keyBagFingerprint = this.lastReconciledKeyBagFingerprint;
|
|
1193
1220
|
}
|
|
1221
|
+
// Persist the pending untrusted-author map (additive field). Written even
|
|
1222
|
+
// when empty: the field's PRESENCE marks the checkpoint as
|
|
1223
|
+
// author-trust-aware, so restore can distinguish it from a legacy
|
|
1224
|
+
// checkpoint that needs the one-time heal scan (see
|
|
1225
|
+
// reconcileAuthorTrustAtOpen).
|
|
1226
|
+
const pendingUntrusted = {};
|
|
1227
|
+
for (const [publicKey, docIds] of this.pendingUntrustedAuthors) {
|
|
1228
|
+
if (docIds.size > 0) {
|
|
1229
|
+
pendingUntrusted[publicKey] = Array.from(docIds);
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
checkpoint.pendingUntrustedAuthors = pendingUntrusted;
|
|
1194
1233
|
return new TextEncoder().encode(JSON.stringify(checkpoint));
|
|
1195
1234
|
}
|
|
1196
1235
|
/**
|
|
@@ -1219,6 +1258,24 @@ class BaseMindooDB {
|
|
|
1219
1258
|
}
|
|
1220
1259
|
this.lastReconciledKeyBagFingerprint =
|
|
1221
1260
|
typeof checkpoint.keyBagFingerprint === "string" ? checkpoint.keyBagFingerprint : null;
|
|
1261
|
+
// Pending untrusted-author map. A checkpoint written before this field
|
|
1262
|
+
// existed may hide entries that were silently skipped because their
|
|
1263
|
+
// author was unknown at the time - flag it for the one-time heal scan.
|
|
1264
|
+
this.pendingUntrustedAuthors.clear();
|
|
1265
|
+
if (checkpoint.pendingUntrustedAuthors && typeof checkpoint.pendingUntrustedAuthors === "object") {
|
|
1266
|
+
this.legacyCheckpointNeedsAuthorTrustScan = false;
|
|
1267
|
+
for (const [publicKey, docIds] of Object.entries(checkpoint.pendingUntrustedAuthors)) {
|
|
1268
|
+
if (!Array.isArray(docIds))
|
|
1269
|
+
continue;
|
|
1270
|
+
const validIds = docIds.filter((id) => typeof id === "string");
|
|
1271
|
+
if (validIds.length > 0) {
|
|
1272
|
+
this.pendingUntrustedAuthors.set(publicKey, new Set(validIds));
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
else {
|
|
1277
|
+
this.legacyCheckpointNeedsAuthorTrustScan = true;
|
|
1278
|
+
}
|
|
1222
1279
|
this.syncScanCursors.clear();
|
|
1223
1280
|
if (checkpoint.syncScanCursors && typeof checkpoint.syncScanCursors === "object") {
|
|
1224
1281
|
for (const [key, value] of Object.entries(checkpoint.syncScanCursors)) {
|
|
@@ -1410,6 +1467,28 @@ class BaseMindooDB {
|
|
|
1410
1467
|
this.publicKeyCache.set(publicKey, cryptoKey);
|
|
1411
1468
|
return cryptoKey;
|
|
1412
1469
|
}
|
|
1470
|
+
/**
|
|
1471
|
+
* Remember that entries of `publicKey` were skipped for `docId` because the
|
|
1472
|
+
* key is not (yet) trusted by the tenant directory. The record is persisted
|
|
1473
|
+
* in the metadata checkpoint and consumed by {@link reconcileAuthorTrust}
|
|
1474
|
+
* once the directory later trusts the key (e.g. a grantaccess document
|
|
1475
|
+
* arrived on a delayed directory sync), so the skipped entries get
|
|
1476
|
+
* re-materialized instead of staying invisible forever.
|
|
1477
|
+
*/
|
|
1478
|
+
recordPendingUntrustedAuthor(publicKey, docId) {
|
|
1479
|
+
let docIds = this.pendingUntrustedAuthors.get(publicKey);
|
|
1480
|
+
if (!docIds) {
|
|
1481
|
+
docIds = new Set();
|
|
1482
|
+
this.pendingUntrustedAuthors.set(publicKey, docIds);
|
|
1483
|
+
}
|
|
1484
|
+
if (docIds.has(docId)) {
|
|
1485
|
+
return;
|
|
1486
|
+
}
|
|
1487
|
+
docIds.add(docId);
|
|
1488
|
+
this.logger.info(`Recorded pending untrusted author for doc ${docId}: entries skipped until the directory trusts the key`);
|
|
1489
|
+
this.cacheMetaDirty = true;
|
|
1490
|
+
this.cacheManager?.markDirty();
|
|
1491
|
+
}
|
|
1413
1492
|
/**
|
|
1414
1493
|
* Compute the author's metadata-binding signature for a new entry (audit
|
|
1415
1494
|
* finding #5). The signature covers the canonical, versioned, length-prefixed
|
|
@@ -1818,6 +1897,12 @@ class BaseMindooDB {
|
|
|
1818
1897
|
if (this.reconcileRestoredIndexOnInit) {
|
|
1819
1898
|
await this.reconcileRestoredIndexWithStore();
|
|
1820
1899
|
}
|
|
1900
|
+
// Author-trust healing: entries skipped in an earlier session because
|
|
1901
|
+
// their author was unknown to the directory become materializable once
|
|
1902
|
+
// a grantaccess arrived while this DB was closed. Also runs the
|
|
1903
|
+
// one-time legacy heal scan for checkpoints that predate the
|
|
1904
|
+
// pending-untrusted-author tracking.
|
|
1905
|
+
await this.reconcileAuthorTrustAtOpen();
|
|
1821
1906
|
this.probeFulltextSetupAtOpen();
|
|
1822
1907
|
this.probeSummarySetupAtOpen();
|
|
1823
1908
|
return;
|
|
@@ -2091,6 +2176,190 @@ class BaseMindooDB {
|
|
|
2091
2176
|
// can skip the scan.
|
|
2092
2177
|
this.markKeyBagFingerprintReconciled(await this.computeCurrentKeyBagFingerprint());
|
|
2093
2178
|
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Re-materialize documents whose entries were previously skipped because the
|
|
2181
|
+
* author's signing key was not (yet) trusted by the tenant directory.
|
|
2182
|
+
*
|
|
2183
|
+
* This is the counterpart to {@link reconcileKeyVisibility} for AUTHOR trust
|
|
2184
|
+
* instead of decryption keys: when a `grantaccess` document arrives on a
|
|
2185
|
+
* delayed directory sync, entries that were quarantined/skipped earlier
|
|
2186
|
+
* (recorded in {@link pendingUntrustedAuthors}) become valid. For each
|
|
2187
|
+
* affected document we purge every cached materialization (L1/L2/summary/
|
|
2188
|
+
* full-text) and force a changeSeq bump on the index so all
|
|
2189
|
+
* `iterateChangesSince` consumers (virtual views, summary buffer, full-text
|
|
2190
|
+
* index) re-process the doc; the body itself re-materializes lazily on the
|
|
2191
|
+
* next load and now includes the previously skipped entries.
|
|
2192
|
+
*
|
|
2193
|
+
* Safe to call repeatedly; concurrent calls are serialized and a call with
|
|
2194
|
+
* no pending or no newly-trusted keys is a fast no-op. Errors are logged,
|
|
2195
|
+
* never thrown (best-effort hook semantics).
|
|
2196
|
+
*
|
|
2197
|
+
* @param newlyTrustedKeys Signing keys that just became trusted (directory
|
|
2198
|
+
* cache diff). When omitted, every pending key is re-checked against the
|
|
2199
|
+
* directory - used at DB open to catch trust changes that happened while
|
|
2200
|
+
* this database was closed.
|
|
2201
|
+
*/
|
|
2202
|
+
reconcileAuthorTrust(newlyTrustedKeys) {
|
|
2203
|
+
const run = () => this.reconcileAuthorTrustNow(newlyTrustedKeys).catch((error) => {
|
|
2204
|
+
this.logger.warn(`reconcileAuthorTrust failed: ${error}`);
|
|
2205
|
+
});
|
|
2206
|
+
const chained = this.authorTrustReconcileChain.then(run, run);
|
|
2207
|
+
this.authorTrustReconcileChain = chained;
|
|
2208
|
+
return chained;
|
|
2209
|
+
}
|
|
2210
|
+
async reconcileAuthorTrustNow(newlyTrustedKeys) {
|
|
2211
|
+
if (this.isTimeTravelMode()) {
|
|
2212
|
+
return;
|
|
2213
|
+
}
|
|
2214
|
+
if (this.pendingUntrustedAuthors.size === 0) {
|
|
2215
|
+
return;
|
|
2216
|
+
}
|
|
2217
|
+
const candidateKeys = Array.from(this.pendingUntrustedAuthors.keys()).filter((key) => !newlyTrustedKeys || newlyTrustedKeys.has(key));
|
|
2218
|
+
if (candidateKeys.length === 0) {
|
|
2219
|
+
return;
|
|
2220
|
+
}
|
|
2221
|
+
// Confirm against the directory even when the caller supplied a diff:
|
|
2222
|
+
// validatePublicSigningKey is the same trust source the materialization
|
|
2223
|
+
// paths use, so both stay consistent.
|
|
2224
|
+
const directory = await this.tenant.openDirectory();
|
|
2225
|
+
const trustedKeys = [];
|
|
2226
|
+
for (const key of candidateKeys) {
|
|
2227
|
+
try {
|
|
2228
|
+
if (await directory.validatePublicSigningKey(key)) {
|
|
2229
|
+
trustedKeys.push(key);
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
catch (error) {
|
|
2233
|
+
// Transient directory failure: keep the pending record so a later
|
|
2234
|
+
// reconcile retries (fail open towards retry, never towards loss).
|
|
2235
|
+
this.logger.warn(`reconcileAuthorTrust: directory validation failed for a pending key: ${error}`);
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
if (trustedKeys.length === 0) {
|
|
2239
|
+
return;
|
|
2240
|
+
}
|
|
2241
|
+
const affectedDocIds = new Set();
|
|
2242
|
+
for (const key of trustedKeys) {
|
|
2243
|
+
for (const docId of this.pendingUntrustedAuthors.get(key) ?? []) {
|
|
2244
|
+
affectedDocIds.add(docId);
|
|
2245
|
+
}
|
|
2246
|
+
this.pendingUntrustedAuthors.delete(key);
|
|
2247
|
+
}
|
|
2248
|
+
this.logger.info(`reconcileAuthorTrust: ${trustedKeys.length} author key(s) became trusted, re-materializing ${affectedDocIds.size} document(s)`);
|
|
2249
|
+
// Deterministic ordering keeps the resulting changefeed stable between
|
|
2250
|
+
// runs (mirrors reconcileKeyVisibility).
|
|
2251
|
+
const docIds = Array.from(affectedDocIds).sort((left, right) => left.localeCompare(right));
|
|
2252
|
+
for (const docId of docIds) {
|
|
2253
|
+
// Drop every cached form of the stale materialization first so the
|
|
2254
|
+
// next load rebuilds from the store with the now-trusted entries.
|
|
2255
|
+
await this.purgeMaterializedDocument(docId);
|
|
2256
|
+
const metadata = await this.scanAllMetadata(this.store, { docId });
|
|
2257
|
+
const lifecycleEntries = metadata.filter((entry) => entry.entryType === "doc_create"
|
|
2258
|
+
|| entry.entryType === "doc_change"
|
|
2259
|
+
|| entry.entryType === "doc_delete"
|
|
2260
|
+
|| entry.entryType === "doc_undelete"
|
|
2261
|
+
|| entry.entryType === "doc_snapshot");
|
|
2262
|
+
const visibility = this.deriveDocumentVisibilityMetadata(metadata);
|
|
2263
|
+
if (!visibility) {
|
|
2264
|
+
continue;
|
|
2265
|
+
}
|
|
2266
|
+
// Author trust and key visibility are independent layers: without the
|
|
2267
|
+
// decryption key the doc stays governed by reconcileKeyVisibility.
|
|
2268
|
+
if (!(await this.tenant.hasDecryptionKey(visibility.decryptionKeyId))) {
|
|
2269
|
+
continue;
|
|
2270
|
+
}
|
|
2271
|
+
const { awaitingWitness, witnessed } = (0, storeEntryTime_1.metadataWitnessState)(lifecycleEntries);
|
|
2272
|
+
// Force the changeSeq bump: the tracked index fields may be identical to
|
|
2273
|
+
// the stale entry, but the materialized content WILL differ once the
|
|
2274
|
+
// previously skipped entries are applied - consumers must re-read.
|
|
2275
|
+
this.updateIndex(docId, visibility.lastModified, visibility.isDeleted, visibility.decryptionKeyId, "visible", awaitingWitness, witnessed, true);
|
|
2276
|
+
}
|
|
2277
|
+
this.cacheMetaDirty = true;
|
|
2278
|
+
this.cacheManager?.markDirty();
|
|
2279
|
+
}
|
|
2280
|
+
/**
|
|
2281
|
+
* Open-time author-trust healing, called from {@link initialize} after a
|
|
2282
|
+
* cache restore. Two jobs:
|
|
2283
|
+
*
|
|
2284
|
+
* 1. Legacy checkpoints (written before {@link pendingUntrustedAuthors}
|
|
2285
|
+
* existed) may hide entries that were silently skipped - detect them via
|
|
2286
|
+
* the persisted automerge-hash lookup and seed the pending map.
|
|
2287
|
+
* 2. Re-check all pending authors against the directory, healing documents
|
|
2288
|
+
* whose authors became trusted while this database was closed (e.g. the
|
|
2289
|
+
* directory was synced in a session that never opened this DB).
|
|
2290
|
+
*/
|
|
2291
|
+
async reconcileAuthorTrustAtOpen() {
|
|
2292
|
+
if (this.isTimeTravelMode() || this._isAdminOnlyDb || this.store.getId() === "directory") {
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
if (this.legacyCheckpointNeedsAuthorTrustScan) {
|
|
2296
|
+
this.legacyCheckpointNeedsAuthorTrustScan = false;
|
|
2297
|
+
try {
|
|
2298
|
+
await this.seedPendingAuthorsFromLegacyCheckpoint();
|
|
2299
|
+
}
|
|
2300
|
+
catch (error) {
|
|
2301
|
+
this.logger.warn(`Legacy author-trust heal scan failed: ${error}`);
|
|
2302
|
+
}
|
|
2303
|
+
// Whether or not entries were found, the checkpoint is upgraded on the
|
|
2304
|
+
// next flush (the pendingUntrustedAuthors field is always written), so
|
|
2305
|
+
// this scan runs at most once per database.
|
|
2306
|
+
this.cacheMetaDirty = true;
|
|
2307
|
+
this.cacheManager?.markDirty();
|
|
2308
|
+
}
|
|
2309
|
+
if (this.pendingUntrustedAuthors.size === 0) {
|
|
2310
|
+
return;
|
|
2311
|
+
}
|
|
2312
|
+
await this.reconcileAuthorTrust();
|
|
2313
|
+
}
|
|
2314
|
+
/**
|
|
2315
|
+
* One-time heal scan for checkpoints written before the pending
|
|
2316
|
+
* untrusted-author tracking existed: entries skipped back then left no
|
|
2317
|
+
* record, but the persisted automerge-hash lookup
|
|
2318
|
+
* ({@link automergeHashToEntryId}) only contains changes that were actually
|
|
2319
|
+
* APPLIED to a materialization. A store head entry whose hash is missing
|
|
2320
|
+
* from that lookup therefore marks a document whose cached state is missing
|
|
2321
|
+
* entries - seed the pending map with the authors of all unapplied entries
|
|
2322
|
+
* so the regular reconcile can heal (or keep waiting for) them.
|
|
2323
|
+
*/
|
|
2324
|
+
async seedPendingAuthorsFromLegacyCheckpoint() {
|
|
2325
|
+
const allMetadata = await this.scanAllMetadata(this.store);
|
|
2326
|
+
const replayByDoc = new Map();
|
|
2327
|
+
for (const entry of allMetadata) {
|
|
2328
|
+
if (!this.isDocumentReplayEntry(entry)) {
|
|
2329
|
+
continue;
|
|
2330
|
+
}
|
|
2331
|
+
const entries = replayByDoc.get(entry.docId) ?? [];
|
|
2332
|
+
entries.push(entry);
|
|
2333
|
+
replayByDoc.set(entry.docId, entries);
|
|
2334
|
+
}
|
|
2335
|
+
let seededDocs = 0;
|
|
2336
|
+
for (const [docId, replayEntries] of replayByDoc) {
|
|
2337
|
+
const applied = this.automergeHashToEntryId.get(docId);
|
|
2338
|
+
// Heads applied => every entry reachable from them was applied too
|
|
2339
|
+
// (Automerge buffers dependents of missing changes), so checking the
|
|
2340
|
+
// active DAG heads is sufficient and avoids false positives for entries
|
|
2341
|
+
// that are only covered by a snapshot.
|
|
2342
|
+
const headEntryIds = this.findActiveReplayHeadEntryIds(replayEntries);
|
|
2343
|
+
const hasUnappliedHead = headEntryIds.some((entryId) => {
|
|
2344
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(entryId);
|
|
2345
|
+
return parsed !== null && !(applied?.has(parsed.automergeHash));
|
|
2346
|
+
});
|
|
2347
|
+
if (!hasUnappliedHead) {
|
|
2348
|
+
continue;
|
|
2349
|
+
}
|
|
2350
|
+
seededDocs++;
|
|
2351
|
+
for (const entry of replayEntries) {
|
|
2352
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(entry.id);
|
|
2353
|
+
if (!parsed || applied?.has(parsed.automergeHash)) {
|
|
2354
|
+
continue;
|
|
2355
|
+
}
|
|
2356
|
+
this.recordPendingUntrustedAuthor(entry.createdByPublicKey, docId);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
if (seededDocs > 0) {
|
|
2360
|
+
this.logger.info(`Legacy author-trust heal scan found ${seededDocs} document(s) with unapplied store entries`);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2094
2363
|
async reconcileRestoredIndexWithStore() {
|
|
2095
2364
|
const lifecycleMetadata = await this.scanAllMetadata(this.store);
|
|
2096
2365
|
const lifecycleDocIds = Array.from(new Set(lifecycleMetadata
|
|
@@ -2780,6 +3049,20 @@ class BaseMindooDB {
|
|
|
2780
3049
|
scannedEntries: 0,
|
|
2781
3050
|
totalSourceEntries: totalSourceEstimate,
|
|
2782
3051
|
});
|
|
3052
|
+
// Upper scan bound: the source head captured BEFORE the scan started.
|
|
3053
|
+
// Pushing to a witnessing target re-anchors the just-transferred
|
|
3054
|
+
// entries to a fresh receiptOrder on the source (applyWitnessReceipts),
|
|
3055
|
+
// i.e. past the running cursor — without this bound the scan would
|
|
3056
|
+
// re-discover its own transfers and never terminate (first-sync loop).
|
|
3057
|
+
// Entries beyond the bound (re-anchored or written concurrently) are
|
|
3058
|
+
// picked up by the next sync.
|
|
3059
|
+
const scanUpperBound = sourceHead?.maxReceiptOrder;
|
|
3060
|
+
// IDs already transferred in this sync session. Guards against
|
|
3061
|
+
// re-pushing entries the scan re-discovers when the target bloom
|
|
3062
|
+
// snapshot is stale (fetched once up front; empty on a first sync, so
|
|
3063
|
+
// it would classify every re-discovered id as "definitely missing"
|
|
3064
|
+
// and skip the exact hasEntries check).
|
|
3065
|
+
const transferredThisSync = new Set();
|
|
2783
3066
|
let currentPage = 0;
|
|
2784
3067
|
// Scan-page pipelining (sync-v5, phase 4): while page N is being
|
|
2785
3068
|
// filtered and transferred, page N+1 is already being fetched.
|
|
@@ -2803,13 +3086,25 @@ class BaseMindooDB {
|
|
|
2803
3086
|
? sourceStore.scanEntriesSince(page.nextCursor, pageSize)
|
|
2804
3087
|
: null;
|
|
2805
3088
|
currentPage++;
|
|
2806
|
-
|
|
3089
|
+
// Clamp the page to the scan bound. Pages are ordered by
|
|
3090
|
+
// (receiptOrder, id), so everything past the first out-of-bound
|
|
3091
|
+
// entry is out of bound as well.
|
|
3092
|
+
let pageEntries = page.entries;
|
|
3093
|
+
let reachedScanBound = false;
|
|
3094
|
+
if (scanUpperBound !== undefined && pageEntries.length > 0) {
|
|
3095
|
+
const inBound = pageEntries.filter((m) => (m.receiptOrder ?? 0) <= scanUpperBound);
|
|
3096
|
+
if (inBound.length < pageEntries.length) {
|
|
3097
|
+
reachedScanBound = true;
|
|
3098
|
+
pageEntries = inBound;
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
scanned += pageEntries.length;
|
|
2807
3102
|
if (signal?.aborted) {
|
|
2808
3103
|
discardPrefetch();
|
|
2809
3104
|
persistScanCursor(cursor);
|
|
2810
3105
|
return { transferred, transferredBytes, scanned, cancelled: true, rejected };
|
|
2811
3106
|
}
|
|
2812
|
-
if (
|
|
3107
|
+
if (pageEntries.length > 0) {
|
|
2813
3108
|
onProgress?.({
|
|
2814
3109
|
phase: 'transferring',
|
|
2815
3110
|
message: `Scanned ${scanned} entries, checking for changes (page ${currentPage})...`,
|
|
@@ -2818,8 +3113,12 @@ class BaseMindooDB {
|
|
|
2818
3113
|
totalSourceEntries: totalSourceEstimate,
|
|
2819
3114
|
currentPage,
|
|
2820
3115
|
});
|
|
2821
|
-
const ids =
|
|
2822
|
-
|
|
3116
|
+
const ids = pageEntries
|
|
3117
|
+
.map((m) => m.id)
|
|
3118
|
+
.filter((id) => !transferredThisSync.has(id));
|
|
3119
|
+
const missingIds = ids.length > 0
|
|
3120
|
+
? await this.filterMissingIds(targetStore, ids, targetBloom)
|
|
3121
|
+
: [];
|
|
2823
3122
|
if (signal?.aborted) {
|
|
2824
3123
|
discardPrefetch();
|
|
2825
3124
|
persistScanCursor(cursor);
|
|
@@ -2843,6 +3142,9 @@ class BaseMindooDB {
|
|
|
2843
3142
|
persistScanCursor(cursor);
|
|
2844
3143
|
return { transferred, transferredBytes, scanned, cancelled: true, rejected };
|
|
2845
3144
|
}
|
|
3145
|
+
for (const id of missingIds) {
|
|
3146
|
+
transferredThisSync.add(id);
|
|
3147
|
+
}
|
|
2846
3148
|
}
|
|
2847
3149
|
}
|
|
2848
3150
|
onProgress?.({
|
|
@@ -2854,6 +3156,18 @@ class BaseMindooDB {
|
|
|
2854
3156
|
totalSourceEntries: totalSourceEstimate,
|
|
2855
3157
|
currentPage,
|
|
2856
3158
|
});
|
|
3159
|
+
if (reachedScanBound) {
|
|
3160
|
+
// Persist the last in-bound position (NOT page.nextCursor, which
|
|
3161
|
+
// already points into the out-of-bound tail): a concurrent write
|
|
3162
|
+
// that landed between the bound and the re-anchored tail must be
|
|
3163
|
+
// scanned by the next sync.
|
|
3164
|
+
const lastInBound = pageEntries.length > 0 ? pageEntries[pageEntries.length - 1] : null;
|
|
3165
|
+
if (lastInBound) {
|
|
3166
|
+
cursor = { receiptOrder: lastInBound.receiptOrder ?? 0, id: lastInBound.id };
|
|
3167
|
+
}
|
|
3168
|
+
discardPrefetch();
|
|
3169
|
+
break;
|
|
3170
|
+
}
|
|
2857
3171
|
cursor = page.nextCursor;
|
|
2858
3172
|
if (!page.hasMore) {
|
|
2859
3173
|
break;
|
|
@@ -3154,6 +3468,7 @@ class BaseMindooDB {
|
|
|
3154
3468
|
`Custom document IDs must match ${types_1.CUSTOM_DOC_ID_REGEX.source}.`);
|
|
3155
3469
|
}
|
|
3156
3470
|
this.validateIdPrefixOption("createDocuments", options);
|
|
3471
|
+
this.validateAssumeUniqueIdOption("createDocuments", options);
|
|
3157
3472
|
}
|
|
3158
3473
|
const results = new Array(inputs.length).fill(null);
|
|
3159
3474
|
const prepared = [];
|
|
@@ -3210,6 +3525,15 @@ class BaseMindooDB {
|
|
|
3210
3525
|
this.logger.info(`Bulk-created ${prepared.length} documents (${inputs.length - prepared.length} already existed)`);
|
|
3211
3526
|
return results;
|
|
3212
3527
|
}
|
|
3528
|
+
/**
|
|
3529
|
+
* Validate `CreateOptions.assumeUniqueId`: only meaningful for a
|
|
3530
|
+
* caller-provided `id` (generated ids are unique by construction).
|
|
3531
|
+
*/
|
|
3532
|
+
validateAssumeUniqueIdOption(methodName, options) {
|
|
3533
|
+
if (options.assumeUniqueId && options.id === undefined) {
|
|
3534
|
+
throw new Error(`${methodName}: assumeUniqueId requires a caller-provided id`);
|
|
3535
|
+
}
|
|
3536
|
+
}
|
|
3213
3537
|
/**
|
|
3214
3538
|
* Validate `CreateOptions.idPrefix`: mutually exclusive with `id` and must
|
|
3215
3539
|
* match {@link DOC_ID_PREFIX_REGEX}.
|
|
@@ -3283,6 +3607,12 @@ class BaseMindooDB {
|
|
|
3283
3607
|
const keyId = await this.resolveCreateKeyId(options);
|
|
3284
3608
|
const useCustomDocId = options.id !== undefined;
|
|
3285
3609
|
this.validateIdPrefixOption("createDocument", options);
|
|
3610
|
+
this.validateAssumeUniqueIdOption("createDocument", options);
|
|
3611
|
+
// A caller-provided id needs the deterministic seed change UNLESS the
|
|
3612
|
+
// caller asserts the id is random/unique (`assumeUniqueId`): convergence
|
|
3613
|
+
// of concurrent same-id creates is then not a concern and the create can
|
|
3614
|
+
// follow the generated-ID path (initialValues baked into doc_create).
|
|
3615
|
+
const useDeterministicSeed = useCustomDocId && !options.assumeUniqueId;
|
|
3286
3616
|
// Sanitize caller-provided initial values: internal/reserved fields (those
|
|
3287
3617
|
// starting with "_", e.g. "_attachments") are managed by MindooDB and must
|
|
3288
3618
|
// not be seeded by callers. `undefined` values are treated as "not set"
|
|
@@ -3296,12 +3626,13 @@ class BaseMindooDB {
|
|
|
3296
3626
|
throw new Error(`createDocument: invalid document id "${options.id}". ` +
|
|
3297
3627
|
`Custom document IDs must match ${types_1.CUSTOM_DOC_ID_REGEX.source}.`);
|
|
3298
3628
|
}
|
|
3299
|
-
//
|
|
3300
|
-
// independent replicas converge on the same Automerge
|
|
3301
|
-
// caller values into that change would diverge the hash per
|
|
3302
|
-
// initialValues is unsupported on the
|
|
3303
|
-
// document first, then apply values via
|
|
3304
|
-
|
|
3629
|
+
// Convergent custom-ID documents are seeded with a hard-coded initial
|
|
3630
|
+
// change so that independent replicas converge on the same Automerge
|
|
3631
|
+
// hash. Baking caller values into that change would diverge the hash per
|
|
3632
|
+
// content, so initialValues is unsupported on the deterministic-seed
|
|
3633
|
+
// path in v1. Create the document first, then apply values via
|
|
3634
|
+
// changeDoc() — or assert `assumeUniqueId` for random ids.
|
|
3635
|
+
if (useDeterministicSeed && hasInitialValues) {
|
|
3305
3636
|
throw new Error("createDocument: initialValues is not supported together with a custom id; " +
|
|
3306
3637
|
"create the document, then apply values with changeDoc().");
|
|
3307
3638
|
}
|
|
@@ -3338,14 +3669,15 @@ class BaseMindooDB {
|
|
|
3338
3669
|
this.logger.debug(`Creating document ${docId} with key ${keyId}${useCustomSigningKey ? ' using custom signing key' : ''}${useCustomDocId ? ' with caller-provided id' : ''}`);
|
|
3339
3670
|
// Build the initial Automerge document and its first change bytes.
|
|
3340
3671
|
//
|
|
3341
|
-
// For UUID7 documents
|
|
3342
|
-
//
|
|
3343
|
-
//
|
|
3344
|
-
//
|
|
3672
|
+
// For UUID7 documents (and unique-by-assertion custom ids) we use the
|
|
3673
|
+
// historical path (Automerge.init + change). For convergent custom-ID
|
|
3674
|
+
// documents we apply a hard-coded initial change so that independent
|
|
3675
|
+
// replicas using the same id produce the same Automerge hash and
|
|
3676
|
+
// `doc_create` entry id, allowing later changes to merge.
|
|
3345
3677
|
const now = (0, timeSource_1.semanticNow)();
|
|
3346
3678
|
let newDoc;
|
|
3347
3679
|
let changeBytes;
|
|
3348
|
-
if (
|
|
3680
|
+
if (useDeterministicSeed) {
|
|
3349
3681
|
this.logger.debug(`Seeding custom-id document ${docId} with hard-coded initial Automerge change`);
|
|
3350
3682
|
changeBytes = getCustomIdInitialChangeBytes();
|
|
3351
3683
|
try {
|
|
@@ -3525,6 +3857,11 @@ class BaseMindooDB {
|
|
|
3525
3857
|
const keyId = await this.resolveCreateKeyId(options);
|
|
3526
3858
|
const useCustomDocId = options.id !== undefined;
|
|
3527
3859
|
this.validateIdPrefixOption("createDocuments", options);
|
|
3860
|
+
this.validateAssumeUniqueIdOption("createDocuments", options);
|
|
3861
|
+
// See createDocumentInternal: `assumeUniqueId` opts a caller-provided id
|
|
3862
|
+
// out of the deterministic seed, so initialValues fold into doc_create
|
|
3863
|
+
// and the document costs a single store entry.
|
|
3864
|
+
const useDeterministicSeed = useCustomDocId && !options.assumeUniqueId;
|
|
3528
3865
|
// Sanitize caller-provided initial values: internal/reserved fields are
|
|
3529
3866
|
// managed by MindooDB and must not be seeded by callers. `undefined`
|
|
3530
3867
|
// values are treated as "not set" (Automerge cannot represent undefined).
|
|
@@ -3547,7 +3884,7 @@ class BaseMindooDB {
|
|
|
3547
3884
|
// following the same seeding rules as the single-create path.
|
|
3548
3885
|
let seedDoc;
|
|
3549
3886
|
let createChangeBytes;
|
|
3550
|
-
if (
|
|
3887
|
+
if (useDeterministicSeed) {
|
|
3551
3888
|
createChangeBytes = getCustomIdInitialChangeBytes();
|
|
3552
3889
|
const fresh = automerge_adapter_1.Automerge.init();
|
|
3553
3890
|
const [appliedDoc] = automerge_adapter_1.Automerge.applyChanges(fresh, [createChangeBytes]);
|
|
@@ -3569,12 +3906,13 @@ class BaseMindooDB {
|
|
|
3569
3906
|
}
|
|
3570
3907
|
createChangeBytes = localChange;
|
|
3571
3908
|
}
|
|
3572
|
-
//
|
|
3573
|
-
// stay byte-identical across replicas, so the values go into a
|
|
3574
|
-
// follow-up change that ships in the same batch.
|
|
3909
|
+
// Convergent custom-id + initialValues (bulk-only): the deterministic seed
|
|
3910
|
+
// change must stay byte-identical across replicas, so the values go into a
|
|
3911
|
+
// separate follow-up change that ships in the same batch. Unique-by-
|
|
3912
|
+
// assertion ids skip this — their values are already in doc_create.
|
|
3575
3913
|
let finalDoc = seedDoc;
|
|
3576
3914
|
let valueChangeBytes = null;
|
|
3577
|
-
if (
|
|
3915
|
+
if (useDeterministicSeed && initialValueEntries.length > 0) {
|
|
3578
3916
|
finalDoc = automerge_adapter_1.Automerge.change(seedDoc, { time: now }, (doc) => {
|
|
3579
3917
|
for (const [key, value] of initialValueEntries) {
|
|
3580
3918
|
doc[key] = value;
|
|
@@ -8160,6 +8498,12 @@ class BaseMindooDB {
|
|
|
8160
8498
|
if (cryptoKey) {
|
|
8161
8499
|
keyMap.set(publicKey, cryptoKey);
|
|
8162
8500
|
}
|
|
8501
|
+
else {
|
|
8502
|
+
// Author key not (yet) trusted by the directory: the entries below
|
|
8503
|
+
// will be skipped. Record the author so a later grantaccess sync
|
|
8504
|
+
// can re-materialize this document (reconcileAuthorTrust).
|
|
8505
|
+
this.recordPendingUntrustedAuthor(publicKey, docId);
|
|
8506
|
+
}
|
|
8163
8507
|
}
|
|
8164
8508
|
// Verify all signatures in parallel using cached keys
|
|
8165
8509
|
const signatureVerificationResults = await Promise.all(validEntries.map(async (entryData) => {
|
|
@@ -8959,7 +9303,14 @@ class BaseMindooDB {
|
|
|
8959
9303
|
}
|
|
8960
9304
|
// Verify the author signature over the encrypted payload.
|
|
8961
9305
|
const cryptoKey = await this.getOrImportPublicKey(entry.createdByPublicKey);
|
|
8962
|
-
if (!cryptoKey
|
|
9306
|
+
if (!cryptoKey) {
|
|
9307
|
+
// Author key not (yet) trusted by the directory. Remember it so a
|
|
9308
|
+
// later grantaccess sync can re-materialize the doc (reconcileAuthorTrust).
|
|
9309
|
+
this.recordPendingUntrustedAuthor(entry.createdByPublicKey, docId);
|
|
9310
|
+
quarantine(meta, "invalid_signature", "author key not trusted by directory");
|
|
9311
|
+
continue;
|
|
9312
|
+
}
|
|
9313
|
+
if (!(await this.verifyEntrySignatureWithKey(cryptoKey, entry))) {
|
|
8963
9314
|
quarantine(meta, "invalid_signature", "signature verification failed");
|
|
8964
9315
|
continue;
|
|
8965
9316
|
}
|
|
@@ -9363,6 +9714,12 @@ class BaseMindooDB {
|
|
|
9363
9714
|
if (cryptoKey) {
|
|
9364
9715
|
keyMap.set(publicKey, cryptoKey);
|
|
9365
9716
|
}
|
|
9717
|
+
else {
|
|
9718
|
+
// Author key not (yet) trusted by the directory: the entries below
|
|
9719
|
+
// will be skipped. Record the author so a later grantaccess sync
|
|
9720
|
+
// can re-materialize this document (reconcileAuthorTrust).
|
|
9721
|
+
this.recordPendingUntrustedAuthor(publicKey, docId);
|
|
9722
|
+
}
|
|
9366
9723
|
}
|
|
9367
9724
|
// Verify all signatures in parallel using cached keys
|
|
9368
9725
|
const signatureStartedAt = Date.now();
|
|
@@ -10251,6 +10608,13 @@ class BaseMindooDB {
|
|
|
10251
10608
|
// single-flight; never blocks or fails the sync.
|
|
10252
10609
|
if (this.store.getId() === "directory") {
|
|
10253
10610
|
await this.tenant.reconcileKeyDistributionsForCurrentUserSafe();
|
|
10611
|
+
// Author-trust reconcile: refresh the directory trust caches NOW
|
|
10612
|
+
// (instead of waiting up to DIRECTORY_SYNC_INTERVAL_MS for the next
|
|
10613
|
+
// cache-miss validation) so grantaccess documents that just arrived
|
|
10614
|
+
// immediately unblock entries other open databases skipped while
|
|
10615
|
+
// the author was still unknown. Awaited so a subsequent content-DB
|
|
10616
|
+
// sync in the same flow observes the healed state.
|
|
10617
|
+
await this.tenant.reconcileAuthorTrustAfterDirectorySyncSafe();
|
|
10254
10618
|
}
|
|
10255
10619
|
}
|
|
10256
10620
|
if (options?.signal?.aborted) {
|