mindoodb 0.0.53 → 0.0.55

package/dist/core/BaseMindooDB.js

@@ -9,7 +9,7 @@ const storeEntryTime_1 = require("./storeEntryTime");
 const uuid_1 = require("uuid");
 const types_1 = require("./types");
 const types_2 = require("./appendonlystores/types");
-const bloom_1 = require("./appendonlystores/bloom");
+const syncStores_1 = require("./appendonlystores/syncStores");
 const types_3 = require("./appendonlystores/network/types");
 const MaterializationPlanner_1 = require("./appendonlystores/MaterializationPlanner");
 const DocumentDagAnalysis_1 = require("./DocumentDagAnalysis");
@@ -352,6 +352,35 @@ class BaseMindooDB {
          * from the cached materialization and seeds the pending map from them.
          */
         this.legacyCheckpointNeedsAuthorTrustScan = false;
+        /**
+         * Ids of documents that exist in the local store but cannot be read with
+         * the current KeyBag, so they are absent from the index and from every
+         * document listing.
+         *
+         * The index alone cannot answer "how much is hidden from me": a document
+         * that was never readable on this device gets no index entry at all (see
+         * {@link reconcileKeyVisibility}), and one that lost its key becomes an
+         * `"inaccessible"` tombstone that listings filter out. This set is the
+         * only place that remembers both cases, which is what
+         * {@link getInaccessibleDocumentCount} reports so a UI can tell an empty
+         * database apart from one whose keys have not arrived yet.
+         *
+         * Counts documents regardless of their deletion state - the question it
+         * answers is "is it here and unreadable", which is knowable without the
+         * key, unlike a reliable lifecycle state for a doc we cannot decrypt.
+         *
+         * Persisted in the metadata checkpoint: a warm start with an unchanged
+         * KeyBag skips the visibility scan, so an in-memory-only tally would
+         * silently reset to zero on every restart.
+         */
+        this.inaccessibleDocIds = new Set();
+        /**
+         * Set while restoring a metadata checkpoint that predates
+         * {@link inaccessibleDocIds}. Forces one visibility scan at open so the
+         * tally starts out complete instead of counting only what happens to
+         * arrive in this session.
+         */
+        this.legacyCheckpointNeedsVisibilityTally = false;
         // Local cache support
         this.cacheManager = null;
         this.cachePrefix = null;
@@ -462,6 +491,8 @@ class BaseMindooDB {
         this.userdirectoryHashByDocId = new Map();
         /** Grant `username_hash` for a signing public key at "now". */
         this.signerUsernameHashByKey = new Map();
+        /** Owning person per personal userdirectory document (its create signer). */
+        this.personalDocCreatorHashByDocId = new Map();
         // ---------------------------------------------------------------------------
         // Author-trust reconciliation: heal documents whose entries were skipped
         // because their author was not yet known to the tenant directory.
@@ -578,15 +609,22 @@ class BaseMindooDB {
      * Hard-wired write gate for `directory` / `userdirectory` (and other
      * admin-only databases). Independent of the ACL master switch.
      */
-    async assertBuiltinWriteAllowed(op, signerPublicKey, documentUsernameHash) {
+    async assertBuiltinWriteAllowed(op, signerPublicKey, documentUsernameHash, docId) {
         const dbId = this.effectiveBuiltinDbId();
         if (!(0, builtinDbInvariants_1.hasBuiltinWriteInvariant)(dbId) && !this._isAdminOnlyDb) {
             return;
         }
         let signerUsernameHash = null;
+        let creatorUsernameHash = null;
         if (dbId === types_1.USER_DIRECTORY_DB_ID) {
             signerUsernameHash = await this.resolveSignerUsernameHash(signerPublicKey);
             this.signerUsernameHashByKey.set(signerPublicKey, signerUsernameHash);
+            if ((0, builtinDbInvariants_1.isPersonalUserdirectoryDocId)(docId)) {
+                creatorUsernameHash =
+                    op === "doc_create"
+                        ? signerUsernameHash
+                        : await this.ensurePersonalDocCreatorHash(docId);
+            }
         }
         const decision = (0, builtinDbInvariants_1.evaluateBuiltinWrite)({
             dbId,
@@ -595,6 +633,8 @@ class BaseMindooDB {
             adminPublicKey: this.getAdminPublicKey(),
             documentUsernameHash,
             signerUsernameHash,
+            docId,
+            creatorUsernameHash,
         });
         if (!decision.allowed) {
             throw new Error(decision.reason);
@@ -611,7 +651,14 @@ class BaseMindooDB {
         const entryType = entry.entryType ?? "doc_change";
         let documentUsernameHash = null;
         let signerUsernameHash = null;
-        if (dbId === types_1.USER_DIRECTORY_DB_ID &&
+        let creatorUsernameHash = null;
+        if (dbId === types_1.USER_DIRECTORY_DB_ID && (0, builtinDbInvariants_1.isPersonalUserdirectoryDocId)(entry.docId)) {
+            if (entryType !== "doc_create") {
+                creatorUsernameHash = await this.ensurePersonalDocCreatorHash(entry.docId);
+                signerUsernameHash = await this.ensureSignerUsernameHash(entry.createdByPublicKey);
+            }
+        }
+        else if (dbId === types_1.USER_DIRECTORY_DB_ID &&
             (entryType === "doc_change" || entryType === "doc_snapshot") &&
             entry.docId) {
             documentUsernameHash = await this.ensureUserdirectoryDocHash(entry.docId);
@@ -624,6 +671,8 @@ class BaseMindooDB {
             adminPublicKey: this.getAdminPublicKey(),
             documentUsernameHash,
             signerUsernameHash,
+            docId: entry.docId,
+            creatorUsernameHash,
         });
     }
     async ensureSignerUsernameHash(signerKey) {
@@ -634,6 +683,34 @@ class BaseMindooDB {
         this.signerUsernameHashByKey.set(signerKey, hash);
         return hash;
     }
+    /**
+     * Owner of a personal `userdirectory` document: the person who signed its
+     * `doc_create`, resolved through grants rather than the payload. A sealed
+     * document is opaque to everyone but its owner, so the payload is not an
+     * option — and grants map every device of a person to one hash, which is
+     * exactly the granularity ownership needs.
+     */
+    async ensurePersonalDocCreatorHash(docId) {
+        const cached = this.personalDocCreatorHashByDocId.get(docId);
+        if (cached !== undefined)
+            return cached;
+        let hash = null;
+        try {
+            const metas = await this.scanAllMetadata(this.store, { docId });
+            const create = metas.find((meta) => meta.entryType === "doc_create");
+            if (create) {
+                hash = await this.ensureSignerUsernameHash(create.createdByPublicKey);
+            }
+        }
+        catch {
+            hash = null;
+        }
+        // Only a resolved owner is worth caching: a miss usually means the create
+        // entry has not arrived yet, and the next write must ask again.
+        if (hash)
+            this.personalDocCreatorHashByDocId.set(docId, hash);
+        return hash;
+    }
     async ensureUserdirectoryDocHash(docId) {
         const cached = this.userdirectoryHashByDocId.get(docId);
         if (cached)
@@ -1472,6 +1549,11 @@ class BaseMindooDB {
             }
         }
         checkpoint.pendingUntrustedAuthors = pendingUntrusted;
+        // Persist the unreadable-document tally (additive field). Written even
+        // when empty, for the same reason as above: the field's PRESENCE marks
+        // the checkpoint as tally-aware, so restore can tell "nothing hidden"
+        // apart from "written before this existed, count unknown".
+        checkpoint.inaccessibleDocIds = Array.from(this.inaccessibleDocIds);
         return new TextEncoder().encode(JSON.stringify(checkpoint));
     }
     /**
@@ -1518,6 +1600,22 @@ class BaseMindooDB {
             else {
                 this.legacyCheckpointNeedsAuthorTrustScan = true;
             }
+            // Unreadable-document tally. Missing field means the checkpoint predates
+            // it: the count would otherwise stay at zero for as long as the KeyBag
+            // is unchanged, because that is exactly when the visibility scan is
+            // skipped. Force one scan instead.
+            this.inaccessibleDocIds.clear();
+            if (Array.isArray(checkpoint.inaccessibleDocIds)) {
+                this.legacyCheckpointNeedsVisibilityTally = false;
+                for (const docId of checkpoint.inaccessibleDocIds) {
+                    if (typeof docId === "string") {
+                        this.inaccessibleDocIds.add(docId);
+                    }
+                }
+            }
+            else {
+                this.legacyCheckpointNeedsVisibilityTally = true;
+            }
             this.syncScanCursors.clear();
             if (checkpoint.syncScanCursors && typeof checkpoint.syncScanCursors === "object") {
                 for (const [key, value] of Object.entries(checkpoint.syncScanCursors)) {
@@ -1871,6 +1969,17 @@ class BaseMindooDB {
         // Every path that writes entries for a document lands here, so this is
         // where retained metadata for it goes stale (see scannedEntryMetadata).
         this.forgetScannedEntryMetadata(docId);
+        // Same reasoning for the unreadable-document tally: every reveal and every
+        // inaccessibility tombstone passes through here, so keeping it in step
+        // needs no bookkeeping at the individual call sites. The one case this
+        // misses is a document that was never readable, which produces no index
+        // entry at all - those call `noteInaccessibleDocument` directly.
+        if (accessState === "visible") {
+            this.forgetInaccessibleDocument(docId);
+        }
+        else {
+            this.noteInaccessibleDocument(docId);
+        }
         const assignedSeq = this.nextChangeSeq;
         const newEntry = {
             docId,
@@ -2138,7 +2247,8 @@ class BaseMindooDB {
                 // transition.
                 const currentFingerprint = await this.computeCurrentKeyBagFingerprint();
                 const fingerprintMatches = currentFingerprint !== null && currentFingerprint === this.lastReconciledKeyBagFingerprint;
-                if (!fingerprintMatches) {
+                if (!fingerprintMatches || this.legacyCheckpointNeedsVisibilityTally) {
+                    this.legacyCheckpointNeedsVisibilityTally = false;
                     await this.reconcileKeyVisibility();
                 }
                 this.markKeyBagFingerprintReconciled(currentFingerprint);
@@ -2257,6 +2367,73 @@ class BaseMindooDB {
         this.cacheMetaDirty = true;
         this.cacheManager?.markDirty();
     }
+    /**
+     * Remember that `docId` is present in the store but unreadable. Idempotent:
+     * the same document is re-observed on every sync batch that carries another
+     * entry for it.
+     */
+    noteInaccessibleDocument(docId) {
+        if (this.inaccessibleDocIds.has(docId)) {
+            return;
+        }
+        this.inaccessibleDocIds.add(docId);
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+    }
+    /** Counterpart of {@link noteInaccessibleDocument} for a doc that became readable. */
+    forgetInaccessibleDocument(docId) {
+        if (!this.inaccessibleDocIds.delete(docId)) {
+            return;
+        }
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+    }
+    /**
+     * How many documents this database holds locally that the current KeyBag
+     * cannot open. They appear in no listing and in no view, so this count is
+     * the only signal that a database looks emptier than it is - typically on a
+     * device whose user key is still waiting for approval.
+     *
+     * Includes deleted documents, and counts what has arrived on this device -
+     * not what the tenant holds.
+     */
+    getInaccessibleDocumentCount() {
+        return this.inaccessibleDocIds.size;
+    }
+    /**
+     * Documents in the local store that this KeyBag cannot open, described
+     * from unsigned origin metadata. See {@link MindooDB.listInaccessibleDocuments}.
+     */
+    async listInaccessibleDocuments(options) {
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
+        const docIds = [...this.inaccessibleDocIds].filter((docId) => !idPrefix || (0, idGeneration_1.matchesDocIdPrefix)(docId, idPrefix));
+        if (docIds.length === 0) {
+            return [];
+        }
+        const results = [];
+        for (const docId of docIds) {
+            const originEntries = await this.scanAllMetadata(this.store, {
+                docId,
+                entryTypes: ["doc_create", "doc_snapshot"],
+            });
+            let origin;
+            for (const entry of originEntries) {
+                if (!origin || entry.createdAt < origin.createdAt) {
+                    origin = entry;
+                }
+            }
+            const indexPos = this.getDocIndexPosition(docId);
+            const indexEntry = indexPos === undefined ? undefined : this.index[indexPos];
+            results.push({
+                docId,
+                createdAt: origin?.createdAt ?? indexEntry?.lastModified ?? 0,
+                decryptionKeyId: origin?.decryptionKeyId ?? indexEntry?.decryptionKeyId ?? "",
+                createdByPublicKey: origin?.createdByPublicKey ?? "",
+            });
+        }
+        results.sort((left, right) => left.docId.localeCompare(right.docId));
+        return results;
+    }
     /**
      * Derive the per-doc visibility summary needed to reconcile the index
      * against the current KeyBag state.
@@ -2392,6 +2569,9 @@ class BaseMindooDB {
             await this.tenant.ingestSealedFromEntries?.(visibility.decryptionKeyId, metadataByDoc.get(docId));
             const canRead = await this.tenant.hasDecryptionKey(visibility.decryptionKeyId);
             if (canRead) {
+                // This scan is the authority on visibility, so it also heals a tally
+                // that a visible->visible short-circuit below would leave untouched.
+                this.forgetInaccessibleDocument(docId);
                 // Reveal-on-add only triggers when the previous state was
                 // missing or inaccessible; visible->visible transitions stay
                 // idempotent via the existing `updateIndex` short-circuit.
@@ -2411,8 +2591,11 @@ class BaseMindooDB {
                 changed = true;
             }
             else {
-                // Already inaccessible (or never seen). Scrub defensively in
-                // case a previous session left an L2 record behind.
+                // Already inaccessible (or never seen, which is the common case on a
+                // device whose keys have not arrived). No index entry is written for
+                // it, so the tally is the only record that it is here at all.
+                this.noteInaccessibleDocument(docId);
+                // Scrub defensively in case a previous session left an L2 record behind.
                 await this.purgeMaterializedDocument(docId);
             }
         }
@@ -2786,6 +2969,10 @@ class BaseMindooDB {
                         const keyAvailable = await this.tenant.hasDecryptionKey(representativeEntry.decryptionKeyId);
                         if (!keyAvailable) {
                             this.logger.debug(`Skipping metadata-first index for doc ${docId} — decryption key "${representativeEntry.decryptionKeyId}" not available`);
+                            // This is where a device that is still waiting for its keys meets
+                            // most of the tenant: the doc is skipped before it ever reaches
+                            // the index, so record it for `getInaccessibleDocumentCount`.
+                            this.noteInaccessibleDocument(docId);
                             const existingIndex = this.getDocIndexPosition(docId);
                             const existing = existingIndex === undefined ? undefined : this.index[existingIndex];
                             if (existing?.accessState === "visible") {
@@ -2840,6 +3027,7 @@ class BaseMindooDB {
                 // doc.
                 if (error instanceof errors_1.SymmetricKeyNotFoundError) {
                     this.logger.debug(`Skipping document ${docId} - missing key: ${error.keyId}`);
+                    this.noteInaccessibleDocument(docId);
                     // Avoid a per-doc scanAllMetadata round-trip here: everything
                     // we need is already in this batch's entryMetadataList plus the
                     // existing index entry (if any). We do NOT need a second pass
@@ -3150,54 +3338,8 @@ class BaseMindooDB {
             nextCursor: cursor,
         };
     }
-    /**
-     * Fetch a store's bloom filter summary, returning null if unsupported or on
-     * error. Used on the target for missing-id pre-screening (callers fall back
-     * to exact checks) and on the source for the total-entry-count progress
-     * denominator.
-     */
-    async getStoreBloomSummary(store) {
-        if (typeof store.getIdBloomSummary !== "function") {
-            return null;
-        }
-        try {
-            return await store.getIdBloomSummary();
-        }
-        catch (error) {
-            this.logger.warn("Failed to get bloom summary from store, falling back to exact checks", error);
-            return null;
-        }
-    }
-    static setSyncAbortSignalOnStore(store, signal) {
-        if ('setSyncAbortSignal' in store && typeof store.setSyncAbortSignal === 'function') {
-            store.setSyncAbortSignal(signal);
-        }
-    }
-    /**
-     * Fetch a store's head descriptor (`{ epoch, maxReceiptOrder }`), returning
-     * null when the store (or the remote server behind it) does not support it
-     * or the request fails. A null head simply disables the persisted-cursor
-     * fast path for this sync.
-     */
-    async getStoreHeadSafe(store) {
-        if (typeof store.getStoreHead !== "function") {
-            return null;
-        }
-        try {
-            return await store.getStoreHead();
-        }
-        catch (error) {
-            this.logger.debug("Store head unavailable, falling back to full scan", error);
-            return null;
-        }
-    }
-    /** Stable identity for a store in the persisted sync-cursor map. */
-    static syncStoreIdentity(store) {
-        return store.getCacheIdentity?.() ?? `${store.getId()}/${store.getStoreKind()}`;
-    }
-    /** Key for {@link syncScanCursors}: one record per (source, target) pair. */
-    syncScanCursorKey(sourceStore, targetStore) {
-        return `${BaseMindooDB.syncStoreIdentity(sourceStore)}->${BaseMindooDB.syncStoreIdentity(targetStore)}`;
+    getStoreBloomSummary(store) {
+        return (0, syncStores_1.getStoreBloomSummary)(store, this.logger);
     }
     /** Persist a sync scan cursor and schedule a checkpoint flush. */
     saveSyncScanCursor(key, record) {
@@ -3213,171 +3355,8 @@ class BaseMindooDB {
         this.cacheMetaDirty = true;
         this.cacheManager?.markDirty();
     }
-    /**
-     * From a list of candidate IDs, return only those the target store is
-     * missing.  Uses bloom-filter pre-screening when available, then falls
-     * back to exact `hasEntries` for the uncertain set.
-     */
-    async filterMissingIds(targetStore, candidateIds, bloom) {
-        let definitelyMissing = [];
-        let maybeExisting = candidateIds;
-        if (bloom) {
-            definitelyMissing = [];
-            maybeExisting = [];
-            for (const id of candidateIds) {
-                if ((0, bloom_1.bloomMightContainId)(bloom, id)) {
-                    maybeExisting.push(id);
-                }
-                else {
-                    definitelyMissing.push(id);
-                }
-            }
-        }
-        let missingIds = definitelyMissing;
-        if (maybeExisting.length > 0) {
-            const existing = await targetStore.hasEntries(maybeExisting);
-            const existingSet = new Set(existing);
-            missingIds = missingIds.concat(maybeExisting.filter((id) => !existingSet.has(id)));
-        }
-        return missingIds;
-    }
-    resolveTransferBatchSize(options) {
-        if (options?.transferBatchSize && options.transferBatchSize > 0) {
-            return options.transferBatchSize;
-        }
-        if (options?.storeKind === types_2.StoreKind.attachments) {
-            return 100;
-        }
-        return Math.min(options?.pageSize ?? 1000, BaseMindooDB.DEFAULT_DOCS_TRANSFER_BATCH_SIZE);
-    }
-    /**
-     * Transfer a set of entry IDs from source to target in fixed-size batches,
-     * emitting progress and checking for cancellation between each batch.
-     *
-     * Callers (cursor-scan path and legacy path) collect the IDs that need
-     * transferring, then delegate to this method instead of issuing one
-     * monolithic `getEntries`.  This gives three benefits:
-     *
-     * 1. The UI receives frequent progress updates with batch metadata so long
-     *    transfers no longer look frozen.
-     * 2. Cancellation can interrupt work between batches rather than waiting
-     *    for one large HTTP response to complete.
-     * 3. Each server-side `getEntries` + RSA encryption unit is smaller,
-     *    reducing the risk of socket timeouts on large payloads.
-     *
-     * Returns partial progress on cancellation so callers can report how much
-     * was actually transferred before the abort.
-     */
-    async transferEntriesInBatches(sourceStore, targetStore, entryIds, options, state) {
-        if (entryIds.length === 0) {
-            return { transferred: state.transferred, transferredBytes: state.transferredBytes, cancelled: false, rejected: [] };
-        }
-        const onProgress = options?.onProgress;
-        const signal = options?.signal;
-        const transferBatchSize = this.resolveTransferBatchSize(options);
-        const totalTransferBatches = Math.max(1, Math.ceil(entryIds.length / transferBatchSize));
-        // Parallel transfer window (sync-v5, phase 4): each batch is one
-        // getEntries + putEntries round trip; a small window of concurrent
-        // batches overlaps network latency and server-side crypto. Witness
-        // receipts are applied per batch and are order-independent.
-        const maxConcurrent = Math.min(Math.max(1, Math.floor(options?.maxConcurrentBatches ?? BaseMindooDB.DEFAULT_MAX_CONCURRENT_TRANSFER_BATCHES)), totalTransferBatches);
-        let transferred = state.transferred;
-        let transferredBytes = state.transferredBytes;
-        let cancelled = false;
-        let firstError = null;
-        let nextBatchIndex = 0;
-        // Per-entry rejections reported by a witnessing target (sync-v5): the
-        // push continues, the rejected entries are surfaced to the caller.
-        const rejected = [];
-        const runWorker = async () => {
-            while (true) {
-                if (firstError || cancelled || signal?.aborted) {
-                    if (signal?.aborted)
-                        cancelled = true;
-                    return;
-                }
-                const batchIndex = nextBatchIndex++;
-                const offset = batchIndex * transferBatchSize;
-                if (offset >= entryIds.length) {
-                    return;
-                }
-                const currentTransferBatch = batchIndex + 1;
-                const batchIds = entryIds.slice(offset, offset + transferBatchSize);
-                const pageSummary = state.currentPage ? `page ${state.currentPage}, ` : "";
-                onProgress?.({
-                    phase: "transferring",
-                    message: `Transferring batch ${currentTransferBatch}/${totalTransferBatches} (${batchIds.length} entries, ${pageSummary}scanned ${state.scanned})...`,
-                    transferredEntries: transferred,
-                    transferredBytes,
-                    scannedEntries: state.scanned,
-                    totalSourceEntries: state.totalSourceEntries,
-                    currentPage: state.currentPage,
-                    currentTransferBatch,
-                    totalTransferBatches,
-                    transferBatchSize,
-                });
-                try {
-                    const batchEntries = await sourceStore.getEntries(batchIds);
-                    if (signal?.aborted) {
-                        cancelled = true;
-                        return;
-                    }
-                    const putResult = await targetStore.putEntries(batchEntries);
-                    // On push, the remote (witnessing) target returns receipts for accepted
-                    // entries; persist them back onto the local source so the revision feed
-                    // re-anchors them from the provisional head to their committed
-                    // `receivedAt` (docs/accesscontrol.md §5.3). Pull targets return void.
-                    // sync-v5 targets return a structured ack that additionally carries
-                    // per-entry rejections (signature-class failures the remote skipped).
-                    const { receipts, batchRejected } = BaseMindooDB.normalizePutResult(putResult);
-                    if (receipts.length > 0 && sourceStore.applyWitnessReceipts) {
-                        await sourceStore.applyWitnessReceipts(receipts);
-                    }
-                    const rejectedIds = batchRejected.length > 0 ? new Set(batchRejected.map((entry) => entry.id)) : null;
-                    if (rejectedIds) {
-                        rejected.push(...batchRejected);
-                        this.logger.warn(`Target rejected ${batchRejected.length} of ${batchEntries.length} entries in transfer batch ${currentTransferBatch}/${totalTransferBatches}`);
-                    }
-                    for (const entry of batchEntries) {
-                        if (rejectedIds?.has(entry.id)) {
-                            continue;
-                        }
-                        transferredBytes += entry.encryptedSize ?? entry.encryptedData.length;
-                    }
-                    transferred += batchEntries.length - batchRejected.length;
-                }
-                catch (error) {
-                    if (signal?.aborted) {
-                        cancelled = true;
-                        return;
-                    }
-                    if (!firstError) {
-                        firstError = error;
-                    }
-                    return;
-                }
-                onProgress?.({
-                    phase: "transferring",
-                    message: `Transferred ${transferred} entries after batch ${currentTransferBatch}/${totalTransferBatches}`,
-                    transferredEntries: transferred,
-                    transferredBytes,
-                    scannedEntries: state.scanned,
-                    totalSourceEntries: state.totalSourceEntries,
-                    currentPage: state.currentPage,
-                    currentTransferBatch,
-                    totalTransferBatches,
-                    transferBatchSize,
-                });
-            }
-        };
-        await Promise.all(Array.from({ length: maxConcurrent }, () => runWorker()));
-        if (signal?.aborted || cancelled) {
-            return { transferred, transferredBytes, cancelled: true, rejected };
-        }
-        if (firstError) {
-            throw firstError;
-        }
-        return { transferred, transferredBytes, cancelled: false, rejected };
+    filterMissingIds(targetStore, candidateIds, bloom) {
+        return (0, syncStores_1.filterMissingIds)(targetStore, candidateIds, bloom);
     }
     /**
      * Normalize the polymorphic putEntries result (`void`, receipts array, or
@@ -3395,15 +3374,39 @@ class BaseMindooDB {
         }
         return { receipts: [], batchRejected: [] };
     }
+    /**
+     * Expose this database's persisted {@link syncScanCursors} map to the shared
+     * store-transfer helper. The map lives in the metadata checkpoint, so saving
+     * a cursor also marks the cache dirty.
+     */
+    syncCursorStore() {
+        return {
+            get: (key) => this.syncScanCursors.get(key) ?? null,
+            save: (key, record) => this.saveSyncScanCursor(key, record),
+            delete: (key) => {
+                this.syncScanCursors.delete(key);
+            },
+        };
+    }
     async syncEntriesFromStore(sourceStore, targetStore, options) {
         const signal = options?.signal;
-        BaseMindooDB.setSyncAbortSignalOnStore(sourceStore, signal);
-        BaseMindooDB.setSyncAbortSignalOnStore(targetStore, signal);
+        (0, syncStores_1.setSyncAbortSignalOnStore)(sourceStore, signal);
+        (0, syncStores_1.setSyncAbortSignalOnStore)(targetStore, signal);
         try {
             if (options?.mode === "dense") {
                 return await this.syncEntriesFromStoreDense(sourceStore, targetStore, options);
             }
-            return await this.syncEntriesFromStoreImpl(sourceStore, targetStore, options);
+            // Client rejection semantics: the scan cursor advances past a rejected
+            // entry so a single corrupt or forged entry cannot block this database's
+            // push sync forever (docs/network-sync-protocol.md §5.7.6). The
+            // server-to-server replicator passes `"hold"` instead, because there a
+            // rejection usually means "the peer has not replicated the directory
+            // yet" and skipping would drop the entry permanently.
+            return await (0, syncStores_1.syncEntriesBetweenStores)(sourceStore, targetStore, options, {
+                logger: this.logger,
+                cursors: this.syncCursorStore(),
+                rejectionPolicy: "advance",
+            });
         }
         catch (error) {
             if (signal?.aborted) {
@@ -3413,258 +3416,10 @@ class BaseMindooDB {
             throw error;
         }
         finally {
-            BaseMindooDB.setSyncAbortSignalOnStore(sourceStore, undefined);
-            BaseMindooDB.setSyncAbortSignalOnStore(targetStore, undefined);
+            (0, syncStores_1.setSyncAbortSignalOnStore)(sourceStore, undefined);
+            (0, syncStores_1.setSyncAbortSignalOnStore)(targetStore, undefined);
         }
     }
-    async syncEntriesFromStoreImpl(sourceStore, targetStore, options) {
-        let transferred = 0;
-        let transferredBytes = 0;
-        let scanned = 0;
-        // Per-entry rejections reported by a witnessing target (sync-v5): the
-        // sync completes; rejected entries are surfaced to the caller as warnings.
-        const rejected = [];
-        const onProgress = options?.onProgress;
-        const pageSize = options?.pageSize ?? 1000;
-        const signal = options?.signal;
-        if (this.supportsCursorScan(sourceStore)) {
-            // Persisted-cursor fast path (sync-v5, phase 1): resume the metadata
-            // scan where the previous sync between this (source, target) pair left
-            // off — or skip the sync entirely when the source head shows nothing
-            // new. The heads are fetched up front (cheap; one request per side for
-            // network stores) and also anchor the epochs persisted with the cursor.
-            const cursorKey = this.syncScanCursorKey(sourceStore, targetStore);
-            const persisted = options?.forceFullScan
-                ? null
-                : this.syncScanCursors.get(cursorKey) ?? null;
-            const [sourceHead, targetHead] = await Promise.all([
-                this.getStoreHeadSafe(sourceStore),
-                this.getStoreHeadSafe(targetStore),
-            ]);
-            let cursor = null;
-            if (persisted && sourceHead && targetHead) {
-                if (persisted.sourceEpoch === sourceHead.epoch &&
-                    persisted.targetEpoch === targetHead.epoch) {
-                    if (sourceHead.maxReceiptOrder <= persisted.cursor.receiptOrder) {
-                        this.logger.debug(`Sync skip: source head ${sourceHead.maxReceiptOrder} already covered by persisted cursor (${cursorKey})`);
-                        onProgress?.({
-                            phase: 'preparing',
-                            message: 'Source unchanged since last sync, nothing to scan',
-                            transferredEntries: 0,
-                            scannedEntries: 0,
-                        });
-                        return { transferred: 0, scanned: 0, cancelled: false };
-                    }
-                    cursor = persisted.cursor;
-                    this.logger.debug(`Resuming sync scan from persisted cursor receiptOrder=${cursor.receiptOrder} (${cursorKey})`);
-                }
-                else {
-                    // Epoch change on either side: the cursor lineage is broken
-                    // (store reset / receipt-order migration) — full rescan.
-                    this.logger.info(`Sync cursor epoch changed for ${cursorKey}, discarding persisted cursor and re-scanning`);
-                    this.syncScanCursors.delete(cursorKey);
-                }
-            }
-            const persistScanCursor = (finalCursor) => {
-                if (finalCursor && sourceHead && targetHead) {
-                    this.saveSyncScanCursor(cursorKey, {
-                        sourceEpoch: sourceHead.epoch,
-                        targetEpoch: targetHead.epoch,
-                        cursor: finalCursor,
-                    });
-                }
-            };
-            const targetBloom = await this.getStoreBloomSummary(targetStore);
-            // Fixed progress denominator: the SOURCE's total entry count. The cursor
-            // scan below examines every source entry, so `scannedEntries/totalSourceEntries`
-            // is a real completion ratio. (Local stores serve this from a cached bloom
-            // summary; network stores answer with one extra request.) The target bloom
-            // stays dedicated to missing-id pre-screening.
-            const sourceBloom = await this.getStoreBloomSummary(sourceStore);
-            const totalSourceEstimate = sourceBloom?.totalIds;
-            onProgress?.({
-                phase: 'preparing',
-                message: 'Preparing to sync entries...',
-                transferredEntries: 0,
-                scannedEntries: 0,
-                totalSourceEntries: totalSourceEstimate,
-            });
-            // Upper scan bound: the source head captured BEFORE the scan started.
-            // Pushing to a witnessing target re-anchors the just-transferred
-            // entries to a fresh receiptOrder on the source (applyWitnessReceipts),
-            // i.e. past the running cursor — without this bound the scan would
-            // re-discover its own transfers and never terminate (first-sync loop).
-            // Entries beyond the bound (re-anchored or written concurrently) are
-            // picked up by the next sync.
-            const scanUpperBound = sourceHead?.maxReceiptOrder;
-            // IDs already transferred in this sync session. Guards against
-            // re-pushing entries the scan re-discovers when the target bloom
-            // snapshot is stale (fetched once up front; empty on a first sync, so
-            // it would classify every re-discovered id as "definitely missing"
-            // and skip the exact hasEntries check).
-            const transferredThisSync = new Set();
-            let currentPage = 0;
-            // Scan-page pipelining (sync-v5, phase 4): while page N is being
-            // filtered and transferred, page N+1 is already being fetched.
-            let nextPagePromise = null;
-            const discardPrefetch = () => {
-                // Swallow errors of an in-flight prefetch we will never consume
-                // (abort/cancel paths) to avoid unhandled rejections.
-                nextPagePromise?.catch(() => { });
-                nextPagePromise = null;
-            };
-            while (true) {
-                if (signal?.aborted) {
-                    discardPrefetch();
-                    persistScanCursor(cursor);
-                    return { transferred, transferredBytes, scanned, cancelled: true, rejected };
-                }
-                const page = nextPagePromise
-                    ? await nextPagePromise
-                    : await sourceStore.scanEntriesSince(cursor, pageSize);
-                nextPagePromise = page.hasMore
-                    ? sourceStore.scanEntriesSince(page.nextCursor, pageSize)
-                    : null;
-                currentPage++;
-                // Clamp the page to the scan bound. Pages are ordered by
-                // (receiptOrder, id), so everything past the first out-of-bound
-                // entry is out of bound as well.
-                let pageEntries = page.entries;
-                let reachedScanBound = false;
-                if (scanUpperBound !== undefined && pageEntries.length > 0) {
-                    const inBound = pageEntries.filter((m) => (m.receiptOrder ?? 0) <= scanUpperBound);
-                    if (inBound.length < pageEntries.length) {
-                        reachedScanBound = true;
-                        pageEntries = inBound;
-                    }
-                }
-                scanned += pageEntries.length;
-                if (signal?.aborted) {
-                    discardPrefetch();
-                    persistScanCursor(cursor);
-                    return { transferred, transferredBytes, scanned, cancelled: true, rejected };
-                }
-                if (pageEntries.length > 0) {
-                    onProgress?.({
-                        phase: 'transferring',
-                        message: `Scanned ${scanned} entries, checking for changes (page ${currentPage})...`,
-                        transferredEntries: transferred,
-                        scannedEntries: scanned,
-                        totalSourceEntries: totalSourceEstimate,
-                        currentPage,
-                    });
-                    const ids = pageEntries
-                        .map((m) => m.id)
-                        .filter((id) => !transferredThisSync.has(id));
-                    const missingIds = ids.length > 0
-                        ? await this.filterMissingIds(targetStore, ids, targetBloom)
-                        : [];
-                    if (signal?.aborted) {
-                        discardPrefetch();
-                        persistScanCursor(cursor);
-                        return { transferred, transferredBytes, scanned, cancelled: true, rejected };
-                    }
-                    if (missingIds.length > 0) {
-                        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, missingIds, options, {
-                            transferred,
-                            transferredBytes,
-                            scanned,
-                            totalSourceEntries: totalSourceEstimate,
-                            currentPage,
-                        });
-                        transferred = transferResult.transferred;
-                        transferredBytes = transferResult.transferredBytes;
-                        rejected.push(...transferResult.rejected);
-                        if (transferResult.cancelled) {
-                            // The current page's transfer did not complete: persist the
-                            // boundary of the last fully transferred page instead.
-                            discardPrefetch();
-                            persistScanCursor(cursor);
-                            return { transferred, transferredBytes, scanned, cancelled: true, rejected };
-                        }
-                        for (const id of missingIds) {
-                            transferredThisSync.add(id);
-                        }
-                    }
-                }
-                onProgress?.({
-                    phase: 'transferring',
-                    message: `Transferred ${transferred} entries (page ${currentPage}, scanned ${scanned})`,
-                    transferredEntries: transferred,
-                    transferredBytes,
-                    scannedEntries: scanned,
-                    totalSourceEntries: totalSourceEstimate,
-                    currentPage,
-                });
-                if (reachedScanBound) {
-                    // Persist the last in-bound position (NOT page.nextCursor, which
-                    // already points into the out-of-bound tail): a concurrent write
-                    // that landed between the bound and the re-anchored tail must be
-                    // scanned by the next sync.
-                    const lastInBound = pageEntries.length > 0 ? pageEntries[pageEntries.length - 1] : null;
-                    if (lastInBound) {
-                        cursor = { receiptOrder: lastInBound.receiptOrder ?? 0, id: lastInBound.id };
-                    }
-                    discardPrefetch();
-                    break;
-                }
-                cursor = page.nextCursor;
-                if (!page.hasMore) {
-                    break;
-                }
-            }
-            persistScanCursor(cursor);
-            return { transferred, transferredBytes, scanned, cancelled: false, rejected };
-        }
-        onProgress?.({
-            phase: 'preparing',
-            message: 'Finding new entries...',
-            transferredEntries: 0,
-            scannedEntries: 0,
-        });
-        const targetIds = await targetStore.getAllIds();
-        const sourceNewMetadata = await sourceStore.findNewEntries(targetIds);
-        if (sourceNewMetadata.length === 0) {
-            return { transferred: 0, scanned: 0, cancelled: false };
-        }
-        if (options?.signal?.aborted) {
-            return { transferred: 0, scanned: 0, cancelled: true };
-        }
-        onProgress?.({
-            phase: 'transferring',
-            message: `Transferring ${sourceNewMetadata.length} entries...`,
-            transferredEntries: transferred,
-            scannedEntries: sourceNewMetadata.length,
-            totalSourceEntries: sourceNewMetadata.length,
-        });
-        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, sourceNewMetadata.map((m) => m.id), options, {
-            transferred,
-            transferredBytes,
-            scanned: sourceNewMetadata.length,
-            totalSourceEntries: sourceNewMetadata.length,
-        });
-        transferred = transferResult.transferred;
-        transferredBytes = transferResult.transferredBytes;
-        rejected.push(...transferResult.rejected);
-        if (transferResult.cancelled) {
-            return { transferred, transferredBytes, scanned: sourceNewMetadata.length, cancelled: true, rejected };
-        }
-        onProgress?.({
-            phase: 'transferring',
-            message: `Transferred ${transferred} entries`,
-            transferredEntries: transferred,
-            transferredBytes,
-            scannedEntries: sourceNewMetadata.length,
-            totalSourceEntries: sourceNewMetadata.length,
-        });
-        return {
-            transferred,
-            transferredBytes,
-            scanned: sourceNewMetadata.length,
-            cancelled: false,
-            rejected,
-        };
-    }
     /**
      * Dense sync: transfer only the entries required to reconstruct the latest
      * state of each document, using the batch materialization planner to skip
@@ -4082,8 +3837,10 @@ class BaseMindooDB {
         }
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomSigningKey ? signingKeyPair : undefined);
         const createUsernameHash = this.usernameHashFromRecord(Object.fromEntries(initialValueEntries));
-        await this.assertBuiltinWriteAllowed("doc_create", signerPublicKey, createUsernameHash);
+        // The id is settled before the write gate because it decides which rule
+        // applies in `userdirectory`: a personal document is recognized by its id.
         const docId = useCustomDocId ? options.id : this.generateUnusedDocId(options.idPrefix);
+        await this.assertBuiltinWriteAllowed("doc_create", signerPublicKey, createUsernameHash, docId);
         // Idempotent create: when a caller-provided id already exists locally,
         // return the existing document instead of producing a duplicate doc_create.
         if (useCustomDocId) {
@@ -4318,8 +4075,8 @@ class BaseMindooDB {
             : [];
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomSigningKey ? signingKeyPair : undefined);
         const createUsernameHash = this.usernameHashFromRecord(Object.fromEntries(initialValueEntries));
-        await this.assertBuiltinWriteAllowed("doc_create", signerPublicKey, createUsernameHash);
         const docId = useCustomDocId ? options.id : this.generateUnusedDocId(options.idPrefix);
+        await this.assertBuiltinWriteAllowed("doc_create", signerPublicKey, createUsernameHash, docId);
         const now = (0, timeSource_1.semanticNow)();
         const keyId = await this.resolveCreateKeyId(options, docId);
         const sealedCreate = await this.prepareSealedCreate(options, docId, keyId, now, signerPublicKey);
@@ -4631,9 +4388,10 @@ class BaseMindooDB {
     }
     /**
      * Unwrap a `$sealed:` DEK from entry recipient wraps into the tenant session
-     * cache. `loadDocumentInternal` already does this; history / timestamp reads
-     * must as well — otherwise Haven's revision list throws SymmetricKeyNotFoundError
-     * on a cold tenant even when this device is in the wrap list.
+     * cache. `loadDocumentInternal` already does this; history, timestamp, and
+     * DAG reads must as well — otherwise Haven's revision list and DAG explorer
+     * throw SymmetricKeyNotFoundError on a cold tenant even when this device is
+     * in the wrap list.
      */
     async ingestSealedKeyFromEntries(entries) {
         if (entries.length === 0)
@@ -5298,6 +5056,9 @@ class BaseMindooDB {
         const startedAt = Date.now();
         const resolvedTimestamp = this.resolveDagTimestamp(timestamp);
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        if (allEntryMetadata.length > 0) {
+            await this.ingestSealedKeyFromEntries(allEntryMetadata);
+        }
         const relevantEntries = allEntryMetadata
             .filter((entry) => entry.createdAt <= resolvedTimestamp && (0, DocumentDagAnalysis_1.isDagEntry)(entry));
         const result = (0, DocumentDagAnalysis_1.computeDocumentDagAnalysis)(docId, relevantEntries, resolvedTimestamp);
@@ -5496,6 +5257,9 @@ class BaseMindooDB {
         let decodedChange = null;
         if (metadata.entryType !== "doc_snapshot") {
             const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+            if (allEntryMetadata.length > 0) {
+                await this.ingestSealedKeyFromEntries(allEntryMetadata);
+            }
             const entries = await this.store.getEntries([entryId]);
             const entry = entries[0];
             if (entry) {
@@ -6186,6 +5950,9 @@ class BaseMindooDB {
      *   decoded Automerge hash/dependency metadata for private replay.
      */
     async loadVerifiedReplayChanges(replayEntries) {
+        if (replayEntries.length > 0) {
+            await this.ingestSealedKeyFromEntries(replayEntries);
+        }
         const loadedEntries = replayEntries.length > 0
             ? await this.store.getEntries(replayEntries.map((entry) => entry.id))
             : [];
@@ -6747,7 +6514,17 @@ class BaseMindooDB {
         if (docIds.length === 0) {
             return;
         }
-        await this.assertLifecycleMutationAllowed("doc_delete", options.signingKeyPair, options.signingKeyPassword);
+        // One gate call per document id: inside `userdirectory` the applicable rule
+        // depends on the id (personal documents are owned by their creator), so a
+        // single batch-wide check would deny an owner deleting their own data.
+        if ((0, builtinDbInvariants_1.hasBuiltinWriteInvariant)(this.effectiveBuiltinDbId()) || this._isAdminOnlyDb) {
+            for (const docId of docIds) {
+                await this.assertLifecycleMutationAllowed("doc_delete", options.signingKeyPair, options.signingKeyPassword, undefined, docId);
+            }
+        }
+        else {
+            await this.assertLifecycleMutationAllowed("doc_delete", options.signingKeyPair, options.signingKeyPassword);
+        }
         this.logger.info(`Bulk-deleting ${docIds.length} documents`);
         // Unlike creation, a delete must first materialize the CURRENT document
         // (the doc_delete entry is an Automerge change on its heads), which costs
@@ -6794,14 +6571,14 @@ class BaseMindooDB {
     async undeleteDocument(docId, options = {}) {
         return this.undeleteDocInternal(docId, options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
     }
-    async assertLifecycleMutationAllowed(op, signingKeyPair, signingKeyPassword, documentUsernameHash) {
+    async assertLifecycleMutationAllowed(op, signingKeyPair, signingKeyPassword, documentUsernameHash, docId) {
         this.assertWritable("document lifecycle mutation");
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
         if ((signingKeyPair !== undefined) !== (signingKeyPassword !== undefined)) {
             throw new Error("Lifecycle mutation requires both signingKeyPair and signingKeyPassword");
         }
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomSigningKey ? signingKeyPair : undefined);
-        await this.assertBuiltinWriteAllowed(op, signerPublicKey, documentUsernameHash);
+        await this.assertBuiltinWriteAllowed(op, signerPublicKey, documentUsernameHash, docId);
     }
     /**
      * Write a delete or undelete lifecycle entry for an existing document.
@@ -6975,7 +6752,7 @@ class BaseMindooDB {
     async deleteDocInternal(docId, signingKeyPair, signingKeyPassword, bypassPrecheck) {
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
         this.logger.debug(`Deleting document ${docId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
-        await this.assertLifecycleMutationAllowed("doc_delete", signingKeyPair, signingKeyPassword);
+        await this.assertLifecycleMutationAllowed("doc_delete", signingKeyPair, signingKeyPassword, undefined, docId);
         // Get current document
         const internalDoc = await this.loadDocumentInternal(docId);
         if (!internalDoc || internalDoc.isDeleted) {
@@ -6991,7 +6768,7 @@ class BaseMindooDB {
         if (!internalDoc) {
             throw new errors_1.DocumentNotFoundError(docId);
         }
-        await this.assertLifecycleMutationAllowed("doc_undelete", signingKeyPair, signingKeyPassword, this.usernameHashFromInternalDoc(internalDoc));
+        await this.assertLifecycleMutationAllowed("doc_undelete", signingKeyPair, signingKeyPassword, this.usernameHashFromInternalDoc(internalDoc), docId);
         if (!internalDoc.isDeleted) {
             this.logger.debug(`Document ${docId} is already alive; undelete is a no-op`);
             return;
@@ -8293,7 +8070,7 @@ class BaseMindooDB {
             throw new Error("No Automerge changes to persist");
         }
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomKey ? signingKeyPair : undefined);
-        await this.assertBuiltinWriteAllowed("doc_change", signerPublicKey, this.usernameHashFromInternalDoc(internalDoc));
+        await this.assertBuiltinWriteAllowed("doc_change", signerPublicKey, this.usernameHashFromInternalDoc(internalDoc), docId);
         // Signed snapshot of the document's attachments after this change, computed
         // once from the resulting document state so EVERY change path (changeDoc,
         // text patches, merges, ...) carries it consistently on each produced entry.
@@ -8904,6 +8681,9 @@ class BaseMindooDB {
         if (replayEntriesForState.length === 0 && !snapshotEntryId) {
             return null;
         }
+        if (allEntryMetadata.length > 0) {
+            await this.ingestSealedKeyFromEntries(allEntryMetadata);
+        }
         const metadataById = new Map(allEntryMetadata.map((entry) => [entry.id, entry]));
         let startFromSnapshot = snapshotEntryId !== null;
         const snapshotMeta = snapshotEntryId ? (metadataById.get(snapshotEntryId) || null) : null;
@@ -11936,32 +11716,6 @@ class BaseMindooDB {
     }
 }
 exports.BaseMindooDB = BaseMindooDB;
-/**
- * Determine how many entry IDs to fetch per `getEntries` call during sync.
- *
- * This is intentionally separate from the metadata scan `pageSize` so that
- * scanning can page through large ID lists quickly while the heavier
- * payload downloads use a smaller batch to keep progress responsive and
- * cancellation timely.
- *
- * Priority: explicit option > attachment default (100) > docs default (250,
- * capped by pageSize).
- *
- * The docs default is intentionally smaller than the metadata scan pageSize
- * (1000): a full scan page transferred as one `putEntries` call serializes
- * to a multi-megabyte JSON body whose upload cannot report progress, risks
- * the HTTP timeout on slow uplinks (each retry restarts the whole body),
- * and can exceed the remote JSON body limit. Smaller batches keep progress
- * events frequent and make each POST cheap to retry.
- */
-BaseMindooDB.DEFAULT_DOCS_TRANSFER_BATCH_SIZE = 250;
-/**
- * Default window of transfer batches running in parallel (sync-v5,
- * phase 4). Kept small: each in-flight batch holds one decrypted page of
- * entries in memory, and HTTP/2 multiplexes the requests over a single
- * connection anyway. Overridable via `SyncOptions.maxConcurrentBatches`.
- */
-BaseMindooDB.DEFAULT_MAX_CONCURRENT_TRANSFER_BATCHES = 3;
 /** The "access control not enforced" allow decision returned by the
  * non-throwing `canCreate`/`canChange`/`canDelete` prediction helpers. */
 BaseMindooDB.ACL_NOT_ENFORCED_DECISION = {