mindoodb 0.0.33 → 0.0.35

package/dist/core/BaseMindooDB.js

@@ -15,6 +15,7 @@ const DocumentDagAnalysis_1 = require("./DocumentDagAnalysis");
 const DocumentConflictAnalysis_1 = require("./DocumentConflictAnalysis");
 const AttachmentReadPlanner_1 = require("./appendonlystores/AttachmentReadPlanner");
 const idGeneration_1 = require("./utils/idGeneration");
+const timeSource_1 = require("./utils/timeSource");
 const errors_1 = require("./errors");
 const EntrySignature_1 = require("./crypto/EntrySignature");
 const materializationGuard_1 = require("./accesscontrol/materializationGuard");
@@ -190,6 +191,19 @@ class BaseMindooDB {
         // Track which entry IDs we've already processed
         this.processedEntryIds = [];
         this.processedEntryCursor = null;
+        /**
+         * Persisted sync scan cursors (sync-v5, phase 1), keyed by
+         * `"<sourceIdentity>-><targetIdentity>/<storeKind>"` (see
+         * {@link syncScanCursorKey}). Each record remembers how far a previous
+         * pull/push already scanned the source store, together with the epochs of
+         * both stores at that time. On the next sync with the same pair the scan
+         * resumes from the cursor — or is skipped entirely when the source head
+         * shows nothing new. An epoch change on either side invalidates the record.
+         *
+         * Persisted in the metadata checkpoint when a cache store is configured;
+         * otherwise this only accelerates repeat syncs within the same session.
+         */
+        this.syncScanCursors = new Map();
         /**
          * In-memory LRU of per-document merged state used by the revision-grain
          * changefeed ({@link iterateRevisionsInternal}). Each entry holds a document's
@@ -1110,6 +1124,11 @@ class BaseMindooDB {
             hashMap[docId] = Object.fromEntries(inner);
         }
         checkpoint.automergeHashToEntryId = hashMap;
+        // Persisted sync scan cursors (sync-v5, phase 1). Additive field: older
+        // readers ignore it, older checkpoints simply restore an empty map.
+        if (this.syncScanCursors.size > 0) {
+            checkpoint.syncScanCursors = Object.fromEntries(this.syncScanCursors);
+        }
         // For stores without cursor scan, include processedEntryIds
         if (!this.supportsCursorScan(this.store)) {
             checkpoint.processedEntryIds = this.processedEntryIds;
@@ -1149,6 +1168,21 @@ class BaseMindooDB {
             }
             this.lastReconciledKeyBagFingerprint =
                 typeof checkpoint.keyBagFingerprint === "string" ? checkpoint.keyBagFingerprint : null;
+            this.syncScanCursors.clear();
+            if (checkpoint.syncScanCursors && typeof checkpoint.syncScanCursors === "object") {
+                for (const [key, value] of Object.entries(checkpoint.syncScanCursors)) {
+                    if (typeof value?.sourceEpoch === "string" &&
+                        typeof value?.targetEpoch === "string" &&
+                        typeof value?.cursor?.receiptOrder === "number" &&
+                        typeof value?.cursor?.id === "string") {
+                        this.syncScanCursors.set(key, {
+                            sourceEpoch: value.sourceEpoch,
+                            targetEpoch: value.targetEpoch,
+                            cursor: { receiptOrder: value.cursor.receiptOrder, id: value.cursor.id },
+                        });
+                    }
+                }
+            }
             // Rebuild indexLookup from index
             this.indexLookup.clear();
             this.indexLookupStaleFrom = null;
@@ -1405,7 +1439,7 @@ class BaseMindooDB {
             if (cutoff === undefined) {
                 return false;
             }
-            return (0, storeEntryTime_1.entryTrustedTime)(entry, Date.now()) >= cutoff;
+            return (0, storeEntryTime_1.entryTrustedTime)(entry, (0, timeSource_1.semanticNow)()) >= cutoff;
         }
         catch (error) {
             this.logger.warn(`Failed to resolve metadata-signature floor: ${String(error)}`);
@@ -1437,14 +1471,23 @@ class BaseMindooDB {
      * keeps repeated visibility reconciliations from inflating the
      * changefeed.
      *
+     * `lastModified` has millisecond resolution, so two genuine content
+     * changes landing in the same millisecond are indistinguishable by the
+     * tracked fields alone. Call sites that KNOW new content entries were
+     * just written/applied (changeDoc, delete/undelete, sync ingest) must
+     * pass `forceChangeSeqBump: true` so the change is re-emitted on the
+     * changefeed instead of being swallowed by the idempotency check.
+     *
      * @param docId The document ID
      * @param lastModified The new last modified timestamp
      * @param isDeleted Whether the document is a deletion/inaccessibility tombstone
      * @param decryptionKeyId Encryption key id derived from the doc's store metadata
      * @param accessState `"visible"` for readable docs, `"inaccessible"` for
      *   tombstones produced by the key visibility layer
+     * @param forceChangeSeqBump Bypass the no-op identity check because the
+     *   caller knows the document content changed (new store entries)
      */
-    updateIndex(docId, lastModified, isDeleted, decryptionKeyId = "default", accessState = "visible", awaitingWitness = false, witnessed = false) {
+    updateIndex(docId, lastModified, isDeleted, decryptionKeyId = "default", accessState = "visible", awaitingWitness = false, witnessed = false, forceChangeSeqBump = false) {
         const startedAt = Date.now();
         const assignedSeq = this.nextChangeSeq;
         const newEntry = {
@@ -1464,8 +1507,12 @@ class BaseMindooDB {
             // If only metadata flags stayed identical and caller replays same state, skip.
             // `awaitingWitness` is part of this identity so a witness receipt (which
             // changes only `receivedAt`, not Automerge content) still bumps changeSeq
-            // and re-emits the doc on the metadata feed.
-            if (existingEntry.lastModified === lastModified
+            // and re-emits the doc on the metadata feed. Content-change call sites
+            // bypass this via forceChangeSeqBump: two changes within the same
+            // millisecond would otherwise be indistinguishable here and the second
+            // one would silently vanish from the changefeed.
+            if (!forceChangeSeqBump
+                && existingEntry.lastModified === lastModified
                 && existingEntry.isDeleted === isDeleted
                 && existingEntry.decryptionKeyId === decryptionKeyId
                 && existingEntry.accessState === accessState
@@ -2048,8 +2095,11 @@ class BaseMindooDB {
                         updatedDoc = await this.applyNewEntriesToCachedDocument(cachedDoc, entryMetadataList);
                         if (updatedDoc) {
                             this.logger.debug(`Successfully updated cached document ${docId} incrementally`);
-                            // Only update index if document actually changed
-                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false);
+                            // Only update index if document actually changed. A non-null
+                            // result means something observable changed (heads or witness
+                            // state), so force the changeSeq bump — same-millisecond
+                            // lastModified collisions must not swallow the re-emit.
+                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false, true);
                             this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted}, awaitingWitness: ${updatedDoc.awaitingWitness ?? false})`);
                         }
                         else {
@@ -2062,7 +2112,8 @@ class BaseMindooDB {
                         this.docCache.delete(docId);
                         updatedDoc = await this.loadDocumentInternal(docId);
                         if (updatedDoc) {
-                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false);
+                            // New entries triggered this reload, so force the re-emit.
+                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false, true);
                         }
                     }
                 }
@@ -2096,7 +2147,11 @@ class BaseMindooDB {
                             const lastModified = Math.max(...entriesForLastModified.map((e) => e.createdAt));
                             const isDeleted = this.computeIsDeletedFromMetadata(allDocMetadata);
                             const { awaitingWitness, witnessed } = (0, storeEntryTime_1.metadataWitnessState)(allDocMetadata);
-                            this.updateIndex(docId, lastModified, isDeleted, representativeEntry.decryptionKeyId, "visible", awaitingWitness, witnessed);
+                            // Force the re-emit only when the batch contains actual replay
+                            // entries (create/change/delete/undelete): a snapshot-only batch
+                            // changes nothing observable and may keep the idempotent skip.
+                            const batchHasReplayEntries = docLifecycleEntries.some((e) => this.isDocumentReplayEntry(e));
+                            this.updateIndex(docId, lastModified, isDeleted, representativeEntry.decryptionKeyId, "visible", awaitingWitness, witnessed, batchHasReplayEntries);
                             this.logger.debug(`Metadata-first update for uncached doc ${docId} (lastModified: ${lastModified}, isDeleted: ${isDeleted}, awaitingWitness: ${awaitingWitness})`);
                         }
                     }
@@ -2169,6 +2224,16 @@ class BaseMindooDB {
         };
         const documentEntries = Array.from(entriesByDoc.entries())
             .sort((a, b) => {
+            // Prefer the store's receiptOrder (unique, reflects the actual write
+            // order) so a cold rebuild assigns changeSeqs in the same order the
+            // live session did — `createdAt` has millisecond resolution and ties
+            // for same-ms writes, which would otherwise flip the feed order to
+            // the docId tie-break after a restart.
+            const aMinReceipt = Math.min(...a[1].map((e) => e.receiptOrder ?? Number.MAX_SAFE_INTEGER));
+            const bMinReceipt = Math.min(...b[1].map((e) => e.receiptOrder ?? Number.MAX_SAFE_INTEGER));
+            if (aMinReceipt !== bMinReceipt) {
+                return aMinReceipt - bMinReceipt;
+            }
             const aMinCreatedAt = Math.min(...a[1].map((e) => e.createdAt));
             const bMinCreatedAt = Math.min(...b[1].map((e) => e.createdAt));
             if (aMinCreatedAt !== bMinCreatedAt) {
@@ -2281,18 +2346,20 @@ class BaseMindooDB {
         };
     }
     /**
-     * Fetch the target store's bloom filter summary, returning null if
-     * unsupported or on error (callers fall back to exact checks).
+     * Fetch a store's bloom filter summary, returning null if unsupported or on
+     * error. Used on the target for missing-id pre-screening (callers fall back
+     * to exact checks) and on the source for the total-entry-count progress
+     * denominator.
      */
-    async getTargetBloomSummary(targetStore) {
-        if (typeof targetStore.getIdBloomSummary !== "function") {
+    async getStoreBloomSummary(store) {
+        if (typeof store.getIdBloomSummary !== "function") {
             return null;
         }
         try {
-            return await targetStore.getIdBloomSummary();
+            return await store.getIdBloomSummary();
         }
         catch (error) {
-            this.logger.warn("Failed to get bloom summary from target store, falling back to exact checks", error);
+            this.logger.warn("Failed to get bloom summary from store, falling back to exact checks", error);
             return null;
         }
     }
@@ -2301,6 +2368,46 @@ class BaseMindooDB {
             store.setSyncAbortSignal(signal);
         }
     }
+    /**
+     * Fetch a store's head descriptor (`{ epoch, maxReceiptOrder }`), returning
+     * null when the store (or the remote server behind it) does not support it
+     * or the request fails. A null head simply disables the persisted-cursor
+     * fast path for this sync.
+     */
+    async getStoreHeadSafe(store) {
+        if (typeof store.getStoreHead !== "function") {
+            return null;
+        }
+        try {
+            return await store.getStoreHead();
+        }
+        catch (error) {
+            this.logger.debug("Store head unavailable, falling back to full scan", error);
+            return null;
+        }
+    }
+    /** Stable identity for a store in the persisted sync-cursor map. */
+    static syncStoreIdentity(store) {
+        return store.getCacheIdentity?.() ?? `${store.getId()}/${store.getStoreKind()}`;
+    }
+    /** Key for {@link syncScanCursors}: one record per (source, target) pair. */
+    syncScanCursorKey(sourceStore, targetStore) {
+        return `${BaseMindooDB.syncStoreIdentity(sourceStore)}->${BaseMindooDB.syncStoreIdentity(targetStore)}`;
+    }
+    /** Persist a sync scan cursor and schedule a checkpoint flush. */
+    saveSyncScanCursor(key, record) {
+        const existing = this.syncScanCursors.get(key);
+        if (existing &&
+            existing.sourceEpoch === record.sourceEpoch &&
+            existing.targetEpoch === record.targetEpoch &&
+            existing.cursor.receiptOrder === record.cursor.receiptOrder &&
+            existing.cursor.id === record.cursor.id) {
+            return;
+        }
+        this.syncScanCursors.set(key, record);
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+    }
     /**
      * From a list of candidate IDs, return only those the target store is
      * missing.  Uses bloom-filter pre-screening when available, then falls
@@ -2329,16 +2436,6 @@ class BaseMindooDB {
         }
         return missingIds;
     }
-    /**
-     * Determine how many entry IDs to fetch per `getEntries` call during sync.
-     *
-     * This is intentionally separate from the metadata scan `pageSize` so that
-     * scanning can page through large ID lists quickly while the heavier
-     * payload downloads use a smaller batch to keep progress responsive and
-     * cancellation timely.
-     *
-     * Priority: explicit option > attachment default (100) > pageSize fallback.
-     */
     resolveTransferBatchSize(options) {
         if (options?.transferBatchSize && options.transferBatchSize > 0) {
             return options.transferBatchSize;
@@ -2346,7 +2443,7 @@ class BaseMindooDB {
         if (options?.storeKind === types_2.StoreKind.attachments) {
             return 100;
         }
-        return options?.pageSize ?? 1000;
+        return Math.min(options?.pageSize ?? 1000, BaseMindooDB.DEFAULT_DOCS_TRANSFER_BATCH_SIZE);
     }
     /**
      * Transfer a set of entry IDs from source to target in fixed-size batches,
@@ -2368,65 +2465,120 @@ class BaseMindooDB {
      */
     async transferEntriesInBatches(sourceStore, targetStore, entryIds, options, state) {
         if (entryIds.length === 0) {
-            return { transferred: state.transferred, cancelled: false };
+            return { transferred: state.transferred, cancelled: false, rejected: [] };
         }
         const onProgress = options?.onProgress;
         const signal = options?.signal;
         const transferBatchSize = this.resolveTransferBatchSize(options);
         const totalTransferBatches = Math.max(1, Math.ceil(entryIds.length / transferBatchSize));
+        // Parallel transfer window (sync-v5, phase 4): each batch is one
+        // getEntries + putEntries round trip; a small window of concurrent
+        // batches overlaps network latency and server-side crypto. Witness
+        // receipts are applied per batch and are order-independent.
+        const maxConcurrent = Math.min(Math.max(1, Math.floor(options?.maxConcurrentBatches ?? BaseMindooDB.DEFAULT_MAX_CONCURRENT_TRANSFER_BATCHES)), totalTransferBatches);
         let transferred = state.transferred;
-        for (let offset = 0; offset < entryIds.length; offset += transferBatchSize) {
-            if (signal?.aborted) {
-                return { transferred, cancelled: true };
-            }
-            const currentTransferBatch = Math.floor(offset / transferBatchSize) + 1;
-            const batchIds = entryIds.slice(offset, offset + transferBatchSize);
-            const pageSummary = state.currentPage ? `page ${state.currentPage}, ` : "";
-            onProgress?.({
-                phase: "transferring",
-                message: `Transferring batch ${currentTransferBatch}/${totalTransferBatches} (${batchIds.length} entries, ${pageSummary}scanned ${state.scanned})...`,
-                transferredEntries: transferred,
-                scannedEntries: state.scanned,
-                totalSourceEntries: state.totalSourceEntries,
-                currentPage: state.currentPage,
-                currentTransferBatch,
-                totalTransferBatches,
-                transferBatchSize,
-            });
-            try {
-                const batchEntries = await sourceStore.getEntries(batchIds);
-                if (signal?.aborted) {
-                    return { transferred, cancelled: true };
+        let cancelled = false;
+        let firstError = null;
+        let nextBatchIndex = 0;
+        // Per-entry rejections reported by a witnessing target (sync-v5): the
+        // push continues, the rejected entries are surfaced to the caller.
+        const rejected = [];
+        const runWorker = async () => {
+            while (true) {
+                if (firstError || cancelled || signal?.aborted) {
+                    if (signal?.aborted)
+                        cancelled = true;
+                    return;
                 }
-                const putResult = await targetStore.putEntries(batchEntries);
-                // On push, the remote (witnessing) target returns receipts for accepted
-                // entries; persist them back onto the local source so the revision feed
-                // re-anchors them from the provisional head to their committed
-                // `receivedAt` (docs/accesscontrol.md §5.3). Pull targets return void.
-                if (Array.isArray(putResult) && putResult.length > 0 && sourceStore.applyWitnessReceipts) {
-                    await sourceStore.applyWitnessReceipts(putResult);
+                const batchIndex = nextBatchIndex++;
+                const offset = batchIndex * transferBatchSize;
+                if (offset >= entryIds.length) {
+                    return;
                 }
-                transferred += batchEntries.length;
-            }
-            catch (error) {
-                if (signal?.aborted) {
-                    return { transferred, cancelled: true };
+                const currentTransferBatch = batchIndex + 1;
+                const batchIds = entryIds.slice(offset, offset + transferBatchSize);
+                const pageSummary = state.currentPage ? `page ${state.currentPage}, ` : "";
+                onProgress?.({
+                    phase: "transferring",
+                    message: `Transferring batch ${currentTransferBatch}/${totalTransferBatches} (${batchIds.length} entries, ${pageSummary}scanned ${state.scanned})...`,
+                    transferredEntries: transferred,
+                    scannedEntries: state.scanned,
+                    totalSourceEntries: state.totalSourceEntries,
+                    currentPage: state.currentPage,
+                    currentTransferBatch,
+                    totalTransferBatches,
+                    transferBatchSize,
+                });
+                try {
+                    const batchEntries = await sourceStore.getEntries(batchIds);
+                    if (signal?.aborted) {
+                        cancelled = true;
+                        return;
+                    }
+                    const putResult = await targetStore.putEntries(batchEntries);
+                    // On push, the remote (witnessing) target returns receipts for accepted
+                    // entries; persist them back onto the local source so the revision feed
+                    // re-anchors them from the provisional head to their committed
+                    // `receivedAt` (docs/accesscontrol.md §5.3). Pull targets return void.
+                    // sync-v5 targets return a structured ack that additionally carries
+                    // per-entry rejections (signature-class failures the remote skipped).
+                    const { receipts, batchRejected } = BaseMindooDB.normalizePutResult(putResult);
+                    if (receipts.length > 0 && sourceStore.applyWitnessReceipts) {
+                        await sourceStore.applyWitnessReceipts(receipts);
+                    }
+                    if (batchRejected.length > 0) {
+                        rejected.push(...batchRejected);
+                        this.logger.warn(`Target rejected ${batchRejected.length} of ${batchEntries.length} entries in transfer batch ${currentTransferBatch}/${totalTransferBatches}`);
+                    }
+                    transferred += batchEntries.length - batchRejected.length;
                 }
-                throw error;
+                catch (error) {
+                    if (signal?.aborted) {
+                        cancelled = true;
+                        return;
+                    }
+                    if (!firstError) {
+                        firstError = error;
+                    }
+                    return;
+                }
+                onProgress?.({
+                    phase: "transferring",
+                    message: `Transferred ${transferred} entries after batch ${currentTransferBatch}/${totalTransferBatches}`,
+                    transferredEntries: transferred,
+                    scannedEntries: state.scanned,
+                    totalSourceEntries: state.totalSourceEntries,
+                    currentPage: state.currentPage,
+                    currentTransferBatch,
+                    totalTransferBatches,
+                    transferBatchSize,
+                });
             }
-            onProgress?.({
-                phase: "transferring",
-                message: `Transferred ${transferred} entries after batch ${currentTransferBatch}/${totalTransferBatches}`,
-                transferredEntries: transferred,
-                scannedEntries: state.scanned,
-                totalSourceEntries: state.totalSourceEntries,
-                currentPage: state.currentPage,
-                currentTransferBatch,
-                totalTransferBatches,
-                transferBatchSize,
-            });
+        };
+        await Promise.all(Array.from({ length: maxConcurrent }, () => runWorker()));
+        if (signal?.aborted || cancelled) {
+            return { transferred, cancelled: true, rejected };
         }
-        return { transferred, cancelled: false };
+        if (firstError) {
+            throw firstError;
+        }
+        return { transferred, cancelled: false, rejected };
+    }
+    /**
+     * Normalize the polymorphic putEntries result (`void`, receipts array, or
+     * structured {@link PutEntriesAck}) into receipts + per-entry rejections.
+     */
+    static normalizePutResult(putResult) {
+        if (Array.isArray(putResult)) {
+            return { receipts: putResult, batchRejected: [] };
+        }
+        if (putResult && typeof putResult === "object") {
+            return {
+                receipts: putResult.receipts ?? [],
+                batchRejected: putResult.rejected ?? [],
+            };
+        }
+        return { receipts: [], batchRejected: [] };
     }
     async syncEntriesFromStore(sourceStore, targetStore, options) {
         const signal = options?.signal;
@@ -2453,12 +2605,67 @@ class BaseMindooDB {
     async syncEntriesFromStoreImpl(sourceStore, targetStore, options) {
         let transferred = 0;
         let scanned = 0;
+        // Per-entry rejections reported by a witnessing target (sync-v5): the
+        // sync completes; rejected entries are surfaced to the caller as warnings.
+        const rejected = [];
         const onProgress = options?.onProgress;
         const pageSize = options?.pageSize ?? 1000;
         const signal = options?.signal;
-        const targetBloom = await this.getTargetBloomSummary(targetStore);
-        const totalSourceEstimate = targetBloom?.totalIds;
         if (this.supportsCursorScan(sourceStore)) {
+            // Persisted-cursor fast path (sync-v5, phase 1): resume the metadata
+            // scan where the previous sync between this (source, target) pair left
+            // off — or skip the sync entirely when the source head shows nothing
+            // new. The heads are fetched up front (cheap; one request per side for
+            // network stores) and also anchor the epochs persisted with the cursor.
+            const cursorKey = this.syncScanCursorKey(sourceStore, targetStore);
+            const persisted = options?.forceFullScan
+                ? null
+                : this.syncScanCursors.get(cursorKey) ?? null;
+            const [sourceHead, targetHead] = await Promise.all([
+                this.getStoreHeadSafe(sourceStore),
+                this.getStoreHeadSafe(targetStore),
+            ]);
+            let cursor = null;
+            if (persisted && sourceHead && targetHead) {
+                if (persisted.sourceEpoch === sourceHead.epoch &&
+                    persisted.targetEpoch === targetHead.epoch) {
+                    if (sourceHead.maxReceiptOrder <= persisted.cursor.receiptOrder) {
+                        this.logger.debug(`Sync skip: source head ${sourceHead.maxReceiptOrder} already covered by persisted cursor (${cursorKey})`);
+                        onProgress?.({
+                            phase: 'preparing',
+                            message: 'Source unchanged since last sync, nothing to scan',
+                            transferredEntries: 0,
+                            scannedEntries: 0,
+                        });
+                        return { transferred: 0, scanned: 0, cancelled: false };
+                    }
+                    cursor = persisted.cursor;
+                    this.logger.debug(`Resuming sync scan from persisted cursor receiptOrder=${cursor.receiptOrder} (${cursorKey})`);
+                }
+                else {
+                    // Epoch change on either side: the cursor lineage is broken
+                    // (store reset / receipt-order migration) — full rescan.
+                    this.logger.info(`Sync cursor epoch changed for ${cursorKey}, discarding persisted cursor and re-scanning`);
+                    this.syncScanCursors.delete(cursorKey);
+                }
+            }
+            const persistScanCursor = (finalCursor) => {
+                if (finalCursor && sourceHead && targetHead) {
+                    this.saveSyncScanCursor(cursorKey, {
+                        sourceEpoch: sourceHead.epoch,
+                        targetEpoch: targetHead.epoch,
+                        cursor: finalCursor,
+                    });
+                }
+            };
+            const targetBloom = await this.getStoreBloomSummary(targetStore);
+            // Fixed progress denominator: the SOURCE's total entry count. The cursor
+            // scan below examines every source entry, so `scannedEntries/totalSourceEntries`
+            // is a real completion ratio. (Local stores serve this from a cached bloom
+            // summary; network stores answer with one extra request.) The target bloom
+            // stays dedicated to missing-id pre-screening.
+            const sourceBloom = await this.getStoreBloomSummary(sourceStore);
+            const totalSourceEstimate = sourceBloom?.totalIds;
             onProgress?.({
                 phase: 'preparing',
                 message: 'Preparing to sync entries...',
@@ -2466,17 +2673,34 @@ class BaseMindooDB {
                 scannedEntries: 0,
                 totalSourceEntries: totalSourceEstimate,
             });
-            let cursor = null;
             let currentPage = 0;
+            // Scan-page pipelining (sync-v5, phase 4): while page N is being
+            // filtered and transferred, page N+1 is already being fetched.
+            let nextPagePromise = null;
+            const discardPrefetch = () => {
+                // Swallow errors of an in-flight prefetch we will never consume
+                // (abort/cancel paths) to avoid unhandled rejections.
+                nextPagePromise?.catch(() => { });
+                nextPagePromise = null;
+            };
             while (true) {
                 if (signal?.aborted) {
-                    return { transferred, scanned, cancelled: true };
+                    discardPrefetch();
+                    persistScanCursor(cursor);
+                    return { transferred, scanned, cancelled: true, rejected };
                 }
-                const page = await sourceStore.scanEntriesSince(cursor, pageSize);
+                const page = nextPagePromise
+                    ? await nextPagePromise
+                    : await sourceStore.scanEntriesSince(cursor, pageSize);
+                nextPagePromise = page.hasMore
+                    ? sourceStore.scanEntriesSince(page.nextCursor, pageSize)
+                    : null;
                 currentPage++;
                 scanned += page.entries.length;
                 if (signal?.aborted) {
-                    return { transferred, scanned, cancelled: true };
+                    discardPrefetch();
+                    persistScanCursor(cursor);
+                    return { transferred, scanned, cancelled: true, rejected };
                 }
                 if (page.entries.length > 0) {
                     onProgress?.({
@@ -2490,7 +2714,9 @@ class BaseMindooDB {
                     const ids = page.entries.map((m) => m.id);
                     const missingIds = await this.filterMissingIds(targetStore, ids, targetBloom);
                     if (signal?.aborted) {
-                        return { transferred, scanned, cancelled: true };
+                        discardPrefetch();
+                        persistScanCursor(cursor);
+                        return { transferred, scanned, cancelled: true, rejected };
                     }
                     if (missingIds.length > 0) {
                         const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, missingIds, options, {
@@ -2500,8 +2726,13 @@ class BaseMindooDB {
                             currentPage,
                         });
                         transferred = transferResult.transferred;
+                        rejected.push(...transferResult.rejected);
                         if (transferResult.cancelled) {
-                            return { transferred, scanned, cancelled: true };
+                            // The current page's transfer did not complete: persist the
+                            // boundary of the last fully transferred page instead.
+                            discardPrefetch();
+                            persistScanCursor(cursor);
+                            return { transferred, scanned, cancelled: true, rejected };
                         }
                     }
                 }
@@ -2518,7 +2749,8 @@ class BaseMindooDB {
                     break;
                 }
             }
-            return { transferred, scanned, cancelled: false };
+            persistScanCursor(cursor);
+            return { transferred, scanned, cancelled: false, rejected };
         }
         onProgress?.({
             phase: 'preparing',
@@ -2547,8 +2779,9 @@ class BaseMindooDB {
             totalSourceEntries: sourceNewMetadata.length,
         });
         transferred = transferResult.transferred;
+        rejected.push(...transferResult.rejected);
         if (transferResult.cancelled) {
-            return { transferred, scanned: sourceNewMetadata.length, cancelled: true };
+            return { transferred, scanned: sourceNewMetadata.length, cancelled: true, rejected };
         }
         onProgress?.({
             phase: 'transferring',
@@ -2561,6 +2794,7 @@ class BaseMindooDB {
             transferred,
             scanned: sourceNewMetadata.length,
             cancelled: false,
+            rejected,
         };
     }
     /**
@@ -2636,7 +2870,7 @@ class BaseMindooDB {
         }
         // ── Phase 4: filter out entries the target already has ───────────
         const allNeededArray = Array.from(neededIds);
-        const targetBloom = await this.getTargetBloomSummary(targetStore);
+        const targetBloom = await this.getStoreBloomSummary(targetStore);
         const missingIds = await this.filterMissingIds(targetStore, allNeededArray, targetBloom);
         this.logger.info(`Dense sync: ${missingIds.length} entries to transfer ` +
             `(${allNeededArray.length - missingIds.length} already present)`);
@@ -2649,19 +2883,30 @@ class BaseMindooDB {
         // ── Phase 5: transfer missing entries in pages ───────────────────
         const pageSize = options?.pageSize ?? 500;
         let transferred = 0;
+        const rejected = [];
         for (let offset = 0; offset < missingIds.length; offset += pageSize) {
             if (options?.signal?.aborted) {
-                return { transferred, scanned: allNeededArray.length, cancelled: true };
+                return { transferred, scanned: allNeededArray.length, cancelled: true, rejected };
             }
             const batch = missingIds.slice(offset, offset + pageSize);
             const entries = await sourceStore.getEntries(batch);
             const putResult = await targetStore.putEntries(entries);
             // Persist any witness receipts back onto the local source (see push path
-            // in transferEntriesInBatches; docs/accesscontrol.md §5.3).
-            if (Array.isArray(putResult) && putResult.length > 0 && sourceStore.applyWitnessReceipts) {
-                await sourceStore.applyWitnessReceipts(putResult);
-            }
-            transferred += entries.length;
+            // in transferEntriesInBatches; docs/accesscontrol.md §5.3). sync-v5
+            // targets return a structured ack that additionally carries per-entry
+            // rejections (signature-class failures the remote skipped) — surface
+            // them instead of silently counting rejected entries as transferred:
+            // a dropped lifecycle entry (e.g. doc_delete) would otherwise leave the
+            // target believing the document is still alive.
+            const { receipts, batchRejected } = BaseMindooDB.normalizePutResult(putResult);
+            if (receipts.length > 0 && sourceStore.applyWitnessReceipts) {
+                await sourceStore.applyWitnessReceipts(receipts);
+            }
+            if (batchRejected.length > 0) {
+                rejected.push(...batchRejected);
+                this.logger.warn(`Dense sync: target rejected ${batchRejected.length} of ${entries.length} entries in batch at offset ${offset}`);
+            }
+            transferred += entries.length - batchRejected.length;
             onProgress?.({
                 phase: "transferring",
                 message: `Dense sync: transferred ${transferred}/${missingIds.length} entries`,
@@ -2670,8 +2915,9 @@ class BaseMindooDB {
                 totalSourceEntries: allNeededArray.length,
             });
         }
-        this.logger.info(`Dense sync complete: transferred ${transferred} entries`);
-        return { transferred, scanned: allNeededArray.length, cancelled: false };
+        this.logger.info(`Dense sync complete: transferred ${transferred} entries` +
+            (rejected.length > 0 ? `, ${rejected.length} rejected by target` : ""));
+        return { transferred, scanned: allNeededArray.length, cancelled: false, rejected };
     }
     getStore() {
         return this.store;
@@ -2681,7 +2927,7 @@ class BaseMindooDB {
     }
     async reclaimIncompleteAttachmentUploads(options) {
         const minAgeMs = options?.minAgeMs ?? 5 * 60 * 1000;
-        const cutoff = Date.now() - minAgeMs;
+        const cutoff = (0, timeSource_1.semanticNow)() - minAgeMs;
         const ledgers = await this.store.findEntries("pending_attachment_upload", null, cutoff);
         const result = {
             scannedLedgers: ledgers.length,
@@ -2784,6 +3030,7 @@ class BaseMindooDB {
                 throw new Error(`createDocuments: invalid document id "${options.id}". ` +
                     `Custom document IDs must match ${types_1.CUSTOM_DOC_ID_REGEX.source}.`);
             }
+            this.validateIdPrefixOption("createDocuments", options);
         }
         const results = new Array(inputs.length).fill(null);
         const prepared = [];
@@ -2840,18 +3087,68 @@ class BaseMindooDB {
         this.logger.info(`Bulk-created ${prepared.length} documents (${inputs.length - prepared.length} already existed)`);
         return results;
     }
+    /**
+     * Validate `CreateOptions.idPrefix`: mutually exclusive with `id` and must
+     * match {@link DOC_ID_PREFIX_REGEX}.
+     */
+    validateIdPrefixOption(methodName, options) {
+        if (options.idPrefix === undefined) {
+            return;
+        }
+        if (options.id !== undefined) {
+            throw new Error(`${methodName}: id and idPrefix are mutually exclusive`);
+        }
+        if (!types_1.DOC_ID_PREFIX_REGEX.test(options.idPrefix)) {
+            throw new Error(`${methodName}: invalid idPrefix "${options.idPrefix}". ` +
+                `ID prefixes must match ${types_1.DOC_ID_PREFIX_REGEX.source}.`);
+        }
+    }
+    /**
+     * Normalize a caller-supplied listing/iteration `idPrefix`, tolerating one or
+     * more trailing `_` a developer may append when they don't know the exact
+     * syntax (e.g. `"cls_"` → `"cls"`). Valid prefixes never contain `_` (see
+     * {@link DOC_ID_PREFIX_REGEX}), so trimming trailing underscores is lossless
+     * and makes `matchesDocIdPrefix` re-add the single boundary `_` consistently.
+     * Returns `undefined` for an absent or now-empty prefix (i.e. "no filter").
+     */
+    normalizeIdPrefixFilter(idPrefix) {
+        if (idPrefix === undefined) {
+            return undefined;
+        }
+        const stripped = idPrefix.replace(/_+$/, "");
+        return stripped.length > 0 ? stripped : undefined;
+    }
+    /**
+     * Generate a fresh document id (`<prefix>_<22-char-base62(uuidv7)>`, or the
+     * bare base62 part without a prefix) that does not collide with any document
+     * already present in the local index. A collision is practically impossible
+     * (128-bit UUID7), so this is only a cheap O(1) safety net reusing the
+     * existing `indexLookup` map — no extra bookkeeping.
+     */
+    generateUnusedDocId(prefix) {
+        for (;;) {
+            const candidate = (0, idGeneration_1.generateDocId)(prefix);
+            if (this.getDocIndexPosition(candidate) === undefined) {
+                return candidate;
+            }
+        }
+    }
     /**
      * Internal method to create a new document.
      *
-     * Handles all four creation flavors expressible through `CreateOptions`:
-     * - generated UUID7 ID, current user signs, default tenant key
-     * - generated UUID7 ID, custom signing key (e.g. directory admin operations)
+     * Handles the creation flavors expressible through `CreateOptions`:
+     * - generated ID (time-sortable base62-encoded UUID7), current user signs,
+     *   default tenant key
+     * - generated ID with custom signing key (e.g. directory admin operations)
+     * - generated ID with a caller-provided `idPrefix` (`<prefix>_<base62>`)
      * - caller-provided ID, current user signs (idempotent on existing IDs)
      * - caller-provided ID with custom signing key
      *
      * For caller-provided IDs the initial Automerge change is seeded from a
      * deterministic, hard-coded change so that two replicas creating the same
      * custom ID converge when synced (see `getCustomIdInitialChangeBytes`).
+     * Prefix-generated IDs are unique by construction, so they take the
+     * generated-ID path: one `doc_create` entry with `initialValues` baked in.
      */
     async createDocumentInternal(options) {
         this.assertWritable("createDocument");
@@ -2862,6 +3159,7 @@ class BaseMindooDB {
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
         const keyId = await this.resolveCreateKeyId(options);
         const useCustomDocId = options.id !== undefined;
+        this.validateIdPrefixOption("createDocument", options);
         // Sanitize caller-provided initial values: internal/reserved fields (those
         // starting with "_", e.g. "_attachments") are managed by MindooDB and must
         // not be seeded by callers. `undefined` values are treated as "not set"
@@ -2895,7 +3193,7 @@ class BaseMindooDB {
                 throw new Error("Admin-only database: only the admin key can modify data");
             }
         }
-        const docId = useCustomDocId ? options.id : (0, uuid_1.v7)();
+        const docId = useCustomDocId ? options.id : this.generateUnusedDocId(options.idPrefix);
         // Idempotent create: when a caller-provided id already exists locally,
         // return the existing document instead of producing a duplicate doc_create.
         if (useCustomDocId) {
@@ -2921,7 +3219,7 @@ class BaseMindooDB {
         // For custom-ID documents we apply a hard-coded initial change so that
         // independent replicas using the same id produce the same Automerge hash
         // and `doc_create` entry id, allowing later changes to merge.
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         let newDoc;
         let changeBytes;
         if (useCustomDocId) {
@@ -3103,6 +3401,7 @@ class BaseMindooDB {
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
         const keyId = await this.resolveCreateKeyId(options);
         const useCustomDocId = options.id !== undefined;
+        this.validateIdPrefixOption("createDocuments", options);
         // Sanitize caller-provided initial values: internal/reserved fields are
         // managed by MindooDB and must not be seeded by callers. `undefined`
         // values are treated as "not set" (Automerge cannot represent undefined).
@@ -3119,8 +3418,8 @@ class BaseMindooDB {
                 throw new Error("Admin-only database: only the admin key can modify data");
             }
         }
-        const docId = useCustomDocId ? options.id : (0, uuid_1.v7)();
-        const now = Date.now();
+        const docId = useCustomDocId ? options.id : this.generateUnusedDocId(options.idPrefix);
+        const now = (0, timeSource_1.semanticNow)();
         // Build the initial Automerge document and its first change bytes,
         // following the same seeding rules as the single-create path.
         let seedDoc;
@@ -3283,14 +3582,22 @@ class BaseMindooDB {
         // Filter to document replay metadata plus snapshots up to the timestamp.
         // Attachment chunks are not part of the Automerge replay DAG and must not
         // participate in historical materialization.
-        const relevantEntries = allEntryMetadata
+        //
+        // The slice is then pruned to grounded entries (complete causal chains
+        // within the slice). Clock skew or partial sync can put an entry inside
+        // `createdAt <= timestamp` while its dependency falls outside; Automerge
+        // would silently stash such a change, but the metadata-derived isDeleted
+        // below would still count it — pruning keeps content and lifecycle state
+        // consistent: the state at `timestamp` is the deterministic merge of all
+        // complete causal chains up to that instant.
+        const relevantEntries = (0, DocumentDagAnalysis_1.pruneToGroundedEntries)(allEntryMetadata
             .filter((em) => em.createdAt <= timestamp
             && (em.entryType === "doc_create"
                 || em.entryType === "doc_change"
                 || em.entryType === "doc_delete"
                 || em.entryType === "doc_undelete"
                 || em.entryType === "doc_snapshot"))
-            .sort((a, b) => a.createdAt - b.createdAt);
+            .sort((a, b) => a.createdAt - b.createdAt));
         if (relevantEntries.length === 0) {
             this.performanceCallback?.onHistoryOperation?.({
                 operation: "getDocumentAtTimestamp",
@@ -3411,10 +3718,13 @@ class BaseMindooDB {
         this.logger.debug(`Iterating document history for ${docId}`);
         // Get all entry metadata for this document
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
-        // Filter to document replay entries (exclude snapshots)
-        const relevantEntries = allEntryMetadata
-            .filter((em) => this.isDocumentReplayEntry(em))
-            .sort((a, b) => a.createdAt - b.createdAt);
+        // Filter to document replay entries (exclude snapshots) and order them
+        // causally: a plain chronological sort can place an effect before its
+        // cause when several entries share the same millisecond (or when replica
+        // clock skew inverts timestamps along a dependency chain), which would
+        // make the history yield states out of causal order. Concurrent entries
+        // are still ordered by (createdAt, id) deterministically.
+        const relevantEntries = (0, DocumentDagAnalysis_1.orderDagEntriesCausally)(allEntryMetadata.filter((em) => this.isDocumentReplayEntry(em)));
         if (relevantEntries.length === 0) {
             return; // Document has no history
         }
@@ -3426,7 +3736,8 @@ class BaseMindooDB {
         let currentDoc = null;
         let createdAt = null;
         let decryptionKeyId = "default";
-        for (const entryMetadata of relevantEntries) {
+        for (let entryIndex = 0; entryIndex < relevantEntries.length; entryIndex++) {
+            const entryMetadata = relevantEntries[entryIndex];
             const entryData = entryMap.get(entryMetadata.id);
             if (!entryData) {
                 this.logger.warn(`Entry ${entryMetadata.id} not found in store, skipping`);
@@ -3492,8 +3803,12 @@ class BaseMindooDB {
                     createdAt: createdAt,
                     lastModified: entryMetadata.createdAt,
                     decryptionKeyId,
-                    isDeleted: this.computeIsDeletedFromMetadata(relevantEntries.filter((entry) => entry.createdAt < entryMetadata.createdAt ||
-                        (entry.createdAt === entryMetadata.createdAt && entry.id <= entryMetadata.id))),
+                    // Use the exact prefix of entries applied so far. Rebuilding the
+                    // prefix via a (createdAt, id) comparison can disagree with the
+                    // actual iteration order when several entries share the same
+                    // millisecond (e.g. a fast create→change→delete), which made
+                    // intermediate history states wrongly report isDeleted.
+                    isDeleted: this.computeIsDeletedFromMetadata(relevantEntries.slice(0, entryIndex + 1)),
                 };
                 // Wrap and yield (including lifecycle terminal entries)
                 const wrappedDoc = this.wrapDocument(internalDoc);
@@ -3548,7 +3863,7 @@ class BaseMindooDB {
         }
         // Trusted time for un-witnessed entries is the current wall clock, computed
         // once per run so all un-witnessed entries share a stable head instant.
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         // Affected documents (those with at least one newly-discovered entry), and
         // the set of new entry ids per doc (the trusted-time positions from which the
         // doc's merged state may have changed and must be re-folded / re-emitted).
@@ -3968,24 +4283,28 @@ class BaseMindooDB {
         const offset = Math.max(0, Math.floor(options?.cursor?.offset ?? 0));
         // This API stays metadata-only so large history views can page cheaply
         // without reconstructing every historical Automerge document state.
+        // Entries are ordered causally (dependency-first, (createdAt, id) for
+        // concurrent ties) so the page timeline never shows an effect before its
+        // cause — same ordering as iterateDocumentHistory.
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
-        const relevantEntries = allEntryMetadata
-            .filter((em) => em.entryType === "doc_create" ||
+        const relevantEntries = (0, DocumentDagAnalysis_1.orderDagEntriesCausally)(allEntryMetadata.filter((em) => em.entryType === "doc_create" ||
             em.entryType === "doc_change" ||
             em.entryType === "doc_delete" ||
-            em.entryType === "doc_undelete")
-            .sort((a, b) => a.createdAt - b.createdAt);
+            em.entryType === "doc_undelete"));
         // Cursor paging is offset-based because the result is a bounded timeline view
         // over the doc's sorted change metadata, not a resumable store scan cursor.
         const slice = relevantEntries.slice(offset, offset + limit);
-        const entries = slice.map((entry) => ({
+        const entries = slice.map((entry, sliceIndex) => ({
             entryId: entry.id,
             entryType: entry.entryType,
             changeCreatedAt: entry.createdAt,
             changeCreatedByPublicKey: entry.createdByPublicKey,
             dependencyIds: [...entry.dependencyIds],
-            isDeleted: this.computeIsDeletedFromMetadata(relevantEntries.filter((candidate) => candidate.createdAt < entry.createdAt ||
-                (candidate.createdAt === entry.createdAt && candidate.id <= entry.id))),
+            // Deletion state as of this timeline position = derived from the exact
+            // causal prefix of entries up to and including this one. Rebuilding the
+            // prefix via a (createdAt, id) comparison could disagree with the causal
+            // order for same-millisecond entries.
+            isDeleted: this.computeIsDeletedFromMetadata(relevantEntries.slice(0, offset + sliceIndex + 1)),
         }));
         const nextOffset = offset + entries.length;
         const hasMore = nextOffset < relevantEntries.length;
@@ -4596,7 +4915,7 @@ class BaseMindooDB {
         return {
             changeSeqAsOf: latestCursor?.changeSeq ?? 0,
             storeReceiptOrderAsOf: await this.getStoreMaxReceiptOrder(),
-            takenAt: Date.now(),
+            takenAt: (0, timeSource_1.semanticNow)(),
         };
     }
     /**
@@ -5244,20 +5563,27 @@ class BaseMindooDB {
         const filteredPaths = normalizedPaths.filter((path) => touchedKeySet.has(this.extractLeafPathSegment(path)));
         return filteredPaths.length > 0 ? filteredPaths : normalizedPaths;
     }
-    async getAllDocumentIds() {
-        // Return all non-deleted document IDs from index
+    async getAllDocumentIds(options) {
+        // Return all non-deleted document IDs from index, optionally narrowed to a
+        // single id prefix (matched on the `<prefix>_` boundary; see matchesDocIdPrefix).
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
         const docIds = [];
         for (const entry of this.index) {
             if (entry.accessState === "visible" && !entry.isDeleted) {
+                if (idPrefix && !(0, idGeneration_1.matchesDocIdPrefix)(entry.docId, idPrefix))
+                    continue;
                 docIds.push(entry.docId);
             }
         }
         return docIds;
     }
-    async getDeletedDocumentIds() {
+    async getDeletedDocumentIds(options) {
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
         const docIds = [];
         for (const entry of this.index) {
             if (entry.accessState === "visible" && entry.isDeleted) {
+                if (idPrefix && !(0, idGeneration_1.matchesDocIdPrefix)(entry.docId, idPrefix))
+                    continue;
                 docIds.push(entry.docId);
             }
         }
@@ -5297,24 +5623,36 @@ class BaseMindooDB {
         }
         const docIds = [];
         for (const [docId, lifecycleEntries] of lifecycleEntriesByDocId.entries()) {
-            // Sort deterministically so same-timestamp lifecycle entries resolve the
-            // same way on every replica.
-            const ordered = lifecycleEntries.sort((a, b) => a.createdAt !== b.createdAt ? a.createdAt - b.createdAt : a.id.localeCompare(b.id));
             // A document cannot exist before its create entry, even if malformed or
             // partial metadata somehow contains later lifecycle entries.
-            if (!ordered.some((entry) => entry.entryType === "doc_create" && entry.createdAt <= timestamp)) {
+            if (!lifecycleEntries.some((entry) => entry.entryType === "doc_create" && entry.createdAt <= timestamp)) {
                 continue;
             }
-            const createEntry = ordered.find((entry) => entry.entryType === "doc_create");
+            const createEntry = lifecycleEntries.find((entry) => entry.entryType === "doc_create");
             if (createEntry && !(await this.tenant.hasDecryptionKey(createEntry.decryptionKeyId))) {
                 continue;
             }
-            // Delete and undelete are terminal lifecycle markers. The latest one at
-            // the timestamp decides whether the document existed then.
-            const latestTerminal = ordered
-                .filter((entry) => entry.entryType === "doc_delete" || entry.entryType === "doc_undelete")
-                .at(-1);
-            if (!latestTerminal || latestTerminal.entryType === "doc_undelete") {
+            const terminalEntries = lifecycleEntries.filter((entry) => (entry.entryType === "doc_delete" || entry.entryType === "doc_undelete")
+                && entry.createdAt <= timestamp);
+            if (terminalEntries.length === 0) {
+                // Fast path (the vast majority of documents): no delete/undelete ever
+                // happened up to `timestamp`, so the document is alive and we never
+                // need its change metadata.
+                docIds.push(docId);
+                continue;
+            }
+            // Causal path: the doc has terminal lifecycle entries, so alive/deleted
+            // must be derived exactly like getDocumentAtTimestamp derives it —
+            // grounded slice, dependency-based heads, causal supersession — instead
+            // of a purely chronological "latest terminal wins". A same-instant
+            // delete→undelete pair or cross-replica clock skew would otherwise be
+            // decided by id tie-break, which can disagree with causality and with
+            // getDocumentAtTimestamp for the same instant. The extra per-doc
+            // metadata scan is bounded by the number of ever-deleted documents.
+            const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+            const relevantEntries = (0, DocumentDagAnalysis_1.pruneToGroundedEntries)(allEntryMetadata.filter((entry) => entry.createdAt <= timestamp
+                && (this.isDocumentReplayEntry(entry) || entry.entryType === "doc_snapshot")));
+            if (!this.computeIsDeletedFromMetadata(relevantEntries)) {
                 docIds.push(docId);
             }
         }
@@ -5427,17 +5765,31 @@ class BaseMindooDB {
         // Delete and undelete are encoded as normal Automerge changes so the DAG
         // keeps causal ancestry, while the StoreEntry type carries lifecycle intent.
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const docId = internalDoc.id;
         // Lifecycle changes are intentionally non-destructive: they only bump
         // `_lastModified` and repair missing legacy attachment arrays. The document
         // body remains available for history and future undeletion.
-        const newDoc = this.runChangeWithOutdatedDocRecovery(internalDoc, (doc) => automerge_adapter_1.Automerge.change(doc, { time: now }, (mutableDoc) => {
+        //
+        // Same-millisecond hazard: the change writes ONLY `_lastModified`, and
+        // Automerge dedupes a same-value assignment into an EMPTY change (heads
+        // unchanged, getLastLocalChange returns the PREVIOUS change). A delete in
+        // the same millisecond as the last change would then reuse the previous
+        // change's hash, collide with its entry id, and the doc_delete would
+        // silently never be persisted — the document resurrects on sync/reload.
+        // If the mutation dedupes to nothing, fall back to an explicit
+        // `emptyChange`, which always advances the heads with a fresh hash while
+        // keeping causal ancestry intact.
+        const headsBefore = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
+        let newDoc = this.runChangeWithOutdatedDocRecovery(internalDoc, (doc) => automerge_adapter_1.Automerge.change(doc, { time: now }, (mutableDoc) => {
             if (!Array.isArray(mutableDoc._attachments)) {
                 mutableDoc._attachments = [];
             }
             mutableDoc._lastModified = now;
         }));
+        if (automerge_adapter_1.Automerge.getHeads(newDoc).every((head) => headsBefore.includes(head))) {
+            newDoc = automerge_adapter_1.Automerge.emptyChange(newDoc, { time: now });
+        }
         const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             throw new Error("Failed to get change bytes from Automerge document");
@@ -5542,15 +5894,18 @@ class BaseMindooDB {
         // The doc is now witness-era (we just wrote a versioned entry), so it is
         // witnessed iff it is no longer awaiting a receipt.
         internalDoc.witnessed = (0, storeEntryTime_1.isVersioned)(entryMetadata) && !internalDoc.awaitingWitness;
+        // Force the changeSeq bump: a lifecycle entry was just written, so the
+        // changefeed must re-emit the doc even if all tracked index fields happen
+        // to match the previous state in the same millisecond.
         if (options.markDirty) {
             await this.storeCachedDocument(internalDoc);
-            this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
+            this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed, true);
             this.markDocDirty(docId);
         }
         else {
             this.docCache.delete(docId);
             this.dirtyDocIds.delete(docId);
-            this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
+            this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed, true);
         }
     }
     /**
@@ -5614,7 +5969,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         this.validateTextPatch(patch);
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const headsBeforeChange = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
         const baseHeads = patch.baseHeads;
         const applyEdits = (automergeDoc) => {
@@ -5678,7 +6033,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         this.validateRichTextPatch(patch);
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const headsBeforeChange = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
         const spansSequence = patch.spansSequence ?? (patch.spans ? [patch.spans] : []);
         const applySpans = (automergeDoc) => {
@@ -5753,7 +6108,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         this.validateRichTextStepPatch(patch);
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const headsBeforeChange = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
         const applySteps = (automergeDoc) => {
             this.ensureRichTextPath(automergeDoc, patch.path);
@@ -5875,7 +6230,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         await this.hydrateAutomergeHashMappingsFromStore(docId);
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const headsBeforeChange = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
         if (patch.baseHeads?.length) {
             const baseHeadsKey = [...patch.baseHeads].sort().join("|");
@@ -5982,7 +6337,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         this.validateJsonPatch(patch);
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         const headsBeforeChange = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
         const applyPatch = (automergeDoc) => {
             this.applyJsonPatchOperations(automergeDoc, patch);
@@ -6055,7 +6410,7 @@ class BaseMindooDB {
             throw new errors_1.DocumentDeletedError(docId);
         }
         // Apply the change function
-        const now = Date.now();
+        const now = (0, timeSource_1.semanticNow)();
         this.logger.debug(`Applying change function to document ${docId}`);
         if (this.isDebugEnabled())
             this.logger.debug(`Document state before change: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(internalDoc.doc))}`);
@@ -6960,7 +7315,10 @@ class BaseMindooDB {
             || (internalDoc.witnessed ?? false);
         internalDoc.witnessed = versioned && !internalDoc.awaitingWitness;
         await this.storeCachedDocument(internalDoc);
-        this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
+        // Force the changeSeq bump: new change entries were just written, so the
+        // changefeed must re-emit the doc even when `lastModified` collides with
+        // the previous change in the same millisecond.
+        this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed, true);
         this.markDocDirty(docId);
         this.logger.info(`Document ${docId} ${successMessage} successfully`);
         const createdByPublicKey = useCustomKey
@@ -7078,7 +7436,7 @@ class BaseMindooDB {
                 contentHash,
                 docId,
                 dependencyIds: headEntryIds,
-                createdAt: Date.now(),
+                createdAt: (0, timeSource_1.semanticNow)(),
                 createdByPublicKey: options.createdByPublicKey,
                 decryptionKeyId: internalDoc.decryptionKeyId,
                 snapshotHeadHashes: headHashes,
@@ -7101,7 +7459,7 @@ class BaseMindooDB {
             this.logger.warn(`Failed to create snapshot for document ${docId}, continuing without snapshot`, error);
         }
     }
-    async prefetchIterationWindow(indexSnapshot, startIndex) {
+    async prefetchIterationWindow(indexSnapshot, startIndex, idPrefix) {
         if (this.iteratePrefetchWindowDocs <= 0) {
             return 0;
         }
@@ -7117,6 +7475,10 @@ class BaseMindooDB {
             if (entry.accessState === "inaccessible" || entry.isDeleted) {
                 continue;
             }
+            // Don't prefetch bodies the prefix-filtered iteration will never yield.
+            if (idPrefix && !(0, idGeneration_1.matchesDocIdPrefix)(entry.docId, idPrefix)) {
+                continue;
+            }
             const docId = entry.docId;
             if (seen.has(docId) || this.docCache.has(docId)) {
                 continue;
@@ -7169,7 +7531,7 @@ class BaseMindooDB {
         return startIndex;
     }
     resolveDagTimestamp(timestamp) {
-        return timestamp === "now" ? Date.now() : timestamp;
+        return timestamp === "now" ? (0, timeSource_1.semanticNow)() : timestamp;
     }
     previewChangeValue(value) {
         if (value === undefined) {
@@ -7435,19 +7797,22 @@ class BaseMindooDB {
             witnessed: witnessState.witnessed,
         };
     }
-    async *iterateChangeMetadataSince(cursor) {
+    async *iterateChangeMetadataSince(cursor, options) {
         const startedAt = Date.now();
         const actualCursor = cursor ?? {
             changeSeq: 0,
             lastModified: 0,
             docId: "",
         };
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
         const indexSnapshot = [...this.index];
         const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
         let yieldedDocuments = 0;
         try {
             for (let i = startIndex; i < indexSnapshot.length; i++) {
                 const entry = indexSnapshot[i];
+                if (idPrefix && !(0, idGeneration_1.matchesDocIdPrefix)(entry.docId, idPrefix))
+                    continue;
                 const currentCursor = {
                     changeSeq: entry.changeSeq,
                     lastModified: entry.lastModified,
@@ -7489,20 +7854,32 @@ class BaseMindooDB {
      * them. O(log n) binary search on the internal `(changeSeq, docId)` index —
      * used for progress reporting around {@link iterateChangesSince}.
      */
-    countChangesSince(cursor) {
+    countChangesSince(cursor, options) {
         const actualCursor = cursor ?? {
             changeSeq: 0,
             lastModified: 0,
             docId: "",
         };
         const startIndex = this.getStartIndexForCursor(this.index, actualCursor);
-        return this.index.length - startIndex;
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
+        if (!idPrefix) {
+            return this.index.length - startIndex;
+        }
+        // Prefix-filtered count must walk the tail: the index is ordered by
+        // (changeSeq, docId), not by docId, so matches are not contiguous.
+        let count = 0;
+        for (let i = startIndex; i < this.index.length; i++) {
+            if ((0, idGeneration_1.matchesDocIdPrefix)(this.index[i].docId, idPrefix))
+                count++;
+        }
+        return count;
     }
-    async *iterateChangesSince(cursor) {
+    async *iterateChangesSince(cursor, options) {
         const startedAt = Date.now();
         // Default to initial cursor if null is provided.
         // Prefer deterministic sequence-based cursoring; keep legacy fallback compatibility.
         const actualCursor = cursor ?? { changeSeq: 0, lastModified: 0, docId: "" };
+        const idPrefix = this.normalizeIdPrefixFilter(options?.idPrefix);
         this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
         // Use a stable snapshot of the index for this generator run so concurrent
         // updates do not reorder/skip entries while iterating.
@@ -7510,13 +7887,17 @@ class BaseMindooDB {
         // Find starting position using binary search
         // We want to find the first entry that is greater than the cursor.
         const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
-        let prefetchedDocuments = await this.prefetchIterationWindow(indexSnapshot, startIndex);
+        let prefetchedDocuments = await this.prefetchIterationWindow(indexSnapshot, startIndex, idPrefix);
         let yieldedDocuments = 0;
         let loadedDocuments = 0;
         try {
             // Iterate through the stable snapshot and yield documents one at a time.
             for (let i = startIndex; i < indexSnapshot.length; i++) {
                 const entry = indexSnapshot[i];
+                // Skip entries outside the requested id prefix before any body load,
+                // so prefix-filtered iteration never decrypts/materializes unrelated docs.
+                if (idPrefix && !(0, idGeneration_1.matchesDocIdPrefix)(entry.docId, idPrefix))
+                    continue;
                 if (entry.accessState !== "visible") {
                     // The document was readable before but the current KeyBag can no
                     // longer decrypt it (e.g. a revoked key, §13). Emit a tombstone so
@@ -7531,7 +7912,7 @@ class BaseMindooDB {
                     };
                     yieldedDocuments++;
                     yield { doc: this.buildInaccessibleDoc(entry), cursor: inaccessibleCursor };
-                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
+                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1, idPrefix);
                     continue;
                 }
                 if (entry.isDeleted) {
@@ -7548,7 +7929,7 @@ class BaseMindooDB {
                     };
                     yieldedDocuments++;
                     yield { doc: this.buildDeletedDoc(entry), cursor: deletedCursor };
-                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
+                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1, idPrefix);
                     continue;
                 }
                 try {
@@ -7578,7 +7959,7 @@ class BaseMindooDB {
                     // Deleted documents are included so external indexes can handle deletions
                     yieldedDocuments++;
                     yield { doc, cursor: currentCursor };
-                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
+                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1, idPrefix);
                 }
                 catch (error) {
                     this.logger.error(`Error processing document ${entry.docId}:`, error);
@@ -8223,7 +8604,11 @@ class BaseMindooDB {
                 op: input.op,
                 dbid,
                 signingKey: input.signerKey,
-                trustedTime: Date.now(),
+                // Head selector, NOT Date.now(): un-witnessed directory revisions get a
+                // provisional trusted time stamped at fold time, which can land AFTER a
+                // wall clock captured here; an at-`now` node would then miss just-written
+                // policy docs. Same rationale as the provisional-entry judgment below.
+                trustedTime: Number.MAX_SAFE_INTEGER,
                 isAuthor: input.isAuthor,
                 beforeDoc,
                 afterDoc,
@@ -8399,7 +8784,7 @@ class BaseMindooDB {
         // A verify-only provider (no signer) routes receipts to the right scheme.
         const subtle = this.getSubtle();
         const receiptVerifier = new Ed25519WitnessProvider_1.Ed25519WitnessProvider({ subtle }).createVerifier();
-        const nowMs = Date.now();
+        const nowMs = (0, timeSource_1.semanticNow)();
         // Lifecycle entries only, folded in deterministic causal order. Un-witnessed
         // entries (versioned or legacy) order by their author time `createdAt`, NOT
         // the provisional `now`: this fold computes per-step before/after states for
@@ -8435,7 +8820,7 @@ class BaseMindooDB {
                 // Audit record: the entry's own (stable) time — receipt time if
                 // witnessed, else author time. Not the provisional head `now`.
                 trustedTime: meta.receivedAt ?? meta.createdAt,
-                recordedAt: Date.now(),
+                recordedAt: (0, timeSource_1.semanticNow)(),
             });
         };
         for (const meta of dataEntries) {
@@ -8563,7 +8948,7 @@ class BaseMindooDB {
         const payload = currentDoc;
         const isDeleted = this.computeIsDeletedFromMetadata(acceptedMeta);
         const lastAccepted = acceptedMeta[acceptedMeta.length - 1];
-        const lastModified = payload._lastModified || (lastAccepted ? lastAccepted.createdAt : Date.now());
+        const lastModified = payload._lastModified || (lastAccepted ? lastAccepted.createdAt : (0, timeSource_1.semanticNow)());
         const witnessState = (0, storeEntryTime_1.metadataWitnessState)(acceptedMeta);
         return {
             doc: {
@@ -8934,7 +9319,7 @@ class BaseMindooDB {
         // Get lastModified from payload, or use the timestamp of the last entry
         const lastEntry = entries.length > 0 ? entries[entries.length - 1] : null;
         const lastModified = payload._lastModified ||
-            (lastEntry ? lastEntry.createdAt : Date.now());
+            (lastEntry ? lastEntry.createdAt : (0, timeSource_1.semanticNow)());
         // Get createdAt from the first entry
         const firstEntry = allEntryMetadata.length > 0 ? allEntryMetadata[0] : null;
         const createdAt = firstEntry ? firstEntry.createdAt : lastModified;
@@ -9327,6 +9712,9 @@ class BaseMindooDB {
         const chunks = [];
         let prevChunkId = null;
         let lastChunkId = "";
+        // Guards against case-folded chunk-id collisions on case-insensitive
+        // filesystems; scoped to this single attachment write.
+        const usedCaseFoldedChunkIds = new Set();
         for (let offset = 0; offset < fileData.length; offset += this.chunkSizeBytes) {
             const chunkData = fileData.slice(offset, Math.min(offset + this.chunkSizeBytes, fileData.length));
             // Encrypt chunk
@@ -9334,7 +9722,7 @@ class BaseMindooDB {
             // Compute content hash
             const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedData, this.getSubtle());
             // Generate chunk ID
-            const chunkId = (0, idGeneration_1.generateAttachmentChunkId)(docId, attachmentId);
+            const chunkId = (0, idGeneration_1.generateUniqueAttachmentChunkId)(docId, attachmentId, usedCaseFoldedChunkIds);
             lastChunkId = chunkId;
             // Sign the encrypted chunk
             const signature = await this.tenant.signPayload(encryptedData);
@@ -9408,7 +9796,7 @@ class BaseMindooDB {
             dependencyIds: [],
             createdAt,
             attachmentId,
-            uploadStartedAt: Date.now(),
+            uploadStartedAt: (0, timeSource_1.semanticNow)(),
             createdByPublicKey,
             decryptionKeyId,
             signature,
@@ -9454,6 +9842,9 @@ class BaseMindooDB {
         const pendingEntries = [];
         let pendingEncryptedBytes = 0;
         let ledgerCreated = false;
+        // Guards against case-folded chunk-id collisions on case-insensitive
+        // filesystems; scoped to this single attachment write.
+        const usedCaseFoldedChunkIds = new Set();
         const flushPendingEntries = async (force = false) => {
             if (pendingEntries.length === 0) {
                 return;
@@ -9472,7 +9863,7 @@ class BaseMindooDB {
             // Compute content hash
             const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedData, this.getSubtle());
             // Generate chunk ID
-            const chunkId = (0, idGeneration_1.generateAttachmentChunkId)(docId, attachmentId);
+            const chunkId = (0, idGeneration_1.generateUniqueAttachmentChunkId)(docId, attachmentId, usedCaseFoldedChunkIds);
             // Sign the encrypted chunk
             const signature = await this.tenant.signPayload(encryptedData);
             // Create chunk entry
@@ -9560,6 +9951,9 @@ class BaseMindooDB {
         const chunks = [];
         let prevChunkId = prevLastChunkId;
         let lastChunkId = prevLastChunkId;
+        // Guards against case-folded chunk-id collisions on case-insensitive
+        // filesystems; scoped to this single append.
+        const usedCaseFoldedChunkIds = new Set();
         for (let offset = 0; offset < data.length; offset += this.chunkSizeBytes) {
             const chunkData = data.slice(offset, Math.min(offset + this.chunkSizeBytes, data.length));
             // Encrypt chunk
@@ -9567,7 +9961,7 @@ class BaseMindooDB {
             // Compute content hash
             const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedData, this.getSubtle());
             // Generate chunk ID
-            const chunkId = (0, idGeneration_1.generateAttachmentChunkId)(docId, attachmentId);
+            const chunkId = (0, idGeneration_1.generateUniqueAttachmentChunkId)(docId, attachmentId, usedCaseFoldedChunkIds);
             lastChunkId = chunkId;
             // Sign the encrypted chunk
             const signature = await this.tenant.signPayload(encryptedData);
@@ -9785,12 +10179,18 @@ class BaseMindooDB {
         try {
             const syncResult = await this.syncEntriesFromStore(localStore, remoteStore, options);
             this.logger.debug(`Transferred ${syncResult.transferred} entries to remote store`);
+            const rejectedEntries = syncResult.rejected ?? [];
+            if (rejectedEntries.length > 0) {
+                this.logger.warn(`Remote rejected ${rejectedEntries.length} entries during push (skipped, push continued): ` +
+                    rejectedEntries.map((r) => r.reason).join("; "));
+            }
             if (syncResult.cancelled) {
                 this.logger.info(`Push cancelled after transferring ${syncResult.transferred} entries`);
                 return {
                     transferredEntries: syncResult.transferred,
                     scannedEntries: syncResult.scanned,
                     cancelled: true,
+                    ...(rejectedEntries.length > 0 ? { rejectedEntries } : {}),
                 };
             }
             if (syncResult.transferred === 0) {
@@ -9809,6 +10209,7 @@ class BaseMindooDB {
                 transferredEntries: syncResult.transferred,
                 scannedEntries: syncResult.scanned,
                 cancelled: false,
+                ...(rejectedEntries.length > 0 ? { rejectedEntries } : {}),
             };
         }
         finally {
@@ -9817,6 +10218,32 @@ class BaseMindooDB {
     }
 }
 exports.BaseMindooDB = BaseMindooDB;
+/**
+ * Determine how many entry IDs to fetch per `getEntries` call during sync.
+ *
+ * This is intentionally separate from the metadata scan `pageSize` so that
+ * scanning can page through large ID lists quickly while the heavier
+ * payload downloads use a smaller batch to keep progress responsive and
+ * cancellation timely.
+ *
+ * Priority: explicit option > attachment default (100) > docs default (250,
+ * capped by pageSize).
+ *
+ * The docs default is intentionally smaller than the metadata scan pageSize
+ * (1000): a full scan page transferred as one `putEntries` call serializes
+ * to a multi-megabyte JSON body whose upload cannot report progress, risks
+ * the HTTP timeout on slow uplinks (each retry restarts the whole body),
+ * and can exceed the remote JSON body limit. Smaller batches keep progress
+ * events frequent and make each POST cheap to retry.
+ */
+BaseMindooDB.DEFAULT_DOCS_TRANSFER_BATCH_SIZE = 250;
+/**
+ * Default window of transfer batches running in parallel (sync-v5,
+ * phase 4). Kept small: each in-flight batch holds one decrypted page of
+ * entries in memory, and HTTP/2 multiplexes the requests over a single
+ * connection anyway. Overridable via `SyncOptions.maxConcurrentBatches`.
+ */
+BaseMindooDB.DEFAULT_MAX_CONCURRENT_TRANSFER_BATCHES = 3;
 /** The "access control not enforced" allow decision returned by the
  * non-throwing `canCreate`/`canChange`/`canDelete` prediction helpers. */
 BaseMindooDB.ACL_NOT_ENFORCED_DECISION = {