@vrtmrz/livesync-commonlib 0.1.43 → 0.2.0

package/dist/pouchdb/StreamingFetch.js

@@ -3,7 +3,61 @@ import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
 import { VERSIONING_DOCID } from "../common/types.js";
 import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "./remoteFeatureCompatibility.js";
 import { assertMatchingPathIdentityFeature } from "./pathIdentityCompatibility.js";
+function isObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function revisionRefKey(ref) {
+  return `${ref.id}\0${ref.rev}`;
+}
+function parseRevisionIdentifier(rev) {
+  const separator = rev.indexOf("-");
+  if (separator <= 0) return void 0;
+  const generationText = rev.slice(0, separator);
+  const hash = rev.slice(separator + 1);
+  if (!/^[1-9]\d*$/.test(generationText) || hash.length === 0 || hash.trim() !== hash) return void 0;
+  const generation = Number(generationText);
+  if (!Number.isSafeInteger(generation)) return void 0;
+  return { generation, hash };
+}
+function ensureRevisionIdentity(value, ref) {
+  if (!isObject(value) || value._id !== ref.id || value._rev !== ref.rev) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      `Fast Fetch received an invalid body for revision ${ref.id}@${ref.rev}.`,
+      false
+    );
+  }
+  if (value._deleted !== void 0 && value._deleted !== true) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      `Fast Fetch received an invalid deletion marker for revision ${ref.id}@${ref.rev}.`,
+      false
+    );
+  }
+  return value;
+}
+function hasValidRevisionAncestry(value, revision) {
+  const revisions = value._revisions;
+  return isObject(revisions) && Number.isSafeInteger(revisions.start) && revisions.start === revision.generation && Array.isArray(revisions.ids) && revisions.ids.length > 0 && revisions.ids.length <= revision.generation && revisions.ids[0] === revision.hash && revisions.ids.every((id) => typeof id === "string" && id.length > 0);
+}
+function ensureRevisionDocument(value, ref) {
+  const document = ensureRevisionIdentity(value, ref);
+  const revision = parseRevisionIdentifier(ref.rev);
+  if (revision === void 0 || !hasValidRevisionAncestry(document, revision)) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      `Fast Fetch received revision ${ref.id}@${ref.rev} without valid revision ancestry.`,
+      false
+    );
+  }
+  return document;
+}
+class BulkGetResponseTooLargeError extends Error {
+  name = "BulkGetResponseTooLargeError";
+}
 const FAST_FETCH_CHANGES_PAGE_LIMIT = 1e4;
+const FAST_FETCH_REVISION_GROUP_LIMIT = 100;
+const FAST_FETCH_REVISION_GROUP_BYTE_LIMIT = 512 * 1024;
 const FAST_FETCH_CHANGES_PAGE_TIMEOUT_MS = 1e3;
 class StreamingFetchFailure extends Error {
   constructor(stage, message, retryable, options) {
@@ -76,6 +130,46 @@ async function readResponseText(response, operation) {
     throw transportFailure(operation, error);
   }
 }
+async function readResponseTextBounded(response, operation, maxBytes) {
+  if (!response.body) {
+    const text = await readResponseText(response, operation);
+    const byteLength2 = new TextEncoder().encode(text).byteLength;
+    if (byteLength2 > maxBytes) throw new BulkGetResponseTooLargeError();
+    return { text, byteLength: byteLength2 };
+  }
+  const reader = response.body.getReader();
+  const chunks = [];
+  let byteLength = 0;
+  try {
+    while (true) {
+      let result;
+      try {
+        result = await reader.read();
+      } catch (error) {
+        throw transportFailure(operation, error);
+      }
+      if (result.done) break;
+      byteLength += result.value.byteLength;
+      if (byteLength > maxBytes) {
+        try {
+          await reader.cancel();
+        } catch {
+        }
+        throw new BulkGetResponseTooLargeError();
+      }
+      chunks.push(result.value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const bytes = new Uint8Array(byteLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { text: new TextDecoder().decode(bytes), byteLength };
+}
 async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, customHeaders, settings = {}) {
   const headers = new Headers(customHeaders);
   headers.set("Accept", "application/json");
@@ -114,92 +208,125 @@ async function saveCheckpoint(onCheckpoint, sequence) {
     );
   }
 }
-function generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint) {
+function generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint, onStoredRevisions) {
   let batchBuffer = [];
+  let batchStoredRevisions = [];
+  let batchRevisionKeys = /* @__PURE__ */ new Set();
   let currentBatchSizeBytes = 0;
-  let batchLastSequence;
+  let batchCheckpointSequence;
   const BATCH_ITEM_LIMIT = 100;
   const BATCH_SIZE_LIMIT = 2 * 1024 * 1024;
   const flush = async () => {
-    if (batchBuffer.length === 0) return;
     const documents = batchBuffer;
-    const checkpoint = batchLastSequence;
-    let results;
-    try {
-      results = await downloadToDB.bulkDocs(documents, { new_edits: false });
-    } catch (error) {
-      throw new StreamingFetchFailure(
-        "storage",
-        `Fast Fetch could not write a batch to the local database: ${errorMessage(error)}`,
-        false,
-        { cause: error }
-      );
-    }
-    const failedResult = results.find(
-      (result) => "error" in result && Boolean(result.error)
-    );
-    if (failedResult) {
-      const detail = failedResult.message || failedResult.name || "the local database rejected a document";
-      throw new StreamingFetchFailure(
-        "storage",
-        `Fast Fetch could not write a batch to the local database: ${detail}`,
-        false,
-        { cause: failedResult }
+    const storedRevisions = batchStoredRevisions;
+    const checkpoint = batchCheckpointSequence;
+    if (documents.length > 0) {
+      let results;
+      try {
+        results = await downloadToDB.bulkDocs(documents, { new_edits: false });
+      } catch (error) {
+        throw new StreamingFetchFailure(
+          "storage",
+          `Fast Fetch could not write a batch to the local database: ${errorMessage(error)}`,
+          false,
+          { cause: error }
+        );
+      }
+      const failedResult = results.find(
+        (result) => "error" in result && Boolean(result.error)
       );
+      if (failedResult) {
+        const detail = failedResult.message || failedResult.name || "the local database rejected a document";
+        throw new StreamingFetchFailure(
+          "storage",
+          `Fast Fetch could not write a batch to the local database: ${detail}`,
+          false,
+          { cause: failedResult }
+        );
+      }
+      try {
+        await onStoredRevisions?.(storedRevisions);
+      } catch (error) {
+        throw new StreamingFetchFailure(
+          "storage",
+          `Fast Fetch could not record stored revision identities: ${errorMessage(error)}`,
+          false,
+          { cause: error }
+        );
+      }
     }
     if (checkpoint !== void 0) {
       await saveCheckpoint(onCheckpoint, checkpoint);
     }
     batchBuffer = [];
+    batchStoredRevisions = [];
+    batchRevisionKeys = /* @__PURE__ */ new Set();
     currentBatchSizeBytes = 0;
-    batchLastSequence = void 0;
+    batchCheckpointSequence = void 0;
   };
-  return {
-    async write(doc, sequence) {
-      let decryptedDoc;
-      if (doc._deleted) {
-        decryptedDoc = doc;
-      } else {
-        try {
-          decryptedDoc = await decryptFunction(doc);
-        } catch (error) {
-          throw new StreamingFetchFailure(
-            "decryption",
-            `Fast Fetch could not decrypt a document: ${errorMessage(error)}`,
-            false,
-            { cause: error }
-          );
-        }
-      }
-      let serialisedDoc;
+  const prepare = async (doc) => {
+    let decryptedDoc;
+    if (doc._deleted) {
+      decryptedDoc = doc;
+    } else {
       try {
-        const serialised = JSON.stringify(decryptedDoc);
-        if (serialised === void 0) throw new Error("the decrypted value is not a document");
-        serialisedDoc = serialised;
+        decryptedDoc = await decryptFunction(doc);
       } catch (error) {
         throw new StreamingFetchFailure(
           "decryption",
-          `Fast Fetch produced an invalid decrypted document: ${errorMessage(error)}`,
+          `Fast Fetch could not decrypt a document: ${errorMessage(error)}`,
           false,
           { cause: error }
         );
       }
-      batchBuffer.push(decryptedDoc);
-      currentBatchSizeBytes += serialisedDoc.length;
-      batchLastSequence = sequence;
-      if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
-        await flush();
-      }
-    },
-    async flushThrough(sequence) {
+    }
+    let serialisedDoc;
+    try {
+      const serialised = JSON.stringify(decryptedDoc);
+      if (serialised === void 0) throw new Error("the decrypted value is not a document");
+      serialisedDoc = serialised;
+    } catch (error) {
+      throw new StreamingFetchFailure(
+        "decryption",
+        `Fast Fetch produced an invalid decrypted document: ${errorMessage(error)}`,
+        false,
+        { cause: error }
+      );
+    }
+    return { document: decryptedDoc, serialisedSize: serialisedDoc.length };
+  };
+  const writePrepared = async (prepared, revision) => {
+    const key = revisionRefKey(revision);
+    if (batchRevisionKeys.has(key)) return;
+    batchBuffer.push(prepared.document);
+    batchStoredRevisions.push({ id: revision.id, rev: revision.rev });
+    batchRevisionKeys.add(key);
+    currentBatchSizeBytes += prepared.serialisedSize;
+    if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
       await flush();
-      await saveCheckpoint(onCheckpoint, sequence);
+    }
+  };
+  const write = async (doc, revision) => {
+    await writePrepared(await prepare(doc), revision);
+  };
+  return {
+    prepare,
+    writePrepared,
+    write,
+    async advanceCheckpoint(sequence) {
+      batchCheckpointSequence = sequence;
+      if (batchBuffer.length === 0) await flush();
+    },
+    hasPendingRevision(revision) {
+      return batchRevisionKeys.has(revisionRefKey(revision));
     },
     flush,
     abort() {
       batchBuffer = [];
+      batchStoredRevisions = [];
+      batchRevisionKeys = /* @__PURE__ */ new Set();
       currentBatchSizeBytes = 0;
-      batchLastSequence = void 0;
+      batchCheckpointSequence = void 0;
     }
   };
 }
@@ -260,7 +387,7 @@ function parseChangesFeedLine(line) {
   }
   return { type: "change", change };
 }
-async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint, customHeaders) {
+async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint, customHeaders, onStoredRevisions) {
   let totalFetched = 0;
   let totalValidFetched = 0;
   let totalBytes = 0;
@@ -275,6 +402,8 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
   const fetchHeaders = new Headers(customHeaders);
   fetchHeaders.set("Accept", "application/json");
   fetchHeaders.set("Authorization", authHeader);
+  const bulkGetHeaders = new Headers(fetchHeaders);
+  bulkGetHeaders.set("Content-Type", "application/json");
   const targetURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
     ...changesBaseParams,
     feed: "normal",
@@ -296,7 +425,7 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
       false
     );
   }
-  const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint);
+  const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint, onStoredRevisions);
   let docsToFetch = 0;
   let lastProgress = 0;
   let lastReportTime = Date.now();
@@ -312,6 +441,282 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
       totalBytes
     });
   };
+  const rowProgress = [];
+  let nextCheckpointRow = 0;
+  const pendingRevisions = /* @__PURE__ */ new Map();
+  let pendingRevisionBytes = 0;
+  const pendingRemoteRevisions = /* @__PURE__ */ new Map();
+  let pendingRemoteRevisionBytes = 0;
+  const releaseCompletedRows = async () => {
+    while (nextCheckpointRow < rowProgress.length && rowProgress[nextCheckpointRow].complete) {
+      await batchWriter.advanceCheckpoint(rowProgress[nextCheckpointRow].sequence);
+      nextCheckpointRow++;
+    }
+  };
+  const settleRevision = (revision) => {
+    for (const row of revision.rows) {
+      row.pendingRevisions--;
+      if (row.pendingRevisions === 0) row.complete = true;
+    }
+  };
+  const markRevisionDurable = async (revision, doc) => {
+    await batchWriter.write(doc, revision.ref);
+    settleRevision(revision);
+  };
+  const markPreparedRevisionDurable = async (revision, prepared) => {
+    await batchWriter.writePrepared(prepared, revision.ref);
+    settleRevision(revision);
+  };
+  const makeBulkGetURL = () => {
+    const url = new URL(`${remoteDbUrl}/_bulk_get`);
+    url.searchParams.set("revs", "true");
+    return url;
+  };
+  const parseBulkGetDocuments = (source, refs) => {
+    let parsed;
+    try {
+      parsed = JSON.parse(source);
+    } catch (cause) {
+      throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid _bulk_get response.", false, {
+        cause
+      });
+    }
+    if (!isObject(parsed) || !Array.isArray(parsed.results)) {
+      throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid _bulk_get response.", false);
+    }
+    const expected = new Map(refs.map((revision) => [revisionRefKey(revision.ref), revision.ref]));
+    const documents = /* @__PURE__ */ new Map();
+    for (const result of parsed.results) {
+      if (!isObject(result) || typeof result.id !== "string" || !Array.isArray(result.docs)) {
+        throw new StreamingFetchFailure(
+          "protocol",
+          "Fast Fetch received a malformed _bulk_get result.",
+          false
+        );
+      }
+      for (const item of result.docs) {
+        if (!isObject(item)) {
+          throw new StreamingFetchFailure(
+            "protocol",
+            "Fast Fetch received a malformed _bulk_get document result.",
+            false
+          );
+        }
+        if ("ok" in item) {
+          const body = item.ok;
+          if (!isObject(body) || typeof body._rev !== "string") {
+            throw new StreamingFetchFailure(
+              "protocol",
+              "Fast Fetch received a malformed _bulk_get document.",
+              false
+            );
+          }
+          const ref = { id: result.id, rev: body._rev };
+          const key = revisionRefKey(ref);
+          if (!expected.has(key) || documents.has(key)) {
+            throw new StreamingFetchFailure(
+              "protocol",
+              "Fast Fetch received an unexpected or duplicate _bulk_get document.",
+              false
+            );
+          }
+          documents.set(key, ensureRevisionDocument(body, ref));
+          continue;
+        }
+        if (typeof item.error === "string") {
+          if (item.error === "forbidden" || item.error === "unauthorized") {
+            throw new StreamingFetchFailure(
+              "authentication",
+              `Fast Fetch could not read revision ${result.id}: CouchDB denied access.`,
+              false
+            );
+          }
+          throw new StreamingFetchFailure(
+            "protocol",
+            `Fast Fetch could not read a required revision for ${result.id}: ${String(item.reason ?? item.error)}.`,
+            false
+          );
+        }
+        throw new StreamingFetchFailure(
+          "protocol",
+          "Fast Fetch received a malformed _bulk_get document result.",
+          false
+        );
+      }
+    }
+    for (const revision of refs) {
+      if (!documents.has(revisionRefKey(revision.ref))) {
+        throw new StreamingFetchFailure(
+          "protocol",
+          `Fast Fetch did not receive required revision ${revision.ref.id}@${revision.ref.rev}.`,
+          false
+        );
+      }
+    }
+    return documents;
+  };
+  const fetchAndWriteBulkGetGroup = async (revisions) => {
+    const refs = revisions.map(({ ref }) => ref);
+    const requestBody = JSON.stringify({ docs: refs });
+    const response = await fetchResponse(
+      makeBulkGetURL().toString(),
+      { method: "POST", headers: bulkGetHeaders, body: requestBody },
+      "read missing revisions"
+    );
+    let responseText;
+    try {
+      const responseBody = await readResponseTextBounded(
+        response,
+        "read missing revisions",
+        revisions.length === 1 ? Number.MAX_SAFE_INTEGER : FAST_FETCH_REVISION_GROUP_BYTE_LIMIT
+      );
+      responseText = responseBody.text;
+      totalBytes += responseBody.byteLength;
+    } catch (error) {
+      if (error instanceof BulkGetResponseTooLargeError && revisions.length > 1) {
+        const midpoint = Math.ceil(revisions.length / 2);
+        await fetchAndWriteBulkGetGroup(revisions.slice(0, midpoint));
+        await fetchAndWriteBulkGetGroup(revisions.slice(midpoint));
+        return;
+      }
+      throw error;
+    }
+    const documents = parseBulkGetDocuments(responseText, revisions);
+    for (const revision of revisions) {
+      const document = documents.get(revisionRefKey(revision.ref));
+      if (!document) {
+        throw new StreamingFetchFailure(
+          "protocol",
+          `Fast Fetch did not receive required revision ${revision.ref.id}@${revision.ref.rev}.`,
+          false
+        );
+      }
+      await markRevisionDurable(revision, document);
+    }
+  };
+  const flushPendingRemoteRevisions = async () => {
+    if (pendingRemoteRevisions.size === 0) return;
+    const revisions = [...pendingRemoteRevisions.values()];
+    pendingRemoteRevisions.clear();
+    pendingRemoteRevisionBytes = 0;
+    for (let index = 0; index < revisions.length; index += FAST_FETCH_REVISION_GROUP_LIMIT) {
+      await fetchAndWriteBulkGetGroup(revisions.slice(index, index + FAST_FETCH_REVISION_GROUP_LIMIT));
+    }
+  };
+  const enqueueRemoteRevision = async (revision) => {
+    const key = revisionRefKey(revision.ref);
+    const existing = pendingRemoteRevisions.get(key);
+    if (existing) {
+      for (const row of revision.rows) existing.rows.add(row);
+      return;
+    }
+    const estimatedBytes = new TextEncoder().encode(JSON.stringify({ docs: [revision.ref] })).byteLength;
+    if (pendingRemoteRevisions.size > 0 && (pendingRemoteRevisions.size >= FAST_FETCH_REVISION_GROUP_LIMIT || pendingRemoteRevisionBytes + estimatedBytes > FAST_FETCH_REVISION_GROUP_BYTE_LIMIT)) {
+      await flushPendingRemoteRevisions();
+    }
+    pendingRemoteRevisions.set(key, revision);
+    pendingRemoteRevisionBytes += estimatedBytes;
+    if (pendingRemoteRevisions.size >= FAST_FETCH_REVISION_GROUP_LIMIT || pendingRemoteRevisionBytes >= FAST_FETCH_REVISION_GROUP_BYTE_LIMIT) {
+      await flushPendingRemoteRevisions();
+    }
+  };
+  const flushPendingRevisions = async (flushRemoteRemainder = false) => {
+    if (pendingRevisions.size > 0) {
+      const revisions = [...pendingRevisions.values()];
+      pendingRevisions.clear();
+      pendingRevisionBytes = 0;
+      const revisionsToCheck = [];
+      for (const revision of revisions) {
+        if (batchWriter.hasPendingRevision(revision.ref)) {
+          settleRevision(revision);
+        } else {
+          revisionsToCheck.push(revision);
+        }
+      }
+      if (revisionsToCheck.length > 0) {
+        const revisionDiff = /* @__PURE__ */ Object.create(null);
+        for (const { ref } of revisionsToCheck) {
+          const existing = revisionDiff[ref.id] ?? [];
+          existing.push(ref.rev);
+          revisionDiff[ref.id] = existing;
+        }
+        let difference;
+        try {
+          difference = await downloadToDB.revsDiff(revisionDiff);
+        } catch (error) {
+          throw new StreamingFetchFailure(
+            "storage",
+            `Fast Fetch could not check local revision history: ${errorMessage(error)}`,
+            false,
+            { cause: error }
+          );
+        }
+        if (!isObject(difference)) {
+          throw new StreamingFetchFailure(
+            "protocol",
+            "Fast Fetch received an invalid local revision check.",
+            false
+          );
+        }
+        for (const revision of revisionsToCheck) {
+          const item = Object.prototype.hasOwnProperty.call(difference, revision.ref.id) ? difference[revision.ref.id] : void 0;
+          if (item !== void 0 && (!isObject(item) || !Array.isArray(item.missing))) {
+            throw new StreamingFetchFailure(
+              "protocol",
+              "Fast Fetch received an invalid local revision check.",
+              false
+            );
+          }
+          const isMissing = item !== void 0 && item.missing.includes(revision.ref.rev);
+          if (!isMissing) {
+            settleRevision(revision);
+          } else if (revision.includedDocument) {
+            await markPreparedRevisionDurable(revision, revision.includedDocument);
+          } else {
+            await enqueueRemoteRevision(revision);
+          }
+        }
+      }
+    }
+    if (flushRemoteRemainder || pendingRemoteRevisions.size >= FAST_FETCH_REVISION_GROUP_LIMIT || pendingRemoteRevisionBytes >= FAST_FETCH_REVISION_GROUP_BYTE_LIMIT) {
+      await flushPendingRemoteRevisions();
+    }
+    await releaseCompletedRows();
+  };
+  const enqueueRevision = async (ref, row, includedDocument) => {
+    const key = revisionRefKey(ref);
+    const remoteExisting = pendingRemoteRevisions.get(key);
+    if (remoteExisting) {
+      remoteExisting.rows.add(row);
+      if (includedDocument !== void 0) {
+        pendingRemoteRevisions.delete(key);
+        pendingRemoteRevisionBytes -= new TextEncoder().encode(JSON.stringify({ docs: [ref] })).byteLength;
+        await markPreparedRevisionDurable(remoteExisting, includedDocument);
+      }
+      return;
+    }
+    const existing = pendingRevisions.get(key);
+    if (existing) {
+      existing.rows.add(row);
+      if (existing.includedDocument === void 0 && includedDocument !== void 0) {
+        existing.includedDocument = includedDocument;
+      }
+      return;
+    }
+    const estimatedBytes = new TextEncoder().encode(JSON.stringify({ ref, includedDocument })).byteLength;
+    if (pendingRevisions.size > 0 && (pendingRevisions.size >= FAST_FETCH_REVISION_GROUP_LIMIT || pendingRevisionBytes + estimatedBytes > FAST_FETCH_REVISION_GROUP_BYTE_LIMIT)) {
+      await flushPendingRevisions();
+    }
+    pendingRevisions.set(key, {
+      ref,
+      rows: /* @__PURE__ */ new Set([row]),
+      ...includedDocument === void 0 ? {} : { includedDocument }
+    });
+    pendingRevisionBytes += estimatedBytes;
+    if (pendingRevisions.size >= FAST_FETCH_REVISION_GROUP_LIMIT || pendingRevisionBytes >= FAST_FETCH_REVISION_GROUP_BYTE_LIMIT) {
+      await flushPendingRevisions();
+    }
+  };
   const readAvailableChanges = async (pageSince) => {
     const statusURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
       ...changesBaseParams,
@@ -352,6 +757,11 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     return available;
   };
   const fetchPage = async (pageSince, pageLimit) => {
+    if (pendingRevisions.size > 0 || pendingRemoteRevisions.size > 0) {
+      throw new StreamingFetchFailure("protocol", "Fast Fetch began a page with unresolved revisions.", false);
+    }
+    rowProgress.length = 0;
+    nextCheckpointRow = 0;
     const controller = new AbortController();
     let reader;
     let buffer = "";
@@ -366,6 +776,15 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
             true
           );
         }
+        await flushPendingRevisions(true);
+        await releaseCompletedRows();
+        if (nextCheckpointRow !== rowProgress.length) {
+          throw new StreamingFetchFailure(
+            "protocol",
+            "Fast Fetch reached a page terminator with unresolved leaf revisions.",
+            false
+          );
+        }
         await batchWriter.flush();
         await saveCheckpoint(onCheckpoint, parsed.lastSequence);
         reportProgress(true);
@@ -381,12 +800,55 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
       const change = parsed.change;
       fetchedRows++;
       totalFetched++;
-      if (change.doc) {
-        await batchWriter.write(change.doc, change.seq);
-        totalValidFetched++;
-      } else {
-        await batchWriter.flushThrough(change.seq);
+      if (typeof change.id !== "string" || change.id.length === 0 || !Array.isArray(change.changes) || change.changes.length === 0 || change.changes.some((entry) => !isObject(entry) || typeof entry.rev !== "string" || entry.rev.length === 0)) {
+        throw new StreamingFetchFailure(
+          "protocol",
+          "Fast Fetch received a changes row without valid leaf revisions.",
+          false
+        );
+      }
+      const leafRevisions = [...new Set(change.changes.map(({ rev }) => rev))];
+      let includedDocument;
+      if (change.doc !== void 0) {
+        if (!isObject(change.doc) || typeof change.doc._rev !== "string" || !leafRevisions.includes(change.doc._rev)) {
+          throw new StreamingFetchFailure(
+            "protocol",
+            "Fast Fetch received a changes row with an invalid included document.",
+            false
+          );
+        }
+        const includedRef = { id: change.id, rev: change.doc._rev };
+        const includedBody = ensureRevisionIdentity(change.doc, includedRef);
+        const revision = parseRevisionIdentifier(includedRef.rev);
+        if (revision === void 0) {
+          throw new StreamingFetchFailure(
+            "protocol",
+            `Fast Fetch received an invalid revision identifier for ${includedRef.id}.`,
+            false
+          );
+        }
+        if (revision.generation === 1) {
+          const revisionDocument = ensureRevisionDocument(
+            { ...includedBody, _revisions: { start: 1, ids: [revision.hash] } },
+            includedRef
+          );
+          includedDocument = await batchWriter.prepare(revisionDocument);
+        } else if (hasValidRevisionAncestry(includedBody, revision)) {
+          const revisionDocument = ensureRevisionDocument(includedBody, includedRef);
+          includedDocument = await batchWriter.prepare(revisionDocument);
+        }
+      }
+      totalValidFetched++;
+      const row = { sequence: change.seq, pendingRevisions: leafRevisions.length, complete: false };
+      rowProgress.push(row);
+      for (const rev of leafRevisions) {
+        await enqueueRevision(
+          { id: change.id, rev },
+          row,
+          change.doc?._rev === rev ? includedDocument : void 0
+        );
       }
+      await releaseCompletedRows();
       reportProgress();
       return void 0;
     };
@@ -493,6 +955,8 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     const failure = asStreamingFetchFailure(error);
     if (failure.stage !== "storage") {
       try {
+        await flushPendingRevisions(true);
+        await releaseCompletedRows();
         await batchWriter.flush();
       } catch (flushError) {
         batchWriter.abort();