@vrtmrz/livesync-commonlib 0.1.6 → 0.1.8

package/dist/pouchdb/StreamingFetch.js

@@ -1,5 +1,7 @@
 import { _fetch } from "../common/coreEnvFunctions.js";
 import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
+const FAST_FETCH_CHANGES_PAGE_LIMIT = 1e4;
+const FAST_FETCH_CHANGES_PAGE_TIMEOUT_MS = 1e3;
 class StreamingFetchFailure extends Error {
   constructor(stage, message, retryable, options) {
     super(message, options?.cause === void 0 ? void 0 : { cause: options.cause });
@@ -174,10 +176,6 @@ function setParamsToURL(url, params) {
   }
   return url;
 }
-function isTargetSequence(sequence, targetSequence) {
-  if (sequence === void 0 || targetSequence === void 0) return false;
-  return sequence.toString() === targetSequence.toString();
-}
 function parseStatusSource(source) {
   const trimmed = source.trim();
   if (!trimmed) {
@@ -193,7 +191,7 @@ function parseStatusSource(source) {
   }
   throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes status from CouchDB.", false);
 }
-function parseChangeLine(line) {
+function parseChangesFeedLine(line) {
   let parsed;
   try {
     parsed = JSON.parse(line);
@@ -202,7 +200,21 @@ function parseChangeLine(line) {
       cause: error
     });
   }
-  if (!parsed || typeof parsed !== "object" || !("seq" in parsed)) {
+  if (!parsed || typeof parsed !== "object") {
+    throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed line.", false);
+  }
+  if (!("seq" in parsed) && "last_seq" in parsed) {
+    const lastSequence = parsed.last_seq;
+    if (lastSequence === void 0) {
+      throw new StreamingFetchFailure(
+        "protocol",
+        "Fast Fetch received a changes-feed terminator without a sequence.",
+        false
+      );
+    }
+    return { type: "terminator", lastSequence };
+  }
+  if (!("seq" in parsed)) {
     throw new StreamingFetchFailure(
       "protocol",
       "Fast Fetch received a changes-feed row without a sequence.",
@@ -213,7 +225,7 @@ function parseChangeLine(line) {
   if (change.seq === void 0 || change.doc !== void 0 && (!change.doc || typeof change.doc !== "object")) {
     throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed row.", false);
   }
-  return change;
+  return { type: "change", change };
 }
 async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint) {
   let totalFetched = 0;
@@ -225,8 +237,7 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     style: "all_docs",
     conflicts: "true",
     revs: "true",
-    since: since.toString(),
-    heartbeat: "30000"
+    since: since.toString()
   };
   const fetchHeaders = {
     Accept: "application/json",
@@ -236,7 +247,8 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     ...changesBaseParams,
     feed: "normal",
     since: "now",
-    limit: "0"
+    limit: "1",
+    include_docs: "false"
   });
   const targetResponse = await fetchResponse(
     targetURL.toString(),
@@ -244,121 +256,206 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     "capture the changes target"
   );
   const targetStatus = parseStatusSource(await readResponseText(targetResponse, "read the changes target"));
-  const finalTargetSeq = targetStatus.last_seq;
-  if (finalTargetSeq === void 0) {
-    throw new StreamingFetchFailure(
-      "protocol",
-      "Fast Fetch could not obtain an authoritative changes target from CouchDB.",
-      false
-    );
-  }
-  if (isTargetSequence(since, finalTargetSeq)) {
-    Logger("Already at the target sequence. Initial data synchronisation is complete.");
-    return;
-  }
-  const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
-    ...changesBaseParams,
-    feed: "normal",
-    limit: "0"
-  });
-  const infoResponse = await fetchResponse(fetchURL.toString(), { headers: fetchHeaders }, "read changes status");
-  const info = parseStatusSource(await readResponseText(infoResponse, "read changes status"));
-  if (typeof info.pending !== "number" || info.pending < 0) {
+  const progressTargetSeq = targetStatus.last_seq;
+  if (progressTargetSeq === void 0) {
     throw new StreamingFetchFailure(
       "protocol",
-      "Fast Fetch received changes status without a valid pending count.",
+      "Fast Fetch could not obtain a changes progress target from CouchDB.",
       false
     );
   }
-  const pendingDocs = info.pending;
-  const docsToFetch = pendingDocs;
-  if (pendingDocs === 0) {
-    await saveCheckpoint(onCheckpoint, finalTargetSeq);
-    Logger("No changes remain before the captured Fast Fetch target.");
-    return;
-  }
-  Logger(
-    `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Estimated documents to fetch: ${docsToFetch}.`
-  );
-  const controller = new AbortController();
-  const changesURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), changesBaseParams);
-  const response = await fetchResponse(
-    changesURL.toString(),
-    {
-      method: "GET",
-      headers: fetchHeaders,
-      signal: controller.signal
-    },
-    "open the changes feed"
-  );
-  if (!response.body) {
-    throw new StreamingFetchFailure("protocol", "Fast Fetch could not read the CouchDB response stream.", false);
-  }
-  const sizeCaptureStream = new TransformStream({
-    transform(chunk, streamController) {
-      totalBytes += chunk.byteLength;
-      streamController.enqueue(chunk);
-    }
-  });
-  const reader = response.body.pipeThrough(sizeCaptureStream).pipeThrough(new TextDecoderStream()).getReader();
   const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint);
-  let buffer = "";
+  let docsToFetch = 0;
   let lastProgress = 0;
   let lastReportTime = Date.now();
-  const reportProgress = () => {
-    if (totalFetched - lastProgress < 25 && Date.now() - lastReportTime < 2e3) return;
+  const reportProgress = (force = false) => {
+    if (!force && totalFetched - lastProgress < 25 && Date.now() - lastReportTime < 2e3) return;
     lastProgress = totalFetched;
     lastReportTime = Date.now();
     onProgress?.({
       totalFetched,
       totalValidFetched,
-      targetSeq: finalTargetSeq,
+      targetSeq: progressTargetSeq,
       docsToFetch,
       totalBytes
     });
   };
-  const processLine = async (line) => {
-    const parsed = parseChangeLine(line);
-    totalFetched++;
-    if (parsed.doc) {
-      await batchWriter.write(parsed.doc, parsed.seq);
-      totalValidFetched++;
-    } else {
-      await batchWriter.flushThrough(parsed.seq);
+  const readAvailableChanges = async (pageSince) => {
+    const statusURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
+      ...changesBaseParams,
+      feed: "normal",
+      since: pageSince.toString(),
+      limit: "1",
+      include_docs: "false"
+    });
+    const response = await fetchResponse(
+      statusURL.toString(),
+      { method: "GET", headers: fetchHeaders },
+      "read changes status"
+    );
+    const status = parseStatusSource(await readResponseText(response, "read changes status"));
+    if (!Array.isArray(status.results)) {
+      throw new StreamingFetchFailure(
+        "protocol",
+        "Fast Fetch received changes status without a valid results list.",
+        false
+      );
+    }
+    const pending = status.pending;
+    if (typeof pending !== "number" || !Number.isSafeInteger(pending) || pending < 0) {
+      throw new StreamingFetchFailure(
+        "protocol",
+        "Fast Fetch received changes status without a valid pending count.",
+        false
+      );
     }
-    reportProgress();
-    if (!isTargetSequence(parsed.seq, finalTargetSeq)) return false;
-    await batchWriter.flush();
-    Logger("The captured Fast Fetch target is durable in the local database.");
-    controller.abort();
-    reportProgress();
-    return true;
+    const available = status.results.length + pending;
+    if (!Number.isSafeInteger(available)) {
+      throw new StreamingFetchFailure(
+        "protocol",
+        "Fast Fetch received a changes count outside the supported range.",
+        false
+      );
+    }
+    return available;
   };
-  try {
-    while (true) {
-      reportProgress();
-      let readResult;
-      try {
-        readResult = await reader.read();
-      } catch (error) {
-        throw transportFailure("read the changes feed", error);
-      }
-      if (readResult.done) {
-        if (buffer.trim() && await processLine(buffer)) return;
+  const fetchPage = async (pageSince, pageLimit) => {
+    const controller = new AbortController();
+    let reader;
+    let buffer = "";
+    let fetchedRows = 0;
+    const processLine = async (line) => {
+      const parsed = parseChangesFeedLine(line);
+      if (parsed.type === "terminator") {
+        if (fetchedRows === 0) {
+          throw new StreamingFetchFailure(
+            "transport",
+            "Fast Fetch received no rows after its status probe reported available changes.",
+            true
+          );
+        }
         await batchWriter.flush();
+        await saveCheckpoint(onCheckpoint, parsed.lastSequence);
+        reportProgress(true);
+        return parsed.lastSequence;
+      }
+      if (fetchedRows >= pageLimit) {
         throw new StreamingFetchFailure(
-          "transport",
-          "The CouchDB changes feed ended before Fast Fetch reached its captured target.",
-          true
+          "protocol",
+          `Fast Fetch received more than its ${pageLimit}-row page limit.`,
+          false
         );
       }
-      buffer += readResult.value;
-      const lines = buffer.split("\n");
-      buffer = lines.pop() || "";
-      for (const line of lines) {
-        if (!line.trim()) continue;
-        if (await processLine(line)) return;
+      const change = parsed.change;
+      fetchedRows++;
+      totalFetched++;
+      if (change.doc) {
+        await batchWriter.write(change.doc, change.seq);
+        totalValidFetched++;
+      } else {
+        await batchWriter.flushThrough(change.seq);
       }
+      reportProgress();
+      return void 0;
+    };
+    try {
+      const changesURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
+        ...changesBaseParams,
+        since: pageSince.toString(),
+        limit: pageLimit.toString(),
+        timeout: FAST_FETCH_CHANGES_PAGE_TIMEOUT_MS.toString()
+      });
+      const response = await fetchResponse(
+        changesURL.toString(),
+        {
+          method: "GET",
+          headers: fetchHeaders,
+          signal: controller.signal
+        },
+        "open the changes feed"
+      );
+      if (!response.body) {
+        throw new StreamingFetchFailure(
+          "protocol",
+          "Fast Fetch could not read the CouchDB response stream.",
+          false
+        );
+      }
+      const decoder = new TextDecoder();
+      const byteCountingDecoderStream = new TransformStream({
+        transform(chunk, streamController) {
+          totalBytes += chunk.byteLength;
+          const decoded = decoder.decode(chunk, { stream: true });
+          if (decoded) streamController.enqueue(decoded);
+        },
+        flush(streamController) {
+          const decoded = decoder.decode();
+          if (decoded) streamController.enqueue(decoded);
+        }
+      });
+      reader = response.body.pipeThrough(byteCountingDecoderStream).getReader();
+      while (true) {
+        reportProgress();
+        let readResult;
+        try {
+          readResult = await reader.read();
+        } catch (error) {
+          throw transportFailure("read the changes feed", error);
+        }
+        if (readResult.done) {
+          if (buffer.trim()) {
+            const lastSequence = await processLine(buffer);
+            if (lastSequence !== void 0) return lastSequence;
+          }
+          await batchWriter.flush();
+          throw new StreamingFetchFailure(
+            "transport",
+            "The bounded CouchDB changes feed ended without a last_seq terminator.",
+            true
+          );
+        }
+        buffer += readResult.value;
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        for (const line of lines) {
+          if (!line.trim()) continue;
+          const lastSequence = await processLine(line);
+          if (lastSequence !== void 0) return lastSequence;
+        }
+      }
+    } finally {
+      controller.abort();
+      if (reader) {
+        try {
+          await reader.cancel();
+        } catch {
+        }
+        reader.releaseLock();
+      }
+    }
+  };
+  try {
+    let pageSince = since;
+    let started = false;
+    while (true) {
+      const available = await readAvailableChanges(pageSince);
+      docsToFetch = Math.max(docsToFetch, totalFetched + available);
+      if (available === 0) break;
+      if (!started) {
+        started = true;
+        Logger(
+          `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${progressTargetSeq}, Documents to fetch: ${docsToFetch}.`
+        );
+      }
+      const pageLimit = Math.min(FAST_FETCH_CHANGES_PAGE_LIMIT, available);
+      const pageTerminator = await fetchPage(pageSince, pageLimit);
+      pageSince = pageTerminator;
+    }
+    if (started) {
+      Logger("Fast Fetch is caught up and durable in the local database.");
+      reportProgress(true);
+    } else {
+      Logger("No changes remain for Fast Fetch.");
     }
   } catch (error) {
     const failure = asStreamingFetchFailure(error);
@@ -374,13 +471,6 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     Logger(`Fast Fetch failed during ${failure.stage}.`, LOG_LEVEL_VERBOSE);
     Logger(failure, LOG_LEVEL_VERBOSE);
     throw failure;
-  } finally {
-    controller.abort();
-    try {
-      await reader.cancel();
-    } catch {
-    }
-    reader.releaseLock();
   }
 }
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vrtmrz/livesync-commonlib",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Shared data, storage, and replication primitives for Self-hosted LiveSync clients.",
   "type": "module",
   "license": "MIT",