@vrtmrz/livesync-commonlib 0.1.5 → 0.1.6

package/dist/pouchdb/StreamingFetch.d.ts

@@ -1,6 +1,40 @@
 import type { EntryDoc } from "../common/models/db.definition.js";
 import type { AnyEntry, EntryLeaf } from "../common/models/db.type.js";
 type DBSequence = number | string;
+/**
+ * Identifies the boundary at which Fast Fetch stopped.
+ *
+ * - `transport`: the request or response stream was interrupted. Only failures
+ *   explicitly created at this boundary may be retried.
+ * - `authentication`: CouchDB rejected the supplied credentials.
+ * - `protocol`: CouchDB returned an unsuccessful or structurally invalid
+ *   response, or an unexpected exception escaped the processing pipeline.
+ * - `decryption`: a remote document could not be decrypted or did not produce a
+ *   serialisable PouchDB document.
+ * - `storage`: a local batch or its durable checkpoint could not be written.
+ *
+ * This remains a string-literal union because the values are discriminants, not
+ * data to enumerate, persist, or map to user-interface labels. The constructor
+ * and consumers therefore receive typo checking without a runtime constants
+ * object becoming part of the package API.
+ */
+export type StreamingFetchFailureStage = "transport" | "authentication" | "protocol" | "decryption" | "storage";
+/**
+ * A classified Fast Fetch failure. Retryability is explicit rather than inferred
+ * from the stage so that future exceptions can be narrowed without broadening all
+ * failures at that boundary.
+ */
+export declare class StreamingFetchFailure extends Error {
+    readonly stage: StreamingFetchFailureStage;
+    readonly retryable: boolean;
+    readonly name = "StreamingFetchFailure";
+    constructor(stage: StreamingFetchFailureStage, message: string, retryable: boolean, options?: {
+        status?: number;
+        cause?: unknown;
+    });
+    readonly status?: number;
+}
+export declare function isRetryableStreamingFetchFailure(error: unknown): error is StreamingFetchFailure;
 export type FetchChangesForInitialSyncProgress = {
     totalFetched: number;
     totalValidFetched: number;

package/dist/pouchdb/StreamingFetch.js

@@ -1,58 +1,172 @@
 import { _fetch } from "../common/coreEnvFunctions.js";
 import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
-function generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint) {
+class StreamingFetchFailure extends Error {
+  constructor(stage, message, retryable, options) {
+    super(message, options?.cause === void 0 ? void 0 : { cause: options.cause });
+    this.stage = stage;
+    this.retryable = retryable;
+    this.status = options?.status;
+  }
+  stage;
+  retryable;
+  name = "StreamingFetchFailure";
+  status;
+}
+function isRetryableStreamingFetchFailure(error) {
+  return error instanceof StreamingFetchFailure && error.retryable;
+}
+function errorMessage(error) {
+  if (error instanceof Error && error.message) return error.message;
+  return String(error);
+}
+function asStreamingFetchFailure(error) {
+  if (error instanceof StreamingFetchFailure) return error;
+  return new StreamingFetchFailure(
+    "protocol",
+    `Fast Fetch encountered an unexpected processing failure: ${errorMessage(error)}`,
+    false,
+    { cause: error }
+  );
+}
+function transportFailure(operation, error) {
+  return new StreamingFetchFailure("transport", `Fast Fetch could not ${operation}: ${errorMessage(error)}`, true, {
+    cause: error
+  });
+}
+function responseFailure(response, operation) {
+  const status = response.status;
+  if (status === 401 || status === 403) {
+    return new StreamingFetchFailure(
+      "authentication",
+      `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
+      false,
+      { status }
+    );
+  }
+  const retryable = status === 408 || status === 429 || status >= 500;
+  return new StreamingFetchFailure(
+    retryable ? "transport" : "protocol",
+    `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
+    retryable,
+    { status }
+  );
+}
+function ensureResponseOK(response, operation) {
+  if (!response.ok) throw responseFailure(response, operation);
+}
+async function fetchResponse(url, init, operation) {
+  let response;
+  try {
+    response = await _fetch(url, init);
+  } catch (error) {
+    throw transportFailure(operation, error);
+  }
+  ensureResponseOK(response, operation);
+  return response;
+}
+async function readResponseText(response, operation) {
+  try {
+    return await response.text();
+  } catch (error) {
+    throw transportFailure(operation, error);
+  }
+}
+async function saveCheckpoint(onCheckpoint, sequence) {
+  try {
+    await onCheckpoint?.(sequence);
+  } catch (error) {
+    throw new StreamingFetchFailure(
+      "storage",
+      `Fast Fetch could not save its checkpoint: ${errorMessage(error)}`,
+      false,
+      { cause: error }
+    );
+  }
+}
+function generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint) {
   let batchBuffer = [];
   let currentBatchSizeBytes = 0;
   let batchLastSequence;
   const BATCH_ITEM_LIMIT = 100;
   const BATCH_SIZE_LIMIT = 2 * 1024 * 1024;
-  let flushPromise = null;
-  const flushToDB = async () => {
+  const flush = async () => {
     if (batchBuffer.length === 0) return;
-    if (flushPromise) {
-      await flushPromise;
-    }
+    const documents = batchBuffer;
+    const checkpoint = batchLastSequence;
+    let results;
     try {
-      flushPromise = downloadToDB.bulkDocs(batchBuffer, { new_edits: false });
-      await flushPromise;
-      if (batchLastSequence !== void 0) {
-        await onCheckpoint?.(batchLastSequence);
-      }
+      results = await downloadToDB.bulkDocs(documents, { new_edits: false });
     } catch (error) {
-      Logger("Error bulk writing to PouchDB:", LOG_LEVEL_VERBOSE);
-      Logger(error, LOG_LEVEL_VERBOSE);
-      throw error;
-    } finally {
-      batchBuffer = [];
-      currentBatchSizeBytes = 0;
-      batchLastSequence = void 0;
-      flushPromise = null;
+      throw new StreamingFetchFailure(
+        "storage",
+        `Fast Fetch could not write a batch to the local database: ${errorMessage(error)}`,
+        false,
+        { cause: error }
+      );
+    }
+    const failedResult = results.find(
+      (result) => "error" in result && Boolean(result.error)
+    );
+    if (failedResult) {
+      const detail = failedResult.message || failedResult.name || "the local database rejected a document";
+      throw new StreamingFetchFailure(
+        "storage",
+        `Fast Fetch could not write a batch to the local database: ${detail}`,
+        false,
+        { cause: failedResult }
+      );
     }
+    if (checkpoint !== void 0) {
+      await saveCheckpoint(onCheckpoint, checkpoint);
+    }
+    batchBuffer = [];
+    currentBatchSizeBytes = 0;
+    batchLastSequence = void 0;
   };
-  return new WritableStream({
-    async write(chunk) {
+  return {
+    async write(doc, sequence) {
+      let decryptedDoc;
+      try {
+        decryptedDoc = await decryptFunction(doc);
+      } catch (error) {
+        throw new StreamingFetchFailure(
+          "decryption",
+          `Fast Fetch could not decrypt a document: ${errorMessage(error)}`,
+          false,
+          { cause: error }
+        );
+      }
+      let serialisedDoc;
       try {
-        const decryptedDoc = await decryptFunction(chunk.doc);
-        batchBuffer.push(decryptedDoc);
-        currentBatchSizeBytes += JSON.stringify(decryptedDoc).length;
-        batchLastSequence = chunk.seq;
-        if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
-          await flushToDB();
-        }
+        const serialised = JSON.stringify(decryptedDoc);
+        if (serialised === void 0) throw new Error("the decrypted value is not a document");
+        serialisedDoc = serialised;
       } catch (error) {
-        Logger("Error processing document stream:", LOG_LEVEL_VERBOSE);
-        throw error;
+        throw new StreamingFetchFailure(
+          "decryption",
+          `Fast Fetch produced an invalid decrypted document: ${errorMessage(error)}`,
+          false,
+          { cause: error }
+        );
+      }
+      batchBuffer.push(decryptedDoc);
+      currentBatchSizeBytes += serialisedDoc.length;
+      batchLastSequence = sequence;
+      if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
+        await flush();
       }
     },
-    async close() {
-      await flushToDB();
+    async flushThrough(sequence) {
+      await flush();
+      await saveCheckpoint(onCheckpoint, sequence);
     },
-    abort(reason) {
-      Logger(`Stream aborted: ${reason}`, LOG_LEVEL_VERBOSE);
+    flush,
+    abort() {
       batchBuffer = [];
       currentBatchSizeBytes = 0;
+      batchLastSequence = void 0;
     }
-  });
+  };
 }
 function setParamsToURL(url, params) {
   for (const [key, value] of Object.entries(params)) {
@@ -60,21 +174,51 @@ function setParamsToURL(url, params) {
   }
   return url;
 }
-function reachedTargetSequence(seq, targetSeq) {
-  if (seq === void 0 || targetSeq === void 0) return false;
-  const seqStr = seq.toString();
-  const targetSeqStr = targetSeq.toString();
-  if (seqStr === targetSeqStr) return true;
-  const seqNum = parseInt(seqStr.split("-")[0], 10);
-  const targetSeqNum = parseInt(targetSeqStr.split("-")[0], 10);
-  if (!isNaN(seqNum) && !isNaN(targetSeqNum)) {
-    return seqNum >= targetSeqNum;
+function isTargetSequence(sequence, targetSequence) {
+  if (sequence === void 0 || targetSequence === void 0) return false;
+  return sequence.toString() === targetSequence.toString();
+}
+function parseStatusSource(source) {
+  const trimmed = source.trim();
+  if (!trimmed) {
+    throw new StreamingFetchFailure("protocol", "Fast Fetch received no changes status from CouchDB.", false);
+  }
+  const candidates = [trimmed, ...trimmed.split("\n").reverse().filter(Boolean)];
+  for (const candidate of candidates) {
+    try {
+      const parsed = JSON.parse(candidate);
+      if (parsed && typeof parsed === "object") return parsed;
+    } catch {
+    }
+  }
+  throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes status from CouchDB.", false);
+}
+function parseChangeLine(line) {
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch (error) {
+    throw new StreamingFetchFailure("protocol", "Fast Fetch received a malformed changes-feed row.", false, {
+      cause: error
+    });
+  }
+  if (!parsed || typeof parsed !== "object" || !("seq" in parsed)) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      "Fast Fetch received a changes-feed row without a sequence.",
+      false
+    );
   }
-  return false;
+  const change = parsed;
+  if (change.seq === void 0 || change.doc !== void 0 && (!change.doc || typeof change.doc !== "object")) {
+    throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed row.", false);
+  }
+  return change;
 }
 async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint) {
   let totalFetched = 0;
   let totalValidFetched = 0;
+  let totalBytes = 0;
   const changesBaseParams = {
     feed: "continuous",
     include_docs: "true",
@@ -88,92 +232,81 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
     Accept: "application/json",
     Authorization: authHeader
   };
-  let targetSeq = void 0;
-  try {
-    const dbInfoRes = await _fetch(remoteDbUrl, {
-      headers: fetchHeaders
-    });
-    if (dbInfoRes.ok) {
-      const dbInfo = await dbInfoRes.json();
-      targetSeq = dbInfo.update_seq;
-    }
-  } catch (e) {
-    Logger("Failed to fetch database info for target sequence:", LOG_LEVEL_VERBOSE);
-    Logger(e, LOG_LEVEL_VERBOSE);
-  }
-  const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
+  const targetURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
     ...changesBaseParams,
     feed: "normal",
-    limit: "1"
-  });
-  const infoRes = await _fetch(fetchURL.toString(), {
-    headers: fetchHeaders
+    since: "now",
+    limit: "0"
   });
-  const infoSource = await infoRes.text();
-  const infoSourceTrimmed = infoSource.trim();
-  if (!infoSourceTrimmed) {
-    throw new Error("Failed to fetch changes from CouchDB. No data received.");
+  const targetResponse = await fetchResponse(
+    targetURL.toString(),
+    { headers: fetchHeaders },
+    "capture the changes target"
+  );
+  const targetStatus = parseStatusSource(await readResponseText(targetResponse, "read the changes target"));
+  const finalTargetSeq = targetStatus.last_seq;
+  if (finalTargetSeq === void 0) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      "Fast Fetch could not obtain an authoritative changes target from CouchDB.",
+      false
+    );
   }
-  let info;
-  if (infoSourceTrimmed.startsWith("{") && infoSourceTrimmed.endsWith("}")) {
-    try {
-      info = JSON.parse(infoSourceTrimmed);
-    } catch {
-      const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
-      const lastLine = infoLines[infoLines.length - 1];
-      info = JSON.parse(lastLine);
-    }
-  } else {
-    const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
-    const lastLine = infoLines[infoLines.length - 1];
-    info = JSON.parse(lastLine);
+  if (isTargetSequence(since, finalTargetSeq)) {
+    Logger("Already at the target sequence. Initial data synchronisation is complete.");
+    return;
   }
-  const pendingDocs = info.pending || 0;
-  const docsToFetch = pendingDocs + 1;
-  if (targetSeq === void 0) {
-    targetSeq = info.last_seq || info.update_seq || info.seq;
+  const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
+    ...changesBaseParams,
+    feed: "normal",
+    limit: "0"
+  });
+  const infoResponse = await fetchResponse(fetchURL.toString(), { headers: fetchHeaders }, "read changes status");
+  const info = parseStatusSource(await readResponseText(infoResponse, "read changes status"));
+  if (typeof info.pending !== "number" || info.pending < 0) {
+    throw new StreamingFetchFailure(
+      "protocol",
+      "Fast Fetch received changes status without a valid pending count.",
+      false
+    );
   }
-  const finalTargetSeq = targetSeq ?? "";
-  if (reachedTargetSequence(since, finalTargetSeq)) {
-    Logger("Already at the target sequence. Initial data synchronisation is complete.");
+  const pendingDocs = info.pending;
+  const docsToFetch = pendingDocs;
+  if (pendingDocs === 0) {
+    await saveCheckpoint(onCheckpoint, finalTargetSeq);
+    Logger("No changes remain before the captured Fast Fetch target.");
     return;
   }
   Logger(
-    `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Total documents to fetch: ${docsToFetch}.`
+    `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Estimated documents to fetch: ${docsToFetch}.`
   );
   const controller = new AbortController();
-  const url = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
-    ...changesBaseParams
-  });
-  const response = await _fetch(url.toString(), {
-    method: "GET",
-    headers: fetchHeaders,
-    signal: controller.signal
-  });
+  const changesURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), changesBaseParams);
+  const response = await fetchResponse(
+    changesURL.toString(),
+    {
+      method: "GET",
+      headers: fetchHeaders,
+      signal: controller.signal
+    },
+    "open the changes feed"
+  );
   if (!response.body) {
-    throw new Error("ReadableStream is not supported by this browser.");
+    throw new StreamingFetchFailure("protocol", "Fast Fetch could not read the CouchDB response stream.", false);
   }
   const sizeCaptureStream = new TransformStream({
-    transform(chunk, controller2) {
-      const chunkSize = chunk.length || chunk.byteLength || 0;
-      totalBytes += chunkSize;
-      controller2.enqueue(chunk);
+    transform(chunk, streamController) {
+      totalBytes += chunk.byteLength;
+      streamController.enqueue(chunk);
     }
   });
   const reader = response.body.pipeThrough(sizeCaptureStream).pipeThrough(new TextDecoderStream()).getReader();
-  const writeDocStream = generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint);
-  const writer = writeDocStream.getWriter();
+  const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint);
   let buffer = "";
   let lastProgress = 0;
   let lastReportTime = Date.now();
-  let totalBytes = 0;
-  let abortedAfterTargetReached = false;
   const reportProgress = () => {
-    if (totalFetched - lastProgress < 25) {
-      if (Date.now() - lastReportTime < 2e3) {
-        return;
-      }
-    }
+    if (totalFetched - lastProgress < 25 && Date.now() - lastReportTime < 2e3) return;
     lastProgress = totalFetched;
     lastReportTime = Date.now();
     onProgress?.({
@@ -184,80 +317,74 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
       totalBytes
     });
   };
+  const processLine = async (line) => {
+    const parsed = parseChangeLine(line);
+    totalFetched++;
+    if (parsed.doc) {
+      await batchWriter.write(parsed.doc, parsed.seq);
+      totalValidFetched++;
+    } else {
+      await batchWriter.flushThrough(parsed.seq);
+    }
+    reportProgress();
+    if (!isTargetSequence(parsed.seq, finalTargetSeq)) return false;
+    await batchWriter.flush();
+    Logger("The captured Fast Fetch target is durable in the local database.");
+    controller.abort();
+    reportProgress();
+    return true;
+  };
   try {
     while (true) {
       reportProgress();
-      const { value, done } = await reader.read();
-      if (value) {
-        totalBytes += value.length;
+      let readResult;
+      try {
+        readResult = await reader.read();
+      } catch (error) {
+        throw transportFailure("read the changes feed", error);
       }
-      if (done) {
-        if (buffer.trim()) {
-          try {
-            totalFetched++;
-            const parsed = JSON.parse(buffer);
-            if (parsed.doc) {
-              await writer.write({ doc: parsed.doc, seq: parsed.seq });
-              totalValidFetched++;
-            } else {
-              await onCheckpoint?.(parsed.seq);
-            }
-            reportProgress();
-          } catch (e) {
-            Logger(`Failed to parse the final line: ${buffer}. Skipping it.`);
-            Logger(e, LOG_LEVEL_VERBOSE);
-          }
-        }
-        await writer.close();
-        break;
+      if (readResult.done) {
+        if (buffer.trim() && await processLine(buffer)) return;
+        await batchWriter.flush();
+        throw new StreamingFetchFailure(
+          "transport",
+          "The CouchDB changes feed ended before Fast Fetch reached its captured target.",
+          true
+        );
       }
-      buffer += value;
+      buffer += readResult.value;
       const lines = buffer.split("\n");
       buffer = lines.pop() || "";
       for (const line of lines) {
         if (!line.trim()) continue;
-        try {
-          const parsed = JSON.parse(line);
-          totalFetched++;
-          if (parsed.doc) {
-            await writer.write({ doc: parsed.doc, seq: parsed.seq });
-            totalValidFetched++;
-          } else {
-            await onCheckpoint?.(parsed.seq);
-          }
-          reportProgress();
-          if (totalFetched >= docsToFetch || reachedTargetSequence(parsed.seq, finalTargetSeq)) {
-            Logger(
-              `All documents fetched. Stopping the stream and writing remaining documents to PouchDB...`
-            );
-            await writer.close();
-            abortedAfterTargetReached = true;
-            controller.abort();
-            reportProgress();
-            return;
-          }
-        } catch (e) {
-          Logger(`JSON parsing failed. Skipping line: ${line}`, LOG_LEVEL_VERBOSE);
-          Logger(e, LOG_LEVEL_VERBOSE);
-        }
+        if (await processLine(line)) return;
       }
     }
-    Logger("Initial data synchronisation via stream has completed.");
-    reportProgress();
   } catch (error) {
-    if (abortedAfterTargetReached && error instanceof DOMException && error.name === "AbortError") {
-      Logger(
-        "Stream has been aborted as the target sequence has been reached. Finalising the synchronising process..."
-      );
-      return;
+    const failure = asStreamingFetchFailure(error);
+    if (failure.stage !== "storage") {
+      try {
+        await batchWriter.flush();
+      } catch (flushError) {
+        batchWriter.abort();
+        throw asStreamingFetchFailure(flushError);
+      }
     }
-    Logger("An error occurred during synchronisation:", LOG_LEVEL_VERBOSE);
-    Logger(error, LOG_LEVEL_VERBOSE);
-    throw error;
+    batchWriter.abort();
+    Logger(`Fast Fetch failed during ${failure.stage}.`, LOG_LEVEL_VERBOSE);
+    Logger(failure, LOG_LEVEL_VERBOSE);
+    throw failure;
   } finally {
+    controller.abort();
+    try {
+      await reader.cancel();
+    } catch {
+    }
     reader.releaseLock();
   }
 }
 export {
-  fetchChangesForInitialSync
+  StreamingFetchFailure,
+  fetchChangesForInitialSync,
+  isRetryableStreamingFetchFailure
 };

package/dist/serviceModules/Rebuilder.js

@@ -5,7 +5,7 @@ import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/log
 import { delay } from "octagonal-wheels/promises";
 import { EVENT_DATABASE_REBUILT } from "../events/coreEvents.js";
 import { ServiceModuleBase } from "./ServiceModuleBase.js";
-import { fetchChangesForInitialSync } from "../pouchdb/StreamingFetch.js";
+import { fetchChangesForInitialSync, isRetryableStreamingFetchFailure } from "../pouchdb/StreamingFetch.js";
 import { getConfiguredFunctionsForEncryption } from "../pouchdb/encryption.js";
 import { AuthorizationHeaderGenerator, generateCredentialObject } from "../replication/httplib.js";
 import { sizeToHumanReadable } from "octagonal-wheels/number";
@@ -376,7 +376,7 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
         );
         break;
       } catch (ex) {
-        if (attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
+        if (!isRetryableStreamingFetchFailure(ex) || attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
         checkpoint = this.getFastFetchCheckpoint(remote);
         since = checkpoint?.sequence ?? since;
         this._log(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vrtmrz/livesync-commonlib",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Shared data, storage, and replication primitives for Self-hosted LiveSync clients.",
   "type": "module",
   "license": "MIT",