@vrtmrz/livesync-commonlib 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/dist/pouchdb/StreamingFetch.d.ts +34 -0
  2. package/dist/pouchdb/StreamingFetch.js +403 -187
  3. package/dist/serviceModules/Rebuilder.js +2 -2
  4. package/package.json +1 -1
package/dist/pouchdb/StreamingFetch.d.ts CHANGED
@@ -1,6 +1,40 @@
1
1
  import type { EntryDoc } from "../common/models/db.definition.js";
2
2
  import type { AnyEntry, EntryLeaf } from "../common/models/db.type.js";
3
3
  type DBSequence = number | string;
4
+ /**
5
+ * Identifies the boundary at which Fast Fetch stopped.
6
+ *
7
+ * - `transport`: the request or response stream was interrupted. Only failures
8
+ * explicitly created at this boundary may be retried.
9
+ * - `authentication`: CouchDB rejected the supplied credentials.
10
+ * - `protocol`: CouchDB returned an unsuccessful or structurally invalid
11
+ * response, or an unexpected exception escaped the processing pipeline.
12
+ * - `decryption`: a remote document could not be decrypted or did not produce a
13
+ * serialisable PouchDB document.
14
+ * - `storage`: a local batch or its durable checkpoint could not be written.
15
+ *
16
+ * This remains a string-literal union because the values are discriminants, not
17
+ * data to enumerate, persist, or map to user-interface labels. The constructor
18
+ * and consumers therefore receive typo checking without a runtime constants
19
+ * object becoming part of the package API.
20
+ */
21
+ export type StreamingFetchFailureStage = "transport" | "authentication" | "protocol" | "decryption" | "storage";
22
+ /**
23
+ * A classified Fast Fetch failure. Retryability is explicit rather than inferred
24
+ * from the stage so that future exceptions can be narrowed without broadening all
25
+ * failures at that boundary.
26
+ */
27
+ export declare class StreamingFetchFailure extends Error {
28
+ readonly stage: StreamingFetchFailureStage;
29
+ readonly retryable: boolean;
30
+ readonly name = "StreamingFetchFailure";
31
+ constructor(stage: StreamingFetchFailureStage, message: string, retryable: boolean, options?: {
32
+ status?: number;
33
+ cause?: unknown;
34
+ });
35
+ readonly status?: number;
36
+ }
37
+ export declare function isRetryableStreamingFetchFailure(error: unknown): error is StreamingFetchFailure;
4
38
  export type FetchChangesForInitialSyncProgress = {
5
39
  totalFetched: number;
6
40
  totalValidFetched: number;
package/dist/pouchdb/StreamingFetch.js CHANGED
@@ -1,58 +1,173 @@
1
1
  import { _fetch } from "../common/coreEnvFunctions.js";
2
2
  import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
3
- function generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint) {
3
+ const FAST_FETCH_CHANGES_PAGE_LIMIT = 1e4;
4
+ class StreamingFetchFailure extends Error {
5
+ constructor(stage, message, retryable, options) {
6
+ super(message, options?.cause === void 0 ? void 0 : { cause: options.cause });
7
+ this.stage = stage;
8
+ this.retryable = retryable;
9
+ this.status = options?.status;
10
+ }
11
+ stage;
12
+ retryable;
13
+ name = "StreamingFetchFailure";
14
+ status;
15
+ }
16
+ function isRetryableStreamingFetchFailure(error) {
17
+ return error instanceof StreamingFetchFailure && error.retryable;
18
+ }
19
+ function errorMessage(error) {
20
+ if (error instanceof Error && error.message) return error.message;
21
+ return String(error);
22
+ }
23
+ function asStreamingFetchFailure(error) {
24
+ if (error instanceof StreamingFetchFailure) return error;
25
+ return new StreamingFetchFailure(
26
+ "protocol",
27
+ `Fast Fetch encountered an unexpected processing failure: ${errorMessage(error)}`,
28
+ false,
29
+ { cause: error }
30
+ );
31
+ }
32
+ function transportFailure(operation, error) {
33
+ return new StreamingFetchFailure("transport", `Fast Fetch could not ${operation}: ${errorMessage(error)}`, true, {
34
+ cause: error
35
+ });
36
+ }
37
+ function responseFailure(response, operation) {
38
+ const status = response.status;
39
+ if (status === 401 || status === 403) {
40
+ return new StreamingFetchFailure(
41
+ "authentication",
42
+ `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
43
+ false,
44
+ { status }
45
+ );
46
+ }
47
+ const retryable = status === 408 || status === 429 || status >= 500;
48
+ return new StreamingFetchFailure(
49
+ retryable ? "transport" : "protocol",
50
+ `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
51
+ retryable,
52
+ { status }
53
+ );
54
+ }
55
+ function ensureResponseOK(response, operation) {
56
+ if (!response.ok) throw responseFailure(response, operation);
57
+ }
58
+ async function fetchResponse(url, init, operation) {
59
+ let response;
60
+ try {
61
+ response = await _fetch(url, init);
62
+ } catch (error) {
63
+ throw transportFailure(operation, error);
64
+ }
65
+ ensureResponseOK(response, operation);
66
+ return response;
67
+ }
68
+ async function readResponseText(response, operation) {
69
+ try {
70
+ return await response.text();
71
+ } catch (error) {
72
+ throw transportFailure(operation, error);
73
+ }
74
+ }
75
+ async function saveCheckpoint(onCheckpoint, sequence) {
76
+ try {
77
+ await onCheckpoint?.(sequence);
78
+ } catch (error) {
79
+ throw new StreamingFetchFailure(
80
+ "storage",
81
+ `Fast Fetch could not save its checkpoint: ${errorMessage(error)}`,
82
+ false,
83
+ { cause: error }
84
+ );
85
+ }
86
+ }
87
+ function generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint) {
4
88
  let batchBuffer = [];
5
89
  let currentBatchSizeBytes = 0;
6
90
  let batchLastSequence;
7
91
  const BATCH_ITEM_LIMIT = 100;
8
92
  const BATCH_SIZE_LIMIT = 2 * 1024 * 1024;
9
- let flushPromise = null;
10
- const flushToDB = async () => {
93
+ const flush = async () => {
11
94
  if (batchBuffer.length === 0) return;
12
- if (flushPromise) {
13
- await flushPromise;
14
- }
95
+ const documents = batchBuffer;
96
+ const checkpoint = batchLastSequence;
97
+ let results;
15
98
  try {
16
- flushPromise = downloadToDB.bulkDocs(batchBuffer, { new_edits: false });
17
- await flushPromise;
18
- if (batchLastSequence !== void 0) {
19
- await onCheckpoint?.(batchLastSequence);
20
- }
99
+ results = await downloadToDB.bulkDocs(documents, { new_edits: false });
21
100
  } catch (error) {
22
- Logger("Error bulk writing to PouchDB:", LOG_LEVEL_VERBOSE);
23
- Logger(error, LOG_LEVEL_VERBOSE);
24
- throw error;
25
- } finally {
26
- batchBuffer = [];
27
- currentBatchSizeBytes = 0;
28
- batchLastSequence = void 0;
29
- flushPromise = null;
101
+ throw new StreamingFetchFailure(
102
+ "storage",
103
+ `Fast Fetch could not write a batch to the local database: ${errorMessage(error)}`,
104
+ false,
105
+ { cause: error }
106
+ );
30
107
  }
108
+ const failedResult = results.find(
109
+ (result) => "error" in result && Boolean(result.error)
110
+ );
111
+ if (failedResult) {
112
+ const detail = failedResult.message || failedResult.name || "the local database rejected a document";
113
+ throw new StreamingFetchFailure(
114
+ "storage",
115
+ `Fast Fetch could not write a batch to the local database: ${detail}`,
116
+ false,
117
+ { cause: failedResult }
118
+ );
119
+ }
120
+ if (checkpoint !== void 0) {
121
+ await saveCheckpoint(onCheckpoint, checkpoint);
122
+ }
123
+ batchBuffer = [];
124
+ currentBatchSizeBytes = 0;
125
+ batchLastSequence = void 0;
31
126
  };
32
- return new WritableStream({
33
- async write(chunk) {
127
+ return {
128
+ async write(doc, sequence) {
129
+ let decryptedDoc;
34
130
  try {
35
- const decryptedDoc = await decryptFunction(chunk.doc);
36
- batchBuffer.push(decryptedDoc);
37
- currentBatchSizeBytes += JSON.stringify(decryptedDoc).length;
38
- batchLastSequence = chunk.seq;
39
- if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
40
- await flushToDB();
41
- }
131
+ decryptedDoc = await decryptFunction(doc);
42
132
  } catch (error) {
43
- Logger("Error processing document stream:", LOG_LEVEL_VERBOSE);
44
- throw error;
133
+ throw new StreamingFetchFailure(
134
+ "decryption",
135
+ `Fast Fetch could not decrypt a document: ${errorMessage(error)}`,
136
+ false,
137
+ { cause: error }
138
+ );
139
+ }
140
+ let serialisedDoc;
141
+ try {
142
+ const serialised = JSON.stringify(decryptedDoc);
143
+ if (serialised === void 0) throw new Error("the decrypted value is not a document");
144
+ serialisedDoc = serialised;
145
+ } catch (error) {
146
+ throw new StreamingFetchFailure(
147
+ "decryption",
148
+ `Fast Fetch produced an invalid decrypted document: ${errorMessage(error)}`,
149
+ false,
150
+ { cause: error }
151
+ );
152
+ }
153
+ batchBuffer.push(decryptedDoc);
154
+ currentBatchSizeBytes += serialisedDoc.length;
155
+ batchLastSequence = sequence;
156
+ if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
157
+ await flush();
45
158
  }
46
159
  },
47
- async close() {
48
- await flushToDB();
160
+ async flushThrough(sequence) {
161
+ await flush();
162
+ await saveCheckpoint(onCheckpoint, sequence);
49
163
  },
50
- abort(reason) {
51
- Logger(`Stream aborted: ${reason}`, LOG_LEVEL_VERBOSE);
164
+ flush,
165
+ abort() {
52
166
  batchBuffer = [];
53
167
  currentBatchSizeBytes = 0;
168
+ batchLastSequence = void 0;
54
169
  }
55
- });
170
+ };
56
171
  }
57
172
  function setParamsToURL(url, params) {
58
173
  for (const [key, value] of Object.entries(params)) {
@@ -60,21 +175,61 @@ function setParamsToURL(url, params) {
60
175
  }
61
176
  return url;
62
177
  }
63
- function reachedTargetSequence(seq, targetSeq) {
64
- if (seq === void 0 || targetSeq === void 0) return false;
65
- const seqStr = seq.toString();
66
- const targetSeqStr = targetSeq.toString();
67
- if (seqStr === targetSeqStr) return true;
68
- const seqNum = parseInt(seqStr.split("-")[0], 10);
69
- const targetSeqNum = parseInt(targetSeqStr.split("-")[0], 10);
70
- if (!isNaN(seqNum) && !isNaN(targetSeqNum)) {
71
- return seqNum >= targetSeqNum;
178
+ function parseStatusSource(source) {
179
+ const trimmed = source.trim();
180
+ if (!trimmed) {
181
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received no changes status from CouchDB.", false);
72
182
  }
73
- return false;
183
+ const candidates = [trimmed, ...trimmed.split("\n").reverse().filter(Boolean)];
184
+ for (const candidate of candidates) {
185
+ try {
186
+ const parsed = JSON.parse(candidate);
187
+ if (parsed && typeof parsed === "object") return parsed;
188
+ } catch {
189
+ }
190
+ }
191
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes status from CouchDB.", false);
192
+ }
193
+ function parseChangesFeedLine(line) {
194
+ let parsed;
195
+ try {
196
+ parsed = JSON.parse(line);
197
+ } catch (error) {
198
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received a malformed changes-feed row.", false, {
199
+ cause: error
200
+ });
201
+ }
202
+ if (!parsed || typeof parsed !== "object") {
203
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed line.", false);
204
+ }
205
+ if (!("seq" in parsed) && "last_seq" in parsed) {
206
+ const lastSequence = parsed.last_seq;
207
+ if (lastSequence === void 0) {
208
+ throw new StreamingFetchFailure(
209
+ "protocol",
210
+ "Fast Fetch received a changes-feed terminator without a sequence.",
211
+ false
212
+ );
213
+ }
214
+ return { type: "terminator", lastSequence };
215
+ }
216
+ if (!("seq" in parsed)) {
217
+ throw new StreamingFetchFailure(
218
+ "protocol",
219
+ "Fast Fetch received a changes-feed row without a sequence.",
220
+ false
221
+ );
222
+ }
223
+ const change = parsed;
224
+ if (change.seq === void 0 || change.doc !== void 0 && (!change.doc || typeof change.doc !== "object")) {
225
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed row.", false);
226
+ }
227
+ return { type: "change", change };
74
228
  }
75
229
  async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint) {
76
230
  let totalFetched = 0;
77
231
  let totalValidFetched = 0;
232
+ let totalBytes = 0;
78
233
  const changesBaseParams = {
79
234
  feed: "continuous",
80
235
  include_docs: "true",
@@ -88,176 +243,237 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
88
243
  Accept: "application/json",
89
244
  Authorization: authHeader
90
245
  };
91
- let targetSeq = void 0;
92
- try {
93
- const dbInfoRes = await _fetch(remoteDbUrl, {
94
- headers: fetchHeaders
95
- });
96
- if (dbInfoRes.ok) {
97
- const dbInfo = await dbInfoRes.json();
98
- targetSeq = dbInfo.update_seq;
99
- }
100
- } catch (e) {
101
- Logger("Failed to fetch database info for target sequence:", LOG_LEVEL_VERBOSE);
102
- Logger(e, LOG_LEVEL_VERBOSE);
103
- }
104
- const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
246
+ const targetURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
105
247
  ...changesBaseParams,
106
248
  feed: "normal",
107
- limit: "1"
249
+ since: "now",
250
+ limit: "1",
251
+ include_docs: "false"
108
252
  });
109
- const infoRes = await _fetch(fetchURL.toString(), {
110
- headers: fetchHeaders
111
- });
112
- const infoSource = await infoRes.text();
113
- const infoSourceTrimmed = infoSource.trim();
114
- if (!infoSourceTrimmed) {
115
- throw new Error("Failed to fetch changes from CouchDB. No data received.");
116
- }
117
- let info;
118
- if (infoSourceTrimmed.startsWith("{") && infoSourceTrimmed.endsWith("}")) {
119
- try {
120
- info = JSON.parse(infoSourceTrimmed);
121
- } catch {
122
- const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
123
- const lastLine = infoLines[infoLines.length - 1];
124
- info = JSON.parse(lastLine);
125
- }
126
- } else {
127
- const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
128
- const lastLine = infoLines[infoLines.length - 1];
129
- info = JSON.parse(lastLine);
130
- }
131
- const pendingDocs = info.pending || 0;
132
- const docsToFetch = pendingDocs + 1;
133
- if (targetSeq === void 0) {
134
- targetSeq = info.last_seq || info.update_seq || info.seq;
135
- }
136
- const finalTargetSeq = targetSeq ?? "";
137
- if (reachedTargetSequence(since, finalTargetSeq)) {
138
- Logger("Already at the target sequence. Initial data synchronisation is complete.");
139
- return;
140
- }
141
- Logger(
142
- `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Total documents to fetch: ${docsToFetch}.`
253
+ const targetResponse = await fetchResponse(
254
+ targetURL.toString(),
255
+ { headers: fetchHeaders },
256
+ "capture the changes target"
143
257
  );
144
- const controller = new AbortController();
145
- const url = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
146
- ...changesBaseParams
147
- });
148
- const response = await _fetch(url.toString(), {
149
- method: "GET",
150
- headers: fetchHeaders,
151
- signal: controller.signal
152
- });
153
- if (!response.body) {
154
- throw new Error("ReadableStream is not supported by this browser.");
258
+ const targetStatus = parseStatusSource(await readResponseText(targetResponse, "read the changes target"));
259
+ const progressTargetSeq = targetStatus.last_seq;
260
+ if (progressTargetSeq === void 0) {
261
+ throw new StreamingFetchFailure(
262
+ "protocol",
263
+ "Fast Fetch could not obtain a changes progress target from CouchDB.",
264
+ false
265
+ );
155
266
  }
156
- const sizeCaptureStream = new TransformStream({
157
- transform(chunk, controller2) {
158
- const chunkSize = chunk.length || chunk.byteLength || 0;
159
- totalBytes += chunkSize;
160
- controller2.enqueue(chunk);
161
- }
162
- });
163
- const reader = response.body.pipeThrough(sizeCaptureStream).pipeThrough(new TextDecoderStream()).getReader();
164
- const writeDocStream = generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint);
165
- const writer = writeDocStream.getWriter();
166
- let buffer = "";
267
+ const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint);
268
+ let docsToFetch = 0;
167
269
  let lastProgress = 0;
168
270
  let lastReportTime = Date.now();
169
- let totalBytes = 0;
170
- let abortedAfterTargetReached = false;
171
- const reportProgress = () => {
172
- if (totalFetched - lastProgress < 25) {
173
- if (Date.now() - lastReportTime < 2e3) {
174
- return;
175
- }
176
- }
271
+ const reportProgress = (force = false) => {
272
+ if (!force && totalFetched - lastProgress < 25 && Date.now() - lastReportTime < 2e3) return;
177
273
  lastProgress = totalFetched;
178
274
  lastReportTime = Date.now();
179
275
  onProgress?.({
180
276
  totalFetched,
181
277
  totalValidFetched,
182
- targetSeq: finalTargetSeq,
278
+ targetSeq: progressTargetSeq,
183
279
  docsToFetch,
184
280
  totalBytes
185
281
  });
186
282
  };
187
- try {
188
- while (true) {
283
+ const readAvailableChanges = async (pageSince) => {
284
+ const statusURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
285
+ ...changesBaseParams,
286
+ feed: "normal",
287
+ since: pageSince.toString(),
288
+ limit: "1",
289
+ include_docs: "false"
290
+ });
291
+ const response = await fetchResponse(
292
+ statusURL.toString(),
293
+ { method: "GET", headers: fetchHeaders },
294
+ "read changes status"
295
+ );
296
+ const status = parseStatusSource(await readResponseText(response, "read changes status"));
297
+ if (!Array.isArray(status.results)) {
298
+ throw new StreamingFetchFailure(
299
+ "protocol",
300
+ "Fast Fetch received changes status without a valid results list.",
301
+ false
302
+ );
303
+ }
304
+ const pending = status.pending;
305
+ if (typeof pending !== "number" || !Number.isSafeInteger(pending) || pending < 0) {
306
+ throw new StreamingFetchFailure(
307
+ "protocol",
308
+ "Fast Fetch received changes status without a valid pending count.",
309
+ false
310
+ );
311
+ }
312
+ const available = status.results.length + pending;
313
+ if (!Number.isSafeInteger(available)) {
314
+ throw new StreamingFetchFailure(
315
+ "protocol",
316
+ "Fast Fetch received a changes count outside the supported range.",
317
+ false
318
+ );
319
+ }
320
+ return available;
321
+ };
322
+ const fetchPage = async (pageSince, pageLimit) => {
323
+ const controller = new AbortController();
324
+ let reader;
325
+ let buffer = "";
326
+ let fetchedRows = 0;
327
+ const processLine = async (line) => {
328
+ const parsed = parseChangesFeedLine(line);
329
+ if (parsed.type === "terminator") {
330
+ if (fetchedRows === 0) {
331
+ throw new StreamingFetchFailure(
332
+ "transport",
333
+ "Fast Fetch received no rows after its status probe reported available changes.",
334
+ true
335
+ );
336
+ }
337
+ await batchWriter.flush();
338
+ await saveCheckpoint(onCheckpoint, parsed.lastSequence);
339
+ reportProgress(true);
340
+ return parsed.lastSequence;
341
+ }
342
+ if (fetchedRows >= pageLimit) {
343
+ throw new StreamingFetchFailure(
344
+ "protocol",
345
+ `Fast Fetch received more than its ${pageLimit}-row page limit.`,
346
+ false
347
+ );
348
+ }
349
+ const change = parsed.change;
350
+ fetchedRows++;
351
+ totalFetched++;
352
+ if (change.doc) {
353
+ await batchWriter.write(change.doc, change.seq);
354
+ totalValidFetched++;
355
+ } else {
356
+ await batchWriter.flushThrough(change.seq);
357
+ }
189
358
  reportProgress();
190
- const { value, done } = await reader.read();
191
- if (value) {
192
- totalBytes += value.length;
359
+ return void 0;
360
+ };
361
+ try {
362
+ const changesURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
363
+ ...changesBaseParams,
364
+ since: pageSince.toString(),
365
+ limit: pageLimit.toString()
366
+ });
367
+ const response = await fetchResponse(
368
+ changesURL.toString(),
369
+ {
370
+ method: "GET",
371
+ headers: fetchHeaders,
372
+ signal: controller.signal
373
+ },
374
+ "open the changes feed"
375
+ );
376
+ if (!response.body) {
377
+ throw new StreamingFetchFailure(
378
+ "protocol",
379
+ "Fast Fetch could not read the CouchDB response stream.",
380
+ false
381
+ );
193
382
  }
194
- if (done) {
195
- if (buffer.trim()) {
196
- try {
197
- totalFetched++;
198
- const parsed = JSON.parse(buffer);
199
- if (parsed.doc) {
200
- await writer.write({ doc: parsed.doc, seq: parsed.seq });
201
- totalValidFetched++;
202
- } else {
203
- await onCheckpoint?.(parsed.seq);
204
- }
205
- reportProgress();
206
- } catch (e) {
207
- Logger(`Failed to parse the final line: ${buffer}. Skipping it.`);
208
- Logger(e, LOG_LEVEL_VERBOSE);
383
+ const decoder = new TextDecoder();
384
+ const byteCountingDecoderStream = new TransformStream({
385
+ transform(chunk, streamController) {
386
+ totalBytes += chunk.byteLength;
387
+ const decoded = decoder.decode(chunk, { stream: true });
388
+ if (decoded) streamController.enqueue(decoded);
389
+ },
390
+ flush(streamController) {
391
+ const decoded = decoder.decode();
392
+ if (decoded) streamController.enqueue(decoded);
393
+ }
394
+ });
395
+ reader = response.body.pipeThrough(byteCountingDecoderStream).getReader();
396
+ while (true) {
397
+ reportProgress();
398
+ let readResult;
399
+ try {
400
+ readResult = await reader.read();
401
+ } catch (error) {
402
+ throw transportFailure("read the changes feed", error);
403
+ }
404
+ if (readResult.done) {
405
+ if (buffer.trim()) {
406
+ const lastSequence = await processLine(buffer);
407
+ if (lastSequence !== void 0) return lastSequence;
209
408
  }
409
+ await batchWriter.flush();
410
+ throw new StreamingFetchFailure(
411
+ "transport",
412
+ "The bounded CouchDB changes feed ended without a last_seq terminator.",
413
+ true
414
+ );
415
+ }
416
+ buffer += readResult.value;
417
+ const lines = buffer.split("\n");
418
+ buffer = lines.pop() || "";
419
+ for (const line of lines) {
420
+ if (!line.trim()) continue;
421
+ const lastSequence = await processLine(line);
422
+ if (lastSequence !== void 0) return lastSequence;
210
423
  }
211
- await writer.close();
212
- break;
213
424
  }
214
- buffer += value;
215
- const lines = buffer.split("\n");
216
- buffer = lines.pop() || "";
217
- for (const line of lines) {
218
- if (!line.trim()) continue;
425
+ } finally {
426
+ controller.abort();
427
+ if (reader) {
219
428
  try {
220
- const parsed = JSON.parse(line);
221
- totalFetched++;
222
- if (parsed.doc) {
223
- await writer.write({ doc: parsed.doc, seq: parsed.seq });
224
- totalValidFetched++;
225
- } else {
226
- await onCheckpoint?.(parsed.seq);
227
- }
228
- reportProgress();
229
- if (totalFetched >= docsToFetch || reachedTargetSequence(parsed.seq, finalTargetSeq)) {
230
- Logger(
231
- `All documents fetched. Stopping the stream and writing remaining documents to PouchDB...`
232
- );
233
- await writer.close();
234
- abortedAfterTargetReached = true;
235
- controller.abort();
236
- reportProgress();
237
- return;
238
- }
239
- } catch (e) {
240
- Logger(`JSON parsing failed. Skipping line: ${line}`, LOG_LEVEL_VERBOSE);
241
- Logger(e, LOG_LEVEL_VERBOSE);
429
+ await reader.cancel();
430
+ } catch {
242
431
  }
432
+ reader.releaseLock();
243
433
  }
244
434
  }
245
- Logger("Initial data synchronisation via stream has completed.");
246
- reportProgress();
435
+ };
436
+ try {
437
+ let pageSince = since;
438
+ let started = false;
439
+ while (true) {
440
+ const available = await readAvailableChanges(pageSince);
441
+ docsToFetch = Math.max(docsToFetch, totalFetched + available);
442
+ if (available === 0) break;
443
+ if (!started) {
444
+ started = true;
445
+ Logger(
446
+ `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${progressTargetSeq}, Documents to fetch: ${docsToFetch}.`
447
+ );
448
+ }
449
+ const pageLimit = Math.min(FAST_FETCH_CHANGES_PAGE_LIMIT, available);
450
+ const pageTerminator = await fetchPage(pageSince, pageLimit);
451
+ pageSince = pageTerminator;
452
+ }
453
+ if (started) {
454
+ Logger("Fast Fetch is caught up and durable in the local database.");
455
+ reportProgress(true);
456
+ } else {
457
+ Logger("No changes remain for Fast Fetch.");
458
+ }
247
459
  } catch (error) {
248
- if (abortedAfterTargetReached && error instanceof DOMException && error.name === "AbortError") {
249
- Logger(
250
- "Stream has been aborted as the target sequence has been reached. Finalising the synchronising process..."
251
- );
252
- return;
460
+ const failure = asStreamingFetchFailure(error);
461
+ if (failure.stage !== "storage") {
462
+ try {
463
+ await batchWriter.flush();
464
+ } catch (flushError) {
465
+ batchWriter.abort();
466
+ throw asStreamingFetchFailure(flushError);
467
+ }
253
468
  }
254
- Logger("An error occurred during synchronisation:", LOG_LEVEL_VERBOSE);
255
- Logger(error, LOG_LEVEL_VERBOSE);
256
- throw error;
257
- } finally {
258
- reader.releaseLock();
469
+ batchWriter.abort();
470
+ Logger(`Fast Fetch failed during ${failure.stage}.`, LOG_LEVEL_VERBOSE);
471
+ Logger(failure, LOG_LEVEL_VERBOSE);
472
+ throw failure;
259
473
  }
260
474
  }
261
475
  export {
262
- fetchChangesForInitialSync
476
+ StreamingFetchFailure,
477
+ fetchChangesForInitialSync,
478
+ isRetryableStreamingFetchFailure
263
479
  };
package/dist/serviceModules/Rebuilder.js CHANGED
@@ -5,7 +5,7 @@ import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/log
5
5
  import { delay } from "octagonal-wheels/promises";
6
6
  import { EVENT_DATABASE_REBUILT } from "../events/coreEvents.js";
7
7
  import { ServiceModuleBase } from "./ServiceModuleBase.js";
8
- import { fetchChangesForInitialSync } from "../pouchdb/StreamingFetch.js";
8
+ import { fetchChangesForInitialSync, isRetryableStreamingFetchFailure } from "../pouchdb/StreamingFetch.js";
9
9
  import { getConfiguredFunctionsForEncryption } from "../pouchdb/encryption.js";
10
10
  import { AuthorizationHeaderGenerator, generateCredentialObject } from "../replication/httplib.js";
11
11
  import { sizeToHumanReadable } from "octagonal-wheels/number";
@@ -376,7 +376,7 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
376
376
  );
377
377
  break;
378
378
  } catch (ex) {
379
- if (attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
379
+ if (!isRetryableStreamingFetchFailure(ex) || attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
380
380
  checkpoint = this.getFastFetchCheckpoint(remote);
381
381
  since = checkpoint?.sequence ?? since;
382
382
  this._log(
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vrtmrz/livesync-commonlib",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "description": "Shared data, storage, and replication primitives for Self-hosted LiveSync clients.",
5
5
  "type": "module",
6
6
  "license": "MIT",