@vrtmrz/livesync-commonlib 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -190,4 +190,28 @@ export declare const TweakValuesTemplate: {
190
190
  };
191
191
  export type TweakValues = Partial<typeof TweakValuesTemplate>;
192
192
  export declare const DEVICE_ID_PREFERRED = "PREFERRED";
193
+ export declare const RemotePreferredTweakStatuses: {
194
+ readonly AVAILABLE: "available";
195
+ readonly NOT_CONFIGURED: "not-configured";
196
+ readonly UNAVAILABLE: "unavailable";
197
+ readonly UNSUPPORTED: "unsupported";
198
+ };
199
+ export type RemotePreferredTweakStatus = (typeof RemotePreferredTweakStatuses)[keyof typeof RemotePreferredTweakStatuses];
200
+ export declare const RemotePreferredTweakNotConfiguredReasons: {
201
+ readonly MILESTONE_MISSING: "milestone-missing";
202
+ readonly PREFERRED_VALUES_MISSING: "preferred-values-missing";
203
+ };
204
+ export type RemotePreferredTweakNotConfiguredReason = (typeof RemotePreferredTweakNotConfiguredReasons)[keyof typeof RemotePreferredTweakNotConfiguredReasons];
205
+ export type RemotePreferredTweakResult = {
206
+ status: typeof RemotePreferredTweakStatuses.AVAILABLE;
207
+ values: TweakValues;
208
+ } | {
209
+ status: typeof RemotePreferredTweakStatuses.NOT_CONFIGURED;
210
+ reason: RemotePreferredTweakNotConfiguredReason;
211
+ } | {
212
+ status: typeof RemotePreferredTweakStatuses.UNAVAILABLE;
213
+ error: unknown;
214
+ } | {
215
+ status: typeof RemotePreferredTweakStatuses.UNSUPPORTED;
216
+ };
193
217
  export {};
@@ -60,11 +60,23 @@ const TweakValuesTemplate = {
60
60
  tweakModified: 0
61
61
  };
62
62
  const DEVICE_ID_PREFERRED = "PREFERRED";
63
+ const RemotePreferredTweakStatuses = {
64
+ AVAILABLE: "available",
65
+ NOT_CONFIGURED: "not-configured",
66
+ UNAVAILABLE: "unavailable",
67
+ UNSUPPORTED: "unsupported"
68
+ };
69
+ const RemotePreferredTweakNotConfiguredReasons = {
70
+ MILESTONE_MISSING: "milestone-missing",
71
+ PREFERRED_VALUES_MISSING: "preferred-values-missing"
72
+ };
63
73
  export {
64
74
  CompatibleButLossyChanges,
65
75
  DEVICE_ID_PREFERRED,
66
76
  IncompatibleChanges,
67
77
  IncompatibleChangesInSpecificPattern,
78
+ RemotePreferredTweakNotConfiguredReasons,
79
+ RemotePreferredTweakStatuses,
68
80
  TweakValuesDefault,
69
81
  TweakValuesRecommendedTemplate,
70
82
  TweakValuesShouldMatchedTemplate,
@@ -21,7 +21,7 @@ import { SETTING_KEY_P2P_DEVICE_NAME, configURIBase, configURIBaseQR, SuffixData
21
21
  import { configurationNames, LEVEL_ADVANCED, LEVEL_POWER_USER, LEVEL_EDGE_CASE, type ConfigLevel, type ConfigurationItem, statusDisplay, confName, confDesc } from "./models/shared.definition.configNames.js";
22
22
  import type { CustomRegExpSource, CustomRegExpSourceList, ParsedCustomRegExp, Prettify } from "./models/shared.type.util.js";
23
23
  import { ProtocolVersions, type ProtocolVersion, DOCID_SYNC_PARAMETERS, DOCID_JOURNAL_SYNC_PARAMETERS, type SyncParameters, DEFAULT_SYNC_PARAMETERS } from "./models/sync.definition.js";
24
- import { TweakValuesShouldMatchedTemplate, IncompatibleChanges, CompatibleButLossyChanges, IncompatibleChangesInSpecificPattern, TweakValuesRecommendedTemplate, TweakValuesDefault, TweakValuesTemplate, type TweakValues, DEVICE_ID_PREFERRED } from "./models/tweak.definition.js";
24
+ import { TweakValuesShouldMatchedTemplate, IncompatibleChanges, CompatibleButLossyChanges, IncompatibleChangesInSpecificPattern, TweakValuesRecommendedTemplate, TweakValuesDefault, TweakValuesTemplate, type TweakValues, DEVICE_ID_PREFERRED, RemotePreferredTweakStatuses, type RemotePreferredTweakStatus, RemotePreferredTweakNotConfiguredReasons, type RemotePreferredTweakNotConfiguredReason, type RemotePreferredTweakResult } from "./models/tweak.definition.js";
25
25
  import type { diff_result_leaf, dmp_result, diff_result, DIFF_CHECK_RESULT_AUTO, diff_check_result } from "./models/diff.definition.js";
26
26
  import { PREFIXMD_LOGFILE, PREFIXMD_LOGFILE_UC, FlagFilesOriginal, FlagFilesHumanReadable, FLAGMD_REDFLAG, FLAGMD_REDFLAG2, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3, FLAGMD_REDFLAG3_HR } from "./models/redflag.const.js";
27
27
  import { DatabaseConnectingStatuses, type DatabaseConnectingStatus } from "./models/shared.definition.js";
@@ -68,6 +68,11 @@ export { confDesc };
68
68
  export { TweakValuesTemplate };
69
69
  export type { TweakValues };
70
70
  export { DEVICE_ID_PREFERRED };
71
+ export { RemotePreferredTweakStatuses };
72
+ export type { RemotePreferredTweakStatus };
73
+ export { RemotePreferredTweakNotConfiguredReasons };
74
+ export type { RemotePreferredTweakNotConfiguredReason };
75
+ export type { RemotePreferredTweakResult };
71
76
  export type { NodeKey };
72
77
  export type { DeviceInfo };
73
78
  export type { NodeData };
@@ -110,7 +110,9 @@ import {
110
110
  TweakValuesRecommendedTemplate,
111
111
  TweakValuesDefault,
112
112
  TweakValuesTemplate,
113
- DEVICE_ID_PREFERRED
113
+ DEVICE_ID_PREFERRED,
114
+ RemotePreferredTweakStatuses,
115
+ RemotePreferredTweakNotConfiguredReasons
114
116
  } from "./models/tweak.definition.js";
115
117
  import {
116
118
  PREFIXMD_LOGFILE,
@@ -197,6 +199,8 @@ export {
197
199
  REPLICATION_BUSY_TIMEOUT,
198
200
  RESULT_NOT_FOUND,
199
201
  RESULT_TIMED_OUT,
202
+ RemotePreferredTweakNotConfiguredReasons,
203
+ RemotePreferredTweakStatuses,
200
204
  RemoteTypes,
201
205
  SALT_OF_ID,
202
206
  SALT_OF_PASSPHRASE,
@@ -1,6 +1,40 @@
1
1
  import type { EntryDoc } from "../common/models/db.definition.js";
2
2
  import type { AnyEntry, EntryLeaf } from "../common/models/db.type.js";
3
3
  type DBSequence = number | string;
4
+ /**
5
+ * Identifies the boundary at which Fast Fetch stopped.
6
+ *
7
+ * - `transport`: the request or response stream was interrupted. Only failures
8
+ * explicitly created at this boundary may be retried.
9
+ * - `authentication`: CouchDB rejected the supplied credentials.
10
+ * - `protocol`: CouchDB returned an unsuccessful or structurally invalid
11
+ * response, or an unexpected exception escaped the processing pipeline.
12
+ * - `decryption`: a remote document could not be decrypted or did not produce a
13
+ * serialisable PouchDB document.
14
+ * - `storage`: a local batch or its durable checkpoint could not be written.
15
+ *
16
+ * This remains a string-literal union because the values are discriminants, not
17
+ * data to enumerate, persist, or map to user-interface labels. The constructor
18
+ * and consumers therefore receive typo checking without a runtime constants
19
+ * object becoming part of the package API.
20
+ */
21
+ export type StreamingFetchFailureStage = "transport" | "authentication" | "protocol" | "decryption" | "storage";
22
+ /**
23
+ * A classified Fast Fetch failure. Retryability is explicit rather than inferred
24
+ * from the stage so that future exceptions can be narrowed without broadening all
25
+ * failures at that boundary.
26
+ */
27
+ export declare class StreamingFetchFailure extends Error {
28
+ readonly stage: StreamingFetchFailureStage;
29
+ readonly retryable: boolean;
30
+ readonly name = "StreamingFetchFailure";
31
+ constructor(stage: StreamingFetchFailureStage, message: string, retryable: boolean, options?: {
32
+ status?: number;
33
+ cause?: unknown;
34
+ });
35
+ readonly status?: number;
36
+ }
37
+ export declare function isRetryableStreamingFetchFailure(error: unknown): error is StreamingFetchFailure;
4
38
  export type FetchChangesForInitialSyncProgress = {
5
39
  totalFetched: number;
6
40
  totalValidFetched: number;
@@ -1,58 +1,172 @@
1
1
  import { _fetch } from "../common/coreEnvFunctions.js";
2
2
  import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
3
- function generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint) {
3
+ class StreamingFetchFailure extends Error {
4
+ constructor(stage, message, retryable, options) {
5
+ super(message, options?.cause === void 0 ? void 0 : { cause: options.cause });
6
+ this.stage = stage;
7
+ this.retryable = retryable;
8
+ this.status = options?.status;
9
+ }
10
+ stage;
11
+ retryable;
12
+ name = "StreamingFetchFailure";
13
+ status;
14
+ }
15
+ function isRetryableStreamingFetchFailure(error) {
16
+ return error instanceof StreamingFetchFailure && error.retryable;
17
+ }
18
+ function errorMessage(error) {
19
+ if (error instanceof Error && error.message) return error.message;
20
+ return String(error);
21
+ }
22
+ function asStreamingFetchFailure(error) {
23
+ if (error instanceof StreamingFetchFailure) return error;
24
+ return new StreamingFetchFailure(
25
+ "protocol",
26
+ `Fast Fetch encountered an unexpected processing failure: ${errorMessage(error)}`,
27
+ false,
28
+ { cause: error }
29
+ );
30
+ }
31
+ function transportFailure(operation, error) {
32
+ return new StreamingFetchFailure("transport", `Fast Fetch could not ${operation}: ${errorMessage(error)}`, true, {
33
+ cause: error
34
+ });
35
+ }
36
+ function responseFailure(response, operation) {
37
+ const status = response.status;
38
+ if (status === 401 || status === 403) {
39
+ return new StreamingFetchFailure(
40
+ "authentication",
41
+ `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
42
+ false,
43
+ { status }
44
+ );
45
+ }
46
+ const retryable = status === 408 || status === 429 || status >= 500;
47
+ return new StreamingFetchFailure(
48
+ retryable ? "transport" : "protocol",
49
+ `Fast Fetch could not ${operation}: CouchDB returned HTTP ${status}.`,
50
+ retryable,
51
+ { status }
52
+ );
53
+ }
54
+ function ensureResponseOK(response, operation) {
55
+ if (!response.ok) throw responseFailure(response, operation);
56
+ }
57
+ async function fetchResponse(url, init, operation) {
58
+ let response;
59
+ try {
60
+ response = await _fetch(url, init);
61
+ } catch (error) {
62
+ throw transportFailure(operation, error);
63
+ }
64
+ ensureResponseOK(response, operation);
65
+ return response;
66
+ }
67
+ async function readResponseText(response, operation) {
68
+ try {
69
+ return await response.text();
70
+ } catch (error) {
71
+ throw transportFailure(operation, error);
72
+ }
73
+ }
74
+ async function saveCheckpoint(onCheckpoint, sequence) {
75
+ try {
76
+ await onCheckpoint?.(sequence);
77
+ } catch (error) {
78
+ throw new StreamingFetchFailure(
79
+ "storage",
80
+ `Fast Fetch could not save its checkpoint: ${errorMessage(error)}`,
81
+ false,
82
+ { cause: error }
83
+ );
84
+ }
85
+ }
86
+ function generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint) {
4
87
  let batchBuffer = [];
5
88
  let currentBatchSizeBytes = 0;
6
89
  let batchLastSequence;
7
90
  const BATCH_ITEM_LIMIT = 100;
8
91
  const BATCH_SIZE_LIMIT = 2 * 1024 * 1024;
9
- let flushPromise = null;
10
- const flushToDB = async () => {
92
+ const flush = async () => {
11
93
  if (batchBuffer.length === 0) return;
12
- if (flushPromise) {
13
- await flushPromise;
14
- }
94
+ const documents = batchBuffer;
95
+ const checkpoint = batchLastSequence;
96
+ let results;
15
97
  try {
16
- flushPromise = downloadToDB.bulkDocs(batchBuffer, { new_edits: false });
17
- await flushPromise;
18
- if (batchLastSequence !== void 0) {
19
- await onCheckpoint?.(batchLastSequence);
20
- }
98
+ results = await downloadToDB.bulkDocs(documents, { new_edits: false });
21
99
  } catch (error) {
22
- Logger("Error bulk writing to PouchDB:", LOG_LEVEL_VERBOSE);
23
- Logger(error, LOG_LEVEL_VERBOSE);
24
- throw error;
25
- } finally {
26
- batchBuffer = [];
27
- currentBatchSizeBytes = 0;
28
- batchLastSequence = void 0;
29
- flushPromise = null;
100
+ throw new StreamingFetchFailure(
101
+ "storage",
102
+ `Fast Fetch could not write a batch to the local database: ${errorMessage(error)}`,
103
+ false,
104
+ { cause: error }
105
+ );
106
+ }
107
+ const failedResult = results.find(
108
+ (result) => "error" in result && Boolean(result.error)
109
+ );
110
+ if (failedResult) {
111
+ const detail = failedResult.message || failedResult.name || "the local database rejected a document";
112
+ throw new StreamingFetchFailure(
113
+ "storage",
114
+ `Fast Fetch could not write a batch to the local database: ${detail}`,
115
+ false,
116
+ { cause: failedResult }
117
+ );
30
118
  }
119
+ if (checkpoint !== void 0) {
120
+ await saveCheckpoint(onCheckpoint, checkpoint);
121
+ }
122
+ batchBuffer = [];
123
+ currentBatchSizeBytes = 0;
124
+ batchLastSequence = void 0;
31
125
  };
32
- return new WritableStream({
33
- async write(chunk) {
126
+ return {
127
+ async write(doc, sequence) {
128
+ let decryptedDoc;
129
+ try {
130
+ decryptedDoc = await decryptFunction(doc);
131
+ } catch (error) {
132
+ throw new StreamingFetchFailure(
133
+ "decryption",
134
+ `Fast Fetch could not decrypt a document: ${errorMessage(error)}`,
135
+ false,
136
+ { cause: error }
137
+ );
138
+ }
139
+ let serialisedDoc;
34
140
  try {
35
- const decryptedDoc = await decryptFunction(chunk.doc);
36
- batchBuffer.push(decryptedDoc);
37
- currentBatchSizeBytes += JSON.stringify(decryptedDoc).length;
38
- batchLastSequence = chunk.seq;
39
- if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
40
- await flushToDB();
41
- }
141
+ const serialised = JSON.stringify(decryptedDoc);
142
+ if (serialised === void 0) throw new Error("the decrypted value is not a document");
143
+ serialisedDoc = serialised;
42
144
  } catch (error) {
43
- Logger("Error processing document stream:", LOG_LEVEL_VERBOSE);
44
- throw error;
145
+ throw new StreamingFetchFailure(
146
+ "decryption",
147
+ `Fast Fetch produced an invalid decrypted document: ${errorMessage(error)}`,
148
+ false,
149
+ { cause: error }
150
+ );
151
+ }
152
+ batchBuffer.push(decryptedDoc);
153
+ currentBatchSizeBytes += serialisedDoc.length;
154
+ batchLastSequence = sequence;
155
+ if (batchBuffer.length >= BATCH_ITEM_LIMIT || currentBatchSizeBytes >= BATCH_SIZE_LIMIT) {
156
+ await flush();
45
157
  }
46
158
  },
47
- async close() {
48
- await flushToDB();
159
+ async flushThrough(sequence) {
160
+ await flush();
161
+ await saveCheckpoint(onCheckpoint, sequence);
49
162
  },
50
- abort(reason) {
51
- Logger(`Stream aborted: ${reason}`, LOG_LEVEL_VERBOSE);
163
+ flush,
164
+ abort() {
52
165
  batchBuffer = [];
53
166
  currentBatchSizeBytes = 0;
167
+ batchLastSequence = void 0;
54
168
  }
55
- });
169
+ };
56
170
  }
57
171
  function setParamsToURL(url, params) {
58
172
  for (const [key, value] of Object.entries(params)) {
@@ -60,21 +174,51 @@ function setParamsToURL(url, params) {
60
174
  }
61
175
  return url;
62
176
  }
63
- function reachedTargetSequence(seq, targetSeq) {
64
- if (seq === void 0 || targetSeq === void 0) return false;
65
- const seqStr = seq.toString();
66
- const targetSeqStr = targetSeq.toString();
67
- if (seqStr === targetSeqStr) return true;
68
- const seqNum = parseInt(seqStr.split("-")[0], 10);
69
- const targetSeqNum = parseInt(targetSeqStr.split("-")[0], 10);
70
- if (!isNaN(seqNum) && !isNaN(targetSeqNum)) {
71
- return seqNum >= targetSeqNum;
177
+ function isTargetSequence(sequence, targetSequence) {
178
+ if (sequence === void 0 || targetSequence === void 0) return false;
179
+ return sequence.toString() === targetSequence.toString();
180
+ }
181
+ function parseStatusSource(source) {
182
+ const trimmed = source.trim();
183
+ if (!trimmed) {
184
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received no changes status from CouchDB.", false);
185
+ }
186
+ const candidates = [trimmed, ...trimmed.split("\n").reverse().filter(Boolean)];
187
+ for (const candidate of candidates) {
188
+ try {
189
+ const parsed = JSON.parse(candidate);
190
+ if (parsed && typeof parsed === "object") return parsed;
191
+ } catch {
192
+ }
193
+ }
194
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes status from CouchDB.", false);
195
+ }
196
+ function parseChangeLine(line) {
197
+ let parsed;
198
+ try {
199
+ parsed = JSON.parse(line);
200
+ } catch (error) {
201
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received a malformed changes-feed row.", false, {
202
+ cause: error
203
+ });
204
+ }
205
+ if (!parsed || typeof parsed !== "object" || !("seq" in parsed)) {
206
+ throw new StreamingFetchFailure(
207
+ "protocol",
208
+ "Fast Fetch received a changes-feed row without a sequence.",
209
+ false
210
+ );
72
211
  }
73
- return false;
212
+ const change = parsed;
213
+ if (change.seq === void 0 || change.doc !== void 0 && (!change.doc || typeof change.doc !== "object")) {
214
+ throw new StreamingFetchFailure("protocol", "Fast Fetch received an invalid changes-feed row.", false);
215
+ }
216
+ return change;
74
217
  }
75
218
  async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader, decryptFunction, since = "0", onProgress, onCheckpoint) {
76
219
  let totalFetched = 0;
77
220
  let totalValidFetched = 0;
221
+ let totalBytes = 0;
78
222
  const changesBaseParams = {
79
223
  feed: "continuous",
80
224
  include_docs: "true",
@@ -88,92 +232,81 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
88
232
  Accept: "application/json",
89
233
  Authorization: authHeader
90
234
  };
91
- let targetSeq = void 0;
92
- try {
93
- const dbInfoRes = await _fetch(remoteDbUrl, {
94
- headers: fetchHeaders
95
- });
96
- if (dbInfoRes.ok) {
97
- const dbInfo = await dbInfoRes.json();
98
- targetSeq = dbInfo.update_seq;
99
- }
100
- } catch (e) {
101
- Logger("Failed to fetch database info for target sequence:", LOG_LEVEL_VERBOSE);
102
- Logger(e, LOG_LEVEL_VERBOSE);
103
- }
104
- const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
235
+ const targetURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
105
236
  ...changesBaseParams,
106
237
  feed: "normal",
107
- limit: "1"
108
- });
109
- const infoRes = await _fetch(fetchURL.toString(), {
110
- headers: fetchHeaders
238
+ since: "now",
239
+ limit: "0"
111
240
  });
112
- const infoSource = await infoRes.text();
113
- const infoSourceTrimmed = infoSource.trim();
114
- if (!infoSourceTrimmed) {
115
- throw new Error("Failed to fetch changes from CouchDB. No data received.");
241
+ const targetResponse = await fetchResponse(
242
+ targetURL.toString(),
243
+ { headers: fetchHeaders },
244
+ "capture the changes target"
245
+ );
246
+ const targetStatus = parseStatusSource(await readResponseText(targetResponse, "read the changes target"));
247
+ const finalTargetSeq = targetStatus.last_seq;
248
+ if (finalTargetSeq === void 0) {
249
+ throw new StreamingFetchFailure(
250
+ "protocol",
251
+ "Fast Fetch could not obtain an authoritative changes target from CouchDB.",
252
+ false
253
+ );
116
254
  }
117
- let info;
118
- if (infoSourceTrimmed.startsWith("{") && infoSourceTrimmed.endsWith("}")) {
119
- try {
120
- info = JSON.parse(infoSourceTrimmed);
121
- } catch {
122
- const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
123
- const lastLine = infoLines[infoLines.length - 1];
124
- info = JSON.parse(lastLine);
125
- }
126
- } else {
127
- const infoLines = infoSourceTrimmed.split("\n").filter((line) => line.trim() !== "");
128
- const lastLine = infoLines[infoLines.length - 1];
129
- info = JSON.parse(lastLine);
255
+ if (isTargetSequence(since, finalTargetSeq)) {
256
+ Logger("Already at the target sequence. Initial data synchronisation is complete.");
257
+ return;
130
258
  }
131
- const pendingDocs = info.pending || 0;
132
- const docsToFetch = pendingDocs + 1;
133
- if (targetSeq === void 0) {
134
- targetSeq = info.last_seq || info.update_seq || info.seq;
259
+ const fetchURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
260
+ ...changesBaseParams,
261
+ feed: "normal",
262
+ limit: "0"
263
+ });
264
+ const infoResponse = await fetchResponse(fetchURL.toString(), { headers: fetchHeaders }, "read changes status");
265
+ const info = parseStatusSource(await readResponseText(infoResponse, "read changes status"));
266
+ if (typeof info.pending !== "number" || info.pending < 0) {
267
+ throw new StreamingFetchFailure(
268
+ "protocol",
269
+ "Fast Fetch received changes status without a valid pending count.",
270
+ false
271
+ );
135
272
  }
136
- const finalTargetSeq = targetSeq ?? "";
137
- if (reachedTargetSequence(since, finalTargetSeq)) {
138
- Logger("Already at the target sequence. Initial data synchronisation is complete.");
273
+ const pendingDocs = info.pending;
274
+ const docsToFetch = pendingDocs;
275
+ if (pendingDocs === 0) {
276
+ await saveCheckpoint(onCheckpoint, finalTargetSeq);
277
+ Logger("No changes remain before the captured Fast Fetch target.");
139
278
  return;
140
279
  }
141
280
  Logger(
142
- `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Total documents to fetch: ${docsToFetch}.`
281
+ `Starting initial synchronisation. Current sequence: ${since}, Target sequence: ${finalTargetSeq}, Estimated documents to fetch: ${docsToFetch}.`
143
282
  );
144
283
  const controller = new AbortController();
145
- const url = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), {
146
- ...changesBaseParams
147
- });
148
- const response = await _fetch(url.toString(), {
149
- method: "GET",
150
- headers: fetchHeaders,
151
- signal: controller.signal
152
- });
284
+ const changesURL = setParamsToURL(new URL(`${remoteDbUrl}/_changes`), changesBaseParams);
285
+ const response = await fetchResponse(
286
+ changesURL.toString(),
287
+ {
288
+ method: "GET",
289
+ headers: fetchHeaders,
290
+ signal: controller.signal
291
+ },
292
+ "open the changes feed"
293
+ );
153
294
  if (!response.body) {
154
- throw new Error("ReadableStream is not supported by this browser.");
295
+ throw new StreamingFetchFailure("protocol", "Fast Fetch could not read the CouchDB response stream.", false);
155
296
  }
156
297
  const sizeCaptureStream = new TransformStream({
157
- transform(chunk, controller2) {
158
- const chunkSize = chunk.length || chunk.byteLength || 0;
159
- totalBytes += chunkSize;
160
- controller2.enqueue(chunk);
298
+ transform(chunk, streamController) {
299
+ totalBytes += chunk.byteLength;
300
+ streamController.enqueue(chunk);
161
301
  }
162
302
  });
163
303
  const reader = response.body.pipeThrough(sizeCaptureStream).pipeThrough(new TextDecoderStream()).getReader();
164
- const writeDocStream = generatePouchDBWriteStream(downloadToDB, decryptFunction, onCheckpoint);
165
- const writer = writeDocStream.getWriter();
304
+ const batchWriter = generatePouchDBBatchWriter(downloadToDB, decryptFunction, onCheckpoint);
166
305
  let buffer = "";
167
306
  let lastProgress = 0;
168
307
  let lastReportTime = Date.now();
169
- let totalBytes = 0;
170
- let abortedAfterTargetReached = false;
171
308
  const reportProgress = () => {
172
- if (totalFetched - lastProgress < 25) {
173
- if (Date.now() - lastReportTime < 2e3) {
174
- return;
175
- }
176
- }
309
+ if (totalFetched - lastProgress < 25 && Date.now() - lastReportTime < 2e3) return;
177
310
  lastProgress = totalFetched;
178
311
  lastReportTime = Date.now();
179
312
  onProgress?.({
@@ -184,80 +317,74 @@ async function fetchChangesForInitialSync(downloadToDB, remoteDbUrl, authHeader,
184
317
  totalBytes
185
318
  });
186
319
  };
320
+ const processLine = async (line) => {
321
+ const parsed = parseChangeLine(line);
322
+ totalFetched++;
323
+ if (parsed.doc) {
324
+ await batchWriter.write(parsed.doc, parsed.seq);
325
+ totalValidFetched++;
326
+ } else {
327
+ await batchWriter.flushThrough(parsed.seq);
328
+ }
329
+ reportProgress();
330
+ if (!isTargetSequence(parsed.seq, finalTargetSeq)) return false;
331
+ await batchWriter.flush();
332
+ Logger("The captured Fast Fetch target is durable in the local database.");
333
+ controller.abort();
334
+ reportProgress();
335
+ return true;
336
+ };
187
337
  try {
188
338
  while (true) {
189
339
  reportProgress();
190
- const { value, done } = await reader.read();
191
- if (value) {
192
- totalBytes += value.length;
340
+ let readResult;
341
+ try {
342
+ readResult = await reader.read();
343
+ } catch (error) {
344
+ throw transportFailure("read the changes feed", error);
193
345
  }
194
- if (done) {
195
- if (buffer.trim()) {
196
- try {
197
- totalFetched++;
198
- const parsed = JSON.parse(buffer);
199
- if (parsed.doc) {
200
- await writer.write({ doc: parsed.doc, seq: parsed.seq });
201
- totalValidFetched++;
202
- } else {
203
- await onCheckpoint?.(parsed.seq);
204
- }
205
- reportProgress();
206
- } catch (e) {
207
- Logger(`Failed to parse the final line: ${buffer}. Skipping it.`);
208
- Logger(e, LOG_LEVEL_VERBOSE);
209
- }
210
- }
211
- await writer.close();
212
- break;
346
+ if (readResult.done) {
347
+ if (buffer.trim() && await processLine(buffer)) return;
348
+ await batchWriter.flush();
349
+ throw new StreamingFetchFailure(
350
+ "transport",
351
+ "The CouchDB changes feed ended before Fast Fetch reached its captured target.",
352
+ true
353
+ );
213
354
  }
214
- buffer += value;
355
+ buffer += readResult.value;
215
356
  const lines = buffer.split("\n");
216
357
  buffer = lines.pop() || "";
217
358
  for (const line of lines) {
218
359
  if (!line.trim()) continue;
219
- try {
220
- const parsed = JSON.parse(line);
221
- totalFetched++;
222
- if (parsed.doc) {
223
- await writer.write({ doc: parsed.doc, seq: parsed.seq });
224
- totalValidFetched++;
225
- } else {
226
- await onCheckpoint?.(parsed.seq);
227
- }
228
- reportProgress();
229
- if (totalFetched >= docsToFetch || reachedTargetSequence(parsed.seq, finalTargetSeq)) {
230
- Logger(
231
- `All documents fetched. Stopping the stream and writing remaining documents to PouchDB...`
232
- );
233
- await writer.close();
234
- abortedAfterTargetReached = true;
235
- controller.abort();
236
- reportProgress();
237
- return;
238
- }
239
- } catch (e) {
240
- Logger(`JSON parsing failed. Skipping line: ${line}`, LOG_LEVEL_VERBOSE);
241
- Logger(e, LOG_LEVEL_VERBOSE);
242
- }
360
+ if (await processLine(line)) return;
243
361
  }
244
362
  }
245
- Logger("Initial data synchronisation via stream has completed.");
246
- reportProgress();
247
363
  } catch (error) {
248
- if (abortedAfterTargetReached && error instanceof DOMException && error.name === "AbortError") {
249
- Logger(
250
- "Stream has been aborted as the target sequence has been reached. Finalising the synchronising process..."
251
- );
252
- return;
364
+ const failure = asStreamingFetchFailure(error);
365
+ if (failure.stage !== "storage") {
366
+ try {
367
+ await batchWriter.flush();
368
+ } catch (flushError) {
369
+ batchWriter.abort();
370
+ throw asStreamingFetchFailure(flushError);
371
+ }
253
372
  }
254
- Logger("An error occurred during synchronisation:", LOG_LEVEL_VERBOSE);
255
- Logger(error, LOG_LEVEL_VERBOSE);
256
- throw error;
373
+ batchWriter.abort();
374
+ Logger(`Fast Fetch failed during ${failure.stage}.`, LOG_LEVEL_VERBOSE);
375
+ Logger(failure, LOG_LEVEL_VERBOSE);
376
+ throw failure;
257
377
  } finally {
378
+ controller.abort();
379
+ try {
380
+ await reader.cancel();
381
+ } catch {
382
+ }
258
383
  reader.releaseLock();
259
384
  }
260
385
  }
261
386
  export {
262
- fetchChangesForInitialSync
387
+ StreamingFetchFailure,
388
+ fetchChangesForInitialSync,
389
+ isRetryableStreamingFetchFailure
263
390
  };
@@ -1,4 +1,4 @@
1
- import { type EntryDoc, type DatabaseConnectingStatus, type RemoteDBSettings, type EntryLeaf, type TweakValues, type NodeData } from "../common/types.js";
1
+ import { type EntryDoc, type DatabaseConnectingStatus, type RemoteDBSettings, type EntryLeaf, type TweakValues, type RemotePreferredTweakResult, type NodeData } from "../common/types.js";
2
2
  import type { RequiredServices } from "../interfaces/ServiceModule.js";
3
3
  import type { TranslationParameters } from "../services/base/MessageTranslator.js";
4
4
  import type { CommonlibMessageKey } from "../services/base/CommonlibMessages.js";
@@ -60,7 +60,7 @@ export declare abstract class LiveSyncAbstractReplicator {
60
60
  abstract setPreferredRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
61
61
  abstract fetchRemoteChunks(missingChunks: string[], showResult: boolean): Promise<false | EntryLeaf[]>;
62
62
  abstract getRemoteStatus(setting: RemoteDBSettings): Promise<false | RemoteDBStatus>;
63
- abstract getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<false | TweakValues>;
63
+ abstract getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
64
64
  abstract countCompromisedChunks(setting?: RemoteDBSettings): Promise<number | boolean>;
65
65
  abstract getConnectedDeviceList(setting?: RemoteDBSettings): Promise<false | {
66
66
  node_info: Record<string, NodeData>;
@@ -1,4 +1,4 @@
1
- import { type EntryDoc, type RemoteDBSettings, type EntryLeaf, type TweakValues, type SyncParameters, type DatabaseEntry, type NodeData } from "../../common/types.js";
1
+ import { type EntryDoc, type RemoteDBSettings, type EntryLeaf, type SyncParameters, type DatabaseEntry, type NodeData, type RemotePreferredTweakResult } from "../../common/types.js";
2
2
  import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv, type RemoteDBStatus } from "../LiveSyncAbstractReplicator.js";
3
3
  import type { ServiceHub } from "../../services/ServiceHub.js";
4
4
  export interface LiveSyncCouchDBReplicatorEnv extends LiveSyncReplicatorEnv {
@@ -73,7 +73,7 @@ export declare class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicato
73
73
  tryConnectRemote(setting: RemoteDBSettings, showResult?: boolean): Promise<boolean>;
74
74
  resetRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
75
75
  setPreferredRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
76
- getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<TweakValues | false>;
76
+ getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
77
77
  compactRemote(setting: RemoteDBSettings): Promise<boolean>;
78
78
  getRemoteStatus(setting: RemoteDBSettings): Promise<RemoteDBStatus | false>;
79
79
  countCompromisedChunks(setting?: RemoteDBSettings): Promise<number | boolean>;
@@ -9,7 +9,9 @@ import {
9
9
  TweakValuesTemplate,
10
10
  DOCID_SYNC_PARAMETERS,
11
11
  DEFAULT_SYNC_PARAMETERS,
12
- ProtocolVersions
12
+ ProtocolVersions,
13
+ RemotePreferredTweakNotConfiguredReasons,
14
+ RemotePreferredTweakStatuses
13
15
  } from "../../common/types.js";
14
16
  import {
15
17
  resolveWithIgnoreKnownError,
@@ -22,6 +24,7 @@ import {
22
24
  } from "../../common/utils.js";
23
25
  import { Logger } from "../../common/logger.js";
24
26
  import { checkRemoteVersion, countCompromisedChunks } from "../../pouchdb/negotiation.js";
27
+ import { isErrorOfMissingDoc } from "../../pouchdb/utils_couchdb.js";
25
28
  import { preprocessOutgoing } from "../../pouchdb/encryption.js";
26
29
  import { ensureDatabaseIsCompatible } from "../../pouchdb/LiveSyncDBFunctions.js";
27
30
  import {
@@ -1050,23 +1053,73 @@ ${sendingDocs}/(${sentDocsCount} done)`,
1050
1053
  }
1051
1054
  async getRemotePreferredTweakValues(setting) {
1052
1055
  const uri = setting.couchDB_URI.replace(/\/+$/, "") + (setting.couchDB_DBNAME == "" ? "" : "/" + setting.couchDB_DBNAME);
1053
- const dbRet = await this.connectRemoteCouchDBWithSetting(setting, this.isMobile(), true);
1056
+ let dbRet;
1057
+ try {
1058
+ dbRet = await this.connectRemoteCouchDBWithSetting(setting, this.isMobile(), true);
1059
+ } catch (ex) {
1060
+ Logger(`Could not connect to the remote database`, LOG_LEVEL_NOTICE);
1061
+ Logger(ex, LOG_LEVEL_VERBOSE);
1062
+ return {
1063
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
1064
+ error: ex
1065
+ };
1066
+ }
1054
1067
  if (typeof dbRet === "string") {
1055
1068
  Logger(this.translate("liveSyncReplicator.couldNotConnectToURI", { uri, dbRet }), LOG_LEVEL_NOTICE);
1056
- return false;
1069
+ return {
1070
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
1071
+ error: new Error(dbRet)
1072
+ };
1057
1073
  }
1058
- if (!await checkRemoteVersion(dbRet.db, this.migrate.bind(this), VER)) {
1059
- Logger(this.translate("liveSyncReplicator.remoteDbCorrupted"), LOG_LEVEL_NOTICE);
1060
- return false;
1074
+ try {
1075
+ if (!await checkRemoteVersion(dbRet.db, this.migrate.bind(this), VER)) {
1076
+ const error = new Error("The remote database version is not compatible");
1077
+ Logger(this.translate("liveSyncReplicator.remoteDbCorrupted"), LOG_LEVEL_NOTICE);
1078
+ return {
1079
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
1080
+ error
1081
+ };
1082
+ }
1083
+ } catch (ex) {
1084
+ Logger(`Could not check the remote database version`, LOG_LEVEL_NOTICE);
1085
+ Logger(ex, LOG_LEVEL_VERBOSE);
1086
+ return {
1087
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
1088
+ error: ex
1089
+ };
1061
1090
  }
1062
1091
  try {
1063
1092
  const remoteMilestone = await dbRet.db.get(MILESTONE_DOCID);
1064
- if (!remoteMilestone) throw new Error("Remote milestone not found");
1065
- return remoteMilestone?.tweak_values?.[DEVICE_ID_PREFERRED] || false;
1093
+ if (!remoteMilestone) {
1094
+ return {
1095
+ status: RemotePreferredTweakStatuses.NOT_CONFIGURED,
1096
+ reason: RemotePreferredTweakNotConfiguredReasons.MILESTONE_MISSING
1097
+ };
1098
+ }
1099
+ const preferred = remoteMilestone.tweak_values?.[DEVICE_ID_PREFERRED];
1100
+ if (!preferred) {
1101
+ return {
1102
+ status: RemotePreferredTweakStatuses.NOT_CONFIGURED,
1103
+ reason: RemotePreferredTweakNotConfiguredReasons.PREFERRED_VALUES_MISSING
1104
+ };
1105
+ }
1106
+ return {
1107
+ status: RemotePreferredTweakStatuses.AVAILABLE,
1108
+ values: preferred
1109
+ };
1066
1110
  } catch (ex) {
1111
+ if (isErrorOfMissingDoc(ex)) {
1112
+ return {
1113
+ status: RemotePreferredTweakStatuses.NOT_CONFIGURED,
1114
+ reason: RemotePreferredTweakNotConfiguredReasons.MILESTONE_MISSING
1115
+ };
1116
+ }
1067
1117
  Logger(`Could not retrieve remote milestone`, LOG_LEVEL_NOTICE);
1068
1118
  Logger(ex, LOG_LEVEL_VERBOSE);
1069
- return false;
1119
+ return {
1120
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
1121
+ error: ex
1122
+ };
1070
1123
  }
1071
1124
  }
1072
1125
  async compactRemote(setting) {
@@ -3,7 +3,7 @@ import type { ReplicationCallback, ReplicationStat } from "../LiveSyncAbstractRe
3
3
  import { type SimpleStore } from "../../common/utils.js";
4
4
  import { type CheckPointInfo } from "./JournalSyncTypes.js";
5
5
  import type { LiveSyncJournalReplicatorEnv } from "./LiveSyncJournalReplicatorEnv.js";
6
- import type { IJournalStorage } from "./objectstore/JournalStorageAdapter.js";
6
+ import { type IJournalStorage, type JournalStorageReadResult } from "./objectstore/JournalStorageAdapter.js";
7
7
  type ProcessingEntry = PouchDB.Core.PutDocument<EntryDoc> & PouchDB.Core.GetMeta;
8
8
  export declare class JournalSyncCore {
9
9
  _settings: BucketSyncSetting;
@@ -22,6 +22,7 @@ export declare class JournalSyncCore {
22
22
  getHash(settings: BucketSyncSetting): string;
23
23
  constructor(settings: BucketSyncSetting, store: SimpleStore<CheckPointInfo>, env: LiveSyncJournalReplicatorEnv, storage: IJournalStorage);
24
24
  downloadJson<T>(key: string): Promise<T | false>;
25
+ downloadJsonWithResult<T>(key: string): Promise<JournalStorageReadResult<T>>;
25
26
  uploadJson<T>(key: string, body: T): Promise<boolean>;
26
27
  applyNewConfig(settings: BucketSyncSetting, store: SimpleStore<CheckPointInfo>, env: LiveSyncJournalReplicatorEnv): void;
27
28
  updateInfo(info: Partial<ReplicationStat>): void;
@@ -18,6 +18,9 @@ import {
18
18
  import { shareRunningResult } from "octagonal-wheels/concurrency/lock";
19
19
  import { wrappedDeflate, wrappedInflate } from "../../pouchdb/compress.js";
20
20
  import { CheckPointInfoDefault } from "./JournalSyncTypes.js";
21
+ import {
22
+ JournalStorageReadStatuses
23
+ } from "./objectstore/JournalStorageAdapter.js";
21
24
  import {
22
25
  clearHandlers,
23
26
  createSyncParamsHanderForServer,
@@ -116,6 +119,23 @@ class JournalSyncCore {
116
119
  return false;
117
120
  }
118
121
  }
122
+ async downloadJsonWithResult(key) {
123
+ const result = await this.storage.downloadWithResult(key, true);
124
+ if (result.status !== JournalStorageReadStatuses.AVAILABLE) return result;
125
+ try {
126
+ return {
127
+ status: JournalStorageReadStatuses.AVAILABLE,
128
+ value: JSON.parse(new TextDecoder().decode(result.value))
129
+ };
130
+ } catch (ex) {
131
+ Logger(`Could not parse downloaded json ${key}`);
132
+ Logger(ex, LOG_LEVEL_VERBOSE);
133
+ return {
134
+ status: JournalStorageReadStatuses.UNAVAILABLE,
135
+ error: ex
136
+ };
137
+ }
138
+ }
119
139
  async uploadJson(key, body) {
120
140
  try {
121
141
  const data = new TextEncoder().encode(JSON.stringify(body));
@@ -1,4 +1,4 @@
1
- import { type RemoteDBSettings, type EntryLeaf, type ChunkVersionRange, type TweakValues, type NodeData } from "../../common/types.js";
1
+ import { type RemoteDBSettings, type EntryLeaf, type ChunkVersionRange, type NodeData, type RemotePreferredTweakResult } from "../../common/types.js";
2
2
  import { JournalSyncCore } from "./JournalSyncCore.js";
3
3
  import { LiveSyncAbstractReplicator, type RemoteDBStatus } from "../LiveSyncAbstractReplicator.js";
4
4
  import { type ENSURE_DB_RESULT } from "../../pouchdb/LiveSyncDBFunctions.js";
@@ -29,7 +29,7 @@ export declare class LiveSyncJournalReplicator extends LiveSyncAbstractReplicato
29
29
  tryConnectRemote(setting: RemoteDBSettings, showResult?: boolean): Promise<boolean>;
30
30
  resetRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
31
31
  setPreferredRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
32
- getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<false | TweakValues>;
32
+ getRemotePreferredTweakValues(_setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
33
33
  getRemoteStatus(setting: RemoteDBSettings): Promise<false | RemoteDBStatus>;
34
34
  countCompromisedChunks(): Promise<number>;
35
35
  getConnectedDeviceList(setting?: RemoteDBSettings): Promise<false | {
@@ -2,7 +2,9 @@ import {
2
2
  LOG_LEVEL_NOTICE,
3
3
  LOG_LEVEL_VERBOSE,
4
4
  DEVICE_ID_PREFERRED,
5
- TweakValuesTemplate
5
+ TweakValuesTemplate,
6
+ RemotePreferredTweakNotConfiguredReasons,
7
+ RemotePreferredTweakStatuses
6
8
  } from "../../common/types.js";
7
9
  import { Logger } from "../../common/logger.js";
8
10
  import { JournalSyncCore } from "./JournalSyncCore.js";
@@ -11,6 +13,7 @@ import { LiveSyncAbstractReplicator } from "../LiveSyncAbstractReplicator.js";
11
13
  import { ensureRemoteIsCompatible } from "../../pouchdb/LiveSyncDBFunctions.js";
12
14
  import { extractObject } from "../../common/utils.js";
13
15
  import { clearHandlers } from "../SyncParamsHandler.js";
16
+ import { JournalStorageReadStatuses } from "./objectstore/JournalStorageAdapter.js";
14
17
  const MILSTONE_DOCID = "_00000000-milestone.json";
15
18
  const currentVersionRange = {
16
19
  min: 0,
@@ -256,18 +259,31 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
256
259
  throw ex;
257
260
  }
258
261
  }
259
- async getRemotePreferredTweakValues(setting) {
260
- try {
261
- const remoteMilestone = await this.client.downloadJson(MILSTONE_DOCID);
262
- if (!remoteMilestone) {
263
- throw new Error("Missing remote milestone");
264
- }
265
- return remoteMilestone?.tweak_values?.[DEVICE_ID_PREFERRED] || false;
266
- } catch (ex) {
267
- Logger(`Could not retrieve remote milestone`, LOG_LEVEL_NOTICE);
268
- Logger(ex, LOG_LEVEL_VERBOSE);
269
- return false;
262
+ async getRemotePreferredTweakValues(_setting) {
263
+ const result = await this.client.downloadJsonWithResult(MILSTONE_DOCID);
264
+ if (result.status === JournalStorageReadStatuses.NOT_FOUND) {
265
+ return {
266
+ status: RemotePreferredTweakStatuses.NOT_CONFIGURED,
267
+ reason: RemotePreferredTweakNotConfiguredReasons.MILESTONE_MISSING
268
+ };
269
+ }
270
+ if (result.status === JournalStorageReadStatuses.UNAVAILABLE) {
271
+ return {
272
+ status: RemotePreferredTweakStatuses.UNAVAILABLE,
273
+ error: result.error
274
+ };
270
275
  }
276
+ const preferred = result.value.tweak_values?.[DEVICE_ID_PREFERRED];
277
+ if (!preferred) {
278
+ return {
279
+ status: RemotePreferredTweakStatuses.NOT_CONFIGURED,
280
+ reason: RemotePreferredTweakNotConfiguredReasons.PREFERRED_VALUES_MISSING
281
+ };
282
+ }
283
+ return {
284
+ status: RemotePreferredTweakStatuses.AVAILABLE,
285
+ values: preferred
286
+ };
271
287
  }
272
288
  async getRemoteStatus(setting) {
273
289
  const testClient = new MinioStorageAdapter(setting, this.env);
@@ -1,8 +1,23 @@
1
1
  import type { RemoteDBStatus } from "../../LiveSyncAbstractReplicator.js";
2
2
  import type { BucketSyncSetting } from "../../../common/types.js";
3
+ export declare const JournalStorageReadStatuses: {
4
+ readonly AVAILABLE: "available";
5
+ readonly NOT_FOUND: "not-found";
6
+ readonly UNAVAILABLE: "unavailable";
7
+ };
8
+ export type JournalStorageReadResult<T> = {
9
+ status: typeof JournalStorageReadStatuses.AVAILABLE;
10
+ value: T;
11
+ } | {
12
+ status: typeof JournalStorageReadStatuses.NOT_FOUND;
13
+ } | {
14
+ status: typeof JournalStorageReadStatuses.UNAVAILABLE;
15
+ error: unknown;
16
+ };
3
17
  export interface IJournalStorage {
4
18
  upload(key: string, data: Uint8Array, mime: string): Promise<boolean>;
5
19
  download(key: string, ignoreCache?: boolean): Promise<Uint8Array | false>;
20
+ downloadWithResult(key: string, ignoreCache?: boolean): Promise<JournalStorageReadResult<Uint8Array>>;
6
21
  listFiles(from: string, limit?: number): Promise<string[]>;
7
22
  deleteFiles(keys: string[]): Promise<boolean>;
8
23
  isAvailable(): Promise<boolean>;
@@ -0,0 +1,8 @@
1
+ const JournalStorageReadStatuses = {
2
+ AVAILABLE: "available",
3
+ NOT_FOUND: "not-found",
4
+ UNAVAILABLE: "unavailable"
5
+ };
6
+ export {
7
+ JournalStorageReadStatuses
8
+ };
@@ -1,7 +1,7 @@
1
1
  import { S3 } from "@aws-sdk/client-s3";
2
2
  import { type BucketSyncSetting } from "../../../common/types.js";
3
3
  import type { RemoteDBStatus } from "../../LiveSyncAbstractReplicator.js";
4
- import type { IJournalStorage } from "./JournalStorageAdapter.js";
4
+ import { type IJournalStorage, type JournalStorageReadResult } from "./JournalStorageAdapter.js";
5
5
  import type { LiveSyncJournalReplicatorEnv } from "../LiveSyncJournalReplicatorEnv.js";
6
6
  export declare class MinioStorageAdapter implements IJournalStorage {
7
7
  _instance?: S3;
@@ -13,6 +13,7 @@ export declare class MinioStorageAdapter implements IJournalStorage {
13
13
  get customHeaders(): [string, string][];
14
14
  _getClient(): S3;
15
15
  upload(key: string, data: Uint8Array, mime: string): Promise<boolean>;
16
+ downloadWithResult(key: string, ignoreCache?: boolean): Promise<JournalStorageReadResult<Uint8Array>>;
16
17
  download(key: string, ignoreCache?: boolean): Promise<Uint8Array | false>;
17
18
  listFiles(from: string, limit?: number): Promise<string[]>;
18
19
  deleteFiles(keys: string[]): Promise<boolean>;
@@ -11,6 +11,9 @@ import { ConfiguredRetryStrategy } from "@smithy/util-retry";
11
11
  import { promiseWithResolvers } from "octagonal-wheels/promises";
12
12
  import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../../../common/types.js";
13
13
  import { Logger } from "../../../common/logger.js";
14
+ import {
15
+ JournalStorageReadStatuses
16
+ } from "./JournalStorageAdapter.js";
14
17
  import { parseHeaderValues } from "../../../common/utils.js";
15
18
  import { runWithTrackedPhysicalRequest } from "../../../services/lib/remoteActivity.js";
16
19
  class MinioStorageAdapter {
@@ -110,7 +113,7 @@ class MinioStorageAdapter {
110
113
  }
111
114
  return false;
112
115
  }
113
- async download(key, ignoreCache = false) {
116
+ async downloadWithResult(key, ignoreCache = false) {
114
117
  const client = this._getClient();
115
118
  const cmd = new GetObjectCommand({
116
119
  Bucket: this._settings.bucket,
@@ -121,15 +124,31 @@ class MinioStorageAdapter {
121
124
  return await this.runTrackedRequest(async () => {
122
125
  const r = await client.send(cmd);
123
126
  if (r.Body) {
124
- return new Uint8Array(await r.Body.transformToByteArray());
127
+ return {
128
+ status: JournalStorageReadStatuses.AVAILABLE,
129
+ value: new Uint8Array(await r.Body.transformToByteArray())
130
+ };
125
131
  }
126
- return false;
132
+ return {
133
+ status: JournalStorageReadStatuses.UNAVAILABLE,
134
+ error: new Error(`The response body for ${key} was empty`)
135
+ };
127
136
  });
128
137
  } catch (ex) {
138
+ if (isMissingObjectError(ex)) {
139
+ return { status: JournalStorageReadStatuses.NOT_FOUND };
140
+ }
129
141
  Logger(`Could not download ${key}`);
130
142
  Logger(ex, LOG_LEVEL_VERBOSE);
143
+ return {
144
+ status: JournalStorageReadStatuses.UNAVAILABLE,
145
+ error: ex
146
+ };
131
147
  }
132
- return false;
148
+ }
149
+ async download(key, ignoreCache = false) {
150
+ const result = await this.downloadWithResult(key, ignoreCache);
151
+ return result.status === JournalStorageReadStatuses.AVAILABLE ? result.value : false;
133
152
  }
134
153
  async listFiles(from, limit) {
135
154
  const client = this._getClient();
@@ -199,6 +218,12 @@ class MinioStorageAdapter {
199
218
  }
200
219
  }
201
220
  }
221
+ function isMissingObjectError(error) {
222
+ if (!error || typeof error !== "object") return false;
223
+ const { name, Code, code } = error;
224
+ const errorCode = Code ?? code ?? name;
225
+ return errorCode === "NoSuchKey" || errorCode === "NotFound";
226
+ }
202
227
  export {
203
228
  MinioStorageAdapter
204
229
  };
@@ -1,4 +1,4 @@
1
- import { type RemoteDBSettings, type EntryLeaf, type TweakValues, type LOG_LEVEL, type NodeData } from "../../common/types.js";
1
+ import { type RemoteDBSettings, type EntryLeaf, type RemotePreferredTweakResult, type LOG_LEVEL, type NodeData } from "../../common/types.js";
2
2
  import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv, type RemoteDBStatus } from "../LiveSyncAbstractReplicator.js";
3
3
  import { TrysteroReplicator } from "./TrysteroReplicator.js";
4
4
  import { P2PHost, type AcceptanceDecision, type RevokeAcceptanceDecision } from "./TrysteroReplicatorP2PServer.js";
@@ -81,7 +81,7 @@ export declare class LiveSyncTrysteroReplicator extends LiveSyncAbstractReplicat
81
81
  setPreferredRemoteTweakSettings(_setting: RemoteDBSettings): Promise<void>;
82
82
  fetchRemoteChunks(_missingChunks: string[], _showResult: boolean): Promise<false | EntryLeaf[]>;
83
83
  getRemoteStatus(_setting: RemoteDBSettings): Promise<false | RemoteDBStatus>;
84
- getRemotePreferredTweakValues(_setting: RemoteDBSettings): Promise<false | TweakValues>;
84
+ getRemotePreferredTweakValues(_setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
85
85
  countCompromisedChunks(): Promise<number>;
86
86
  getConnectedDeviceList(_setting?: RemoteDBSettings): Promise<false | {
87
87
  node_info: Record<string, NodeData>;
@@ -1,5 +1,6 @@
1
1
  import { Logger } from "../../common/logger.js";
2
2
  import {
3
+ RemotePreferredTweakStatuses,
3
4
  LOG_LEVEL_NOTICE,
4
5
  LOG_LEVEL_INFO,
5
6
  LOG_LEVEL_VERBOSE,
@@ -369,7 +370,7 @@ class LiveSyncTrysteroReplicator extends LiveSyncAbstractReplicator {
369
370
  "Trying to get tweak values but P2P replication does not support to do this. This operation has been ignored",
370
371
  LOG_LEVEL_INFO
371
372
  );
372
- return Promise.resolve(false);
373
+ return Promise.resolve({ status: RemotePreferredTweakStatuses.UNSUPPORTED });
373
374
  }
374
375
  countCompromisedChunks() {
375
376
  Logger("P2P Replicator cannot count compromised chunks", LOG_LEVEL_VERBOSE);
@@ -5,7 +5,7 @@ import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/log
5
5
  import { delay } from "octagonal-wheels/promises";
6
6
  import { EVENT_DATABASE_REBUILT } from "../events/coreEvents.js";
7
7
  import { ServiceModuleBase } from "./ServiceModuleBase.js";
8
- import { fetchChangesForInitialSync } from "../pouchdb/StreamingFetch.js";
8
+ import { fetchChangesForInitialSync, isRetryableStreamingFetchFailure } from "../pouchdb/StreamingFetch.js";
9
9
  import { getConfiguredFunctionsForEncryption } from "../pouchdb/encryption.js";
10
10
  import { AuthorizationHeaderGenerator, generateCredentialObject } from "../replication/httplib.js";
11
11
  import { sizeToHumanReadable } from "octagonal-wheels/number";
@@ -376,7 +376,7 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
376
376
  );
377
377
  break;
378
378
  } catch (ex) {
379
- if (attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
379
+ if (!isRetryableStreamingFetchFailure(ex) || attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
380
380
  checkpoint = this.getFastFetchCheckpoint(remote);
381
381
  since = checkpoint?.sequence ?? since;
382
382
  this._log(
@@ -1,6 +1,6 @@
1
1
  import type { FetchHttpHandler } from "@smithy/fetch-http-handler";
2
2
  import { type LOG_LEVEL } from "octagonal-wheels/common/logger";
3
- import type { AnyEntry, AUTO_MERGED, CouchDBCredentials, diff_result, DocumentID, EntryDoc, EntryHasPath, FileEventItem, FilePath, FilePathWithPrefix, LoadedEntry, MetaEntry, MISSING_OR_ERROR, ObsidianLiveSyncSettings, RemoteDBSettings, SettingsMigrationState, TweakValues, UXFileInfo, UXFileInfoStub } from "../../common/types.js";
3
+ import type { AnyEntry, AUTO_MERGED, CouchDBCredentials, diff_result, DocumentID, EntryDoc, EntryHasPath, FileEventItem, FilePath, FilePathWithPrefix, LoadedEntry, MetaEntry, MISSING_OR_ERROR, ObsidianLiveSyncSettings, RemoteDBSettings, RemotePreferredTweakResult, SettingsMigrationState, TweakValues, UXFileInfo, UXFileInfoStub } from "../../common/types.js";
4
4
  import type { LiveSyncLocalDB } from "../../pouchdb/LiveSyncLocalDB.js";
5
5
  import type { LiveSyncAbstractReplicator } from "../../replication/LiveSyncAbstractReplicator.js";
6
6
  import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
@@ -221,7 +221,7 @@ export interface ISettingService {
221
221
  deleteSmallConfig(key: string): void;
222
222
  }
223
223
  export interface ITweakValueService {
224
- fetchRemotePreferred(trialSetting: RemoteDBSettings): Promise<TweakValues | false>;
224
+ fetchRemotePreferred(trialSetting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
225
225
  checkAndAskResolvingMismatched(preferred: Partial<TweakValues>): Promise<[TweakValues | boolean, boolean]>;
226
226
  askResolvingMismatched(preferredSource: TweakValues): Promise<"OK" | "CHECKAGAIN" | "IGNORE">;
227
227
  checkAndAskUseRemoteConfiguration(settings: RemoteDBSettings): Promise<{
@@ -1,4 +1,4 @@
1
- import type { RemoteDBSettings, TweakValues } from "../../common/types.js";
1
+ import type { RemoteDBSettings, RemotePreferredTweakResult, TweakValues } from "../../common/types.js";
2
2
  import type { ITweakValueService } from "./IService.js";
3
3
  import { ServiceBase, type ServiceContext } from "./ServiceBase.js";
4
4
  /**
@@ -9,7 +9,7 @@ export declare abstract class TweakValueService<T extends ServiceContext = Servi
9
9
  * Fetch and trial the remote database settings to determine if they are preferred.
10
10
  * @param trialSetting The remote database settings to connect.
11
11
  */
12
- abstract fetchRemotePreferred(trialSetting: RemoteDBSettings): Promise<TweakValues | false>;
12
+ abstract fetchRemotePreferred(trialSetting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
13
13
  /**
14
14
  * Check and ask the user to resolve any mismatched tweak values.
15
15
  * @param preferred The preferred tweak values to check against.
@@ -1,7 +1,7 @@
1
1
  import type { ServiceContext } from "../../base/ServiceBase.js";
2
2
  import { TweakValueService } from "../../base/TweakValueService.js";
3
3
  export declare class InjectableTweakValueService<T extends ServiceContext> extends TweakValueService<T> {
4
- fetchRemotePreferred: import("../../lib/HandlerUtils.js").HandlerFunction<(trialSetting: import("../../../settings.js").RemoteDBSettings) => Promise<import("../../../common/types.js").TweakValues | false>, unknown>;
4
+ fetchRemotePreferred: import("../../lib/HandlerUtils.js").HandlerFunction<(trialSetting: import("../../../settings.js").RemoteDBSettings) => Promise<import("../../../common/types.js").RemotePreferredTweakResult>, unknown>;
5
5
  checkAndAskResolvingMismatched: import("../../lib/HandlerUtils.js").HandlerFunction<(preferred: Partial<import("../../../common/types.js").TweakValues>) => Promise<[import("../../../common/types.js").TweakValues | boolean, boolean]>, unknown>;
6
6
  askResolvingMismatched: import("../../lib/HandlerUtils.js").HandlerFunction<(preferredSource: import("../../../common/types.js").TweakValues) => Promise<"OK" | "CHECKAGAIN" | "IGNORE">, unknown>;
7
7
  checkAndAskUseRemoteConfiguration: import("../../lib/HandlerUtils.js").HandlerFunction<(settings: import("../../../settings.js").RemoteDBSettings) => Promise<{
@@ -20,6 +20,8 @@ Commonlib's chunk-reachability tests use real PouchDB revision trees. They verif
20
20
 
21
21
  Commonlib's injected CouchDB tests verify that a mixed remote chunk batch preserves available documents when another requested ID is absent. Its chunk-delivery tests separately verify that available chunks reach local storage and only absent IDs are marked unavailable. Self-hosted LiveSync supplies the real CouchDB connection and start-up composition, so device-specific replication timing remains downstream integration evidence rather than Commonlib unit coverage.
22
22
 
23
+ Commonlib's remote-preference result contract distinguishes available synchronisation settings, a remote which has not stored those settings yet, a failed read, and a remote type which does not support central preferences. Focused CouchDB and Object Storage tests cover absent milestone data separately from transport failures, while Self-hosted LiveSync owns the initialisation choices, cancellation policy, and user-facing explanation.
24
+
23
25
  Most replication and storage services used by the plug-in still enter through explicit `compat/*` paths. Those imports prove that the published compatibility boundary supports the current migration; they are not examples of a finished high-level client API.
24
26
 
25
27
  The plug-in's manual onboarding and Remote Databases pane use the focused `/remote-configurations` entry to create opaque profile IDs, retain user-visible names, and select main and P2P remotes. Commonlib owns the in-memory profile result contract; Self-hosted LiveSync adds dialogue, Setup URI classification, Fetch or Rebuild scheduling, persistence, restart ordering, and real-Obsidian presentation checks.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vrtmrz/livesync-commonlib",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "description": "Shared data, storage, and replication primitives for Self-hosted LiveSync clients.",
5
5
  "type": "module",
6
6
  "license": "MIT",