@vrtmrz/livesync-commonlib 0.1.33 → 0.1.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,11 @@ import { DEFAULT_SETTINGS } from "../common/types.js";
5
5
  import {} from "./ChunkManager.js";
6
6
  import { compatGlobal } from "../common/coreEnvFunctions.js";
7
7
  import { DEFAULT_CHUNK_DELIVERY_STALL_TIMEOUT_MS } from "./ChunkDeliveryCoordinator.js";
8
+ import { chunkFetchCounts, collectingChunks } from "../mock_and_interop/stores.js";
9
+ import {
10
+ classifyFetchedChunks,
11
+ decideMissingChunkAction
12
+ } from "./chunkFetcher.helper.js";
8
13
  const EVENT_MISSING_CHUNKS = "missingChunks";
9
14
  const EVENT_MISSING_CHUNK_REMOTE = "missingChunkRemote";
10
15
  const EVENT_CHUNK_FETCHED = "chunkFetched";
@@ -19,6 +24,13 @@ class ChunkFetcher {
19
24
  }
20
25
  queue = [];
21
26
  pendingClaims = /* @__PURE__ */ new Map();
27
+ reportedPendingCount = 0;
28
+ reportedFetchCounts = { initial: 0, retrying: 0 };
29
+ requestTimer;
30
+ requestTimerDueAt = 0;
31
+ finiteActivityWasActive;
32
+ finiteCompletionVersion = 0;
33
+ stopObservingDelivery;
22
34
  destroyed = false;
23
35
  get interval() {
24
36
  const settings = this.options.settingService.currentSettings();
@@ -31,6 +43,18 @@ class ChunkFetcher {
31
43
  abort = new AbortController();
32
44
  constructor(options) {
33
45
  this.options = options;
46
+ this.finiteActivityWasActive = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
47
+ this.stopObservingDelivery = this.chunkManager.deliveryCoordinator.onChanged(() => {
48
+ const active = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
49
+ const completed = this.finiteActivityWasActive && !active;
50
+ this.finiteActivityWasActive = active;
51
+ if (!completed) return;
52
+ this.finiteCompletionVersion++;
53
+ for (const pending of this.pendingClaims.values()) {
54
+ if (pending.missingResponses > 0 && !pending.inFlight) pending.nextRequestAt = Date.now();
55
+ }
56
+ this.scheduleRequest();
57
+ });
34
58
  this.chunkManager.addListener(EVENT_MISSING_CHUNKS, this.onEventHandler, {
35
59
  signal: this.abort.signal
36
60
  });
@@ -39,9 +63,12 @@ class ChunkFetcher {
39
63
  if (this.destroyed) return;
40
64
  this.destroyed = true;
41
65
  this.abort.abort();
66
+ this.stopObservingDelivery();
67
+ this.clearRequestTimer();
42
68
  this.queue = [];
43
- const pendingDeliveries = new Set(this.pendingClaims.values());
69
+ const pendingDeliveries = new Set([...this.pendingClaims.values()].map((pending) => pending.delivery));
44
70
  this.pendingClaims.clear();
71
+ this.updatePendingCount();
45
72
  for (const pending of pendingDeliveries) {
46
73
  pending.resolveActivityBoundary(false);
47
74
  pending.claim.release();
@@ -52,9 +79,35 @@ class ChunkFetcher {
52
79
  if (this.destroyed) return;
53
80
  const claimedIds = this.ensureClaims(ids);
54
81
  this.queue = unique([...this.queue, ...claimedIds]);
55
- if (this.canRequestMore()) {
56
- compatGlobal.setTimeout(() => void this.requestMissingChunks(), 1);
82
+ this.scheduleRequest(1);
83
+ }
84
+ clearRequestTimer() {
85
+ if (this.requestTimer !== void 0) compatGlobal.clearTimeout(this.requestTimer);
86
+ this.requestTimer = void 0;
87
+ }
88
+ scheduleRequest(minimumDelayMs = 0) {
89
+ if (this.destroyed || this.currentProcessing >= this.concurrency || this.queue.length === 0) {
90
+ this.clearRequestTimer();
91
+ return;
92
+ }
93
+ let nextRequestAt = Infinity;
94
+ for (const id of this.queue) {
95
+ const pending = this.pendingClaims.get(id);
96
+ if (!pending?.inFlight) nextRequestAt = Math.min(nextRequestAt, pending?.nextRequestAt ?? 0);
57
97
  }
98
+ if (nextRequestAt === Infinity) {
99
+ this.clearRequestTimer();
100
+ return;
101
+ }
102
+ const now = Date.now();
103
+ const dueAt = Math.max(nextRequestAt, now + minimumDelayMs);
104
+ if (this.requestTimer !== void 0 && this.requestTimerDueAt <= dueAt) return;
105
+ this.clearRequestTimer();
106
+ this.requestTimerDueAt = dueAt;
107
+ this.requestTimer = compatGlobal.setTimeout(() => {
108
+ this.requestTimer = void 0;
109
+ void this.requestMissingChunks();
110
+ }, dueAt - now);
58
111
  }
59
112
  ensureClaims(ids) {
60
113
  const unclaimedIds = unique(ids.filter((id) => !this.pendingClaims.has(id)));
@@ -71,8 +124,9 @@ class ChunkFetcher {
71
124
  resolveActivityBoundary: activityBoundary.resolve
72
125
  };
73
126
  for (const id of unclaimedIds) {
74
- this.pendingClaims.set(id, pending);
127
+ this.pendingClaims.set(id, { delivery: pending, missingResponses: 0, nextRequestAt: 0, inFlight: false });
75
128
  }
129
+ this.updatePendingCount();
76
130
  void this.options.replicatorService.runBoundedRemoteActivity(
77
131
  () => {
78
132
  pending.claim.touch();
@@ -92,45 +146,142 @@ class ChunkFetcher {
92
146
  }
93
147
  removePendingDelivery(pending, ids) {
94
148
  for (const id of ids) {
95
- if (this.pendingClaims.get(id) === pending) {
149
+ if (this.pendingClaims.get(id)?.delivery === pending) {
96
150
  this.pendingClaims.delete(id);
97
151
  }
98
152
  }
99
153
  const removed = new Set(ids);
100
154
  this.queue = this.queue.filter((id) => !(removed.has(id) && !this.pendingClaims.has(id)));
155
+ this.updatePendingCount();
156
+ this.scheduleRequest();
157
+ }
158
+ updatePendingCount() {
159
+ const nextCount = this.pendingClaims.size;
160
+ const difference = nextCount - this.reportedPendingCount;
161
+ this.reportedPendingCount = nextCount;
162
+ if (difference !== 0) collectingChunks.value += difference;
163
+ const nextFetchCounts = { initial: 0, retrying: 0 };
164
+ for (const pending of this.pendingClaims.values()) {
165
+ if (pending.missingResponses > 0) nextFetchCounts.retrying++;
166
+ else nextFetchCounts.initial++;
167
+ }
168
+ const initialDifference = nextFetchCounts.initial - this.reportedFetchCounts.initial;
169
+ const retryingDifference = nextFetchCounts.retrying - this.reportedFetchCounts.retrying;
170
+ this.reportedFetchCounts = nextFetchCounts;
171
+ if (initialDifference !== 0 || retryingDifference !== 0) {
172
+ chunkFetchCounts.value = {
173
+ initial: chunkFetchCounts.value.initial + initialDifference,
174
+ retrying: chunkFetchCounts.value.retrying + retryingDifference
175
+ };
176
+ }
101
177
  }
102
178
  onClaimStalled(pending, stalledIds) {
103
179
  pending.resolveActivityBoundary(false);
104
180
  this.removePendingDelivery(pending, stalledIds);
105
181
  Logger(`Chunk delivery stalled for the following IDs: ${stalledIds.join(", ")}`, LOG_LEVEL_VERBOSE);
106
182
  }
107
- settleClaim(id) {
108
- const pending = this.pendingClaims.get(id);
109
- if (!pending) return;
183
+ settleClaim(id, pending) {
184
+ if (!pending || this.pendingClaims.get(id) !== pending) return;
110
185
  this.pendingClaims.delete(id);
111
- pending.claim.settle(id);
186
+ pending.delivery.claim.settle(id);
187
+ this.updatePendingCount();
112
188
  }
113
- touchClaims(ids) {
114
- const pendingDeliveries = new Set(
115
- ids.map((id) => this.pendingClaims.get(id)).filter((pending) => pending !== void 0)
116
- );
189
+ getActiveClaimIds(ids, claims) {
190
+ return ids.filter((id) => claims.has(id) && this.pendingClaims.get(id) === claims.get(id));
191
+ }
192
+ touchClaims(ids, claims) {
193
+ const pendingDeliveries = new Set(this.getActiveClaimIds(ids, claims).map((id) => claims.get(id).delivery));
117
194
  for (const pending of pendingDeliveries) {
118
195
  pending.claim.touch();
119
196
  }
120
197
  }
121
- async waitForActivityBoundary(ids) {
122
- const pendingDeliveries = new Set(
123
- ids.map((id) => this.pendingClaims.get(id)).filter((pending) => pending !== void 0)
124
- );
198
+ async waitForActivityBoundary(claims) {
199
+ const pendingDeliveries = new Set([...claims.values()].map((pending) => pending.delivery));
125
200
  const entered = new Map(
126
201
  await Promise.all(
127
202
  [...pendingDeliveries].map(async (pending) => [pending, await pending.activityBoundaryEntered])
128
203
  )
129
204
  );
130
- return ids.filter((id) => {
131
- const pending = this.pendingClaims.get(id);
132
- return pending !== void 0 && entered.get(pending) === true;
205
+ return [...claims].flatMap(
206
+ ([id, pending]) => this.pendingClaims.get(id) === pending && entered.get(pending.delivery) === true ? [id] : []
207
+ );
208
+ }
209
+ async waitBeforeRemoteRequest(ids, claims, localCheckCompletionVersion) {
210
+ this.touchClaims(ids, claims);
211
+ while (!this.destroyed) {
212
+ const activeIds = this.getActiveClaimIds(ids, claims);
213
+ if (activeIds.length === 0) return [];
214
+ if (localCheckCompletionVersion !== this.finiteCompletionVersion) {
215
+ localCheckCompletionVersion = this.finiteCompletionVersion;
216
+ await this.resolveLocallyAvailableRetries(claims);
217
+ continue;
218
+ }
219
+ const now = Date.now();
220
+ const timeToWait = this.previousRequestTime + this.interval - now;
221
+ if (timeToWait > 0) {
222
+ await delay(timeToWait);
223
+ continue;
224
+ }
225
+ this.previousRequestTime = now;
226
+ this.touchClaims(activeIds, claims);
227
+ return activeIds;
228
+ }
229
+ return [];
230
+ }
231
+ async resolveLocallyAvailableRetries(claims) {
232
+ const retryIds = this.getActiveClaimIds([...claims.keys()], claims).filter(
233
+ (id) => claims.get(id).missingResponses > 0
234
+ );
235
+ if (retryIds.length === 0) return;
236
+ const localChunks = await this.chunkManager.read(retryIds, {
237
+ preventRemoteRequest: true,
238
+ skipCache: true,
239
+ waitForDelivery: false
133
240
  });
241
+ for (const chunk of localChunks) {
242
+ if (!chunk) continue;
243
+ this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
244
+ this.settleClaim(chunk._id, claims.get(chunk._id));
245
+ }
246
+ }
247
+ logFetchedChunks(requestedIds, { chunks, invalidChunks }) {
248
+ if (invalidChunks.length > 0) {
249
+ Logger(
250
+ `Some fetched chunks are invalid and will be ignored: (${invalidChunks.length} / ${chunks.length + invalidChunks.length}).`,
251
+ LOG_LEVEL_VERBOSE
252
+ );
253
+ for (const chunk of invalidChunks) {
254
+ Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
255
+ }
256
+ }
257
+ if (chunks.length === 0) {
258
+ Logger(`No valid chunks were found for the following IDs: ${requestedIds.join(", ")}`);
259
+ }
260
+ }
261
+ applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion) {
262
+ for (const chunkID of this.getActiveClaimIds(missingIds, requestClaims)) {
263
+ const pending = requestClaims.get(chunkID);
264
+ const completedDuringRequest = requestCompletionVersion !== this.finiteCompletionVersion;
265
+ const action = decideMissingChunkAction({
266
+ missingResponses: pending.missingResponses,
267
+ finiteReplicationActive: this.chunkManager.deliveryCoordinator.isFiniteReplicationActive(),
268
+ completedDuringRequest
269
+ });
270
+ if (action.kind === "missing") {
271
+ this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
272
+ this.settleClaim(chunkID, pending);
273
+ } else {
274
+ pending.missingResponses = action.missingResponses;
275
+ pending.inFlight = false;
276
+ pending.nextRequestAt = Date.now() + action.delayMs;
277
+ this.queue.push(chunkID);
278
+ Logger(
279
+ `Remote chunk is not available yet; retrying ${chunkID} in ${action.delayMs} ms.`,
280
+ LOG_LEVEL_VERBOSE
281
+ );
282
+ }
283
+ }
284
+ this.updatePendingCount();
134
285
  }
135
286
  /**
136
287
  * Processing requests
@@ -143,30 +294,39 @@ class ChunkFetcher {
143
294
  */
144
295
  previousRequestTime = 0;
145
296
  canRequestMore() {
146
- return this.currentProcessing < this.concurrency && this.queue.length > 0;
297
+ return this.currentProcessing < this.concurrency && this.queue.some((id) => this.isReady(id));
298
+ }
299
+ isReady(id) {
300
+ const pending = this.pendingClaims.get(id);
301
+ return !pending || !pending.inFlight && pending.nextRequestAt <= Date.now();
147
302
  }
148
303
  async requestMissingChunks() {
149
304
  if (this.destroyed || !this.canRequestMore()) {
305
+ this.scheduleRequest();
150
306
  return;
151
307
  }
152
308
  let requestIDs = [];
309
+ const requestClaims = /* @__PURE__ */ new Map();
153
310
  try {
154
- let isValidChunk2 = function(chunk) {
155
- return chunk && typeof chunk?._id === "string" && typeof chunk?.data === "string";
156
- };
157
- var isValidChunk = isValidChunk2;
158
311
  this.currentProcessing++;
159
- requestIDs = this.queue.splice(0, BATCH_SIZE);
312
+ requestIDs = unique(this.queue.filter((id) => this.isReady(id))).slice(0, BATCH_SIZE);
313
+ const selectedIds = new Set(requestIDs);
314
+ this.queue = this.queue.filter((id) => !selectedIds.has(id));
160
315
  this.ensureClaims(requestIDs);
161
- requestIDs = await this.waitForActivityBoundary(requestIDs);
316
+ for (const id of requestIDs) {
317
+ const pending = this.pendingClaims.get(id);
318
+ if (pending) {
319
+ pending.inFlight = true;
320
+ requestClaims.set(id, pending);
321
+ }
322
+ }
323
+ this.scheduleRequest();
324
+ requestIDs = await this.waitForActivityBoundary(requestClaims);
162
325
  if (requestIDs.length === 0) return;
163
- this.touchClaims(requestIDs);
164
- const now = Date.now();
165
- const timeSinceLastRequest = now - this.previousRequestTime;
166
- this.previousRequestTime = now;
167
- const timeToWait = Math.max(this.interval - timeSinceLastRequest, 0);
168
- if (timeToWait > 0) await delay(timeToWait);
169
- this.touchClaims(requestIDs);
326
+ const localCheckCompletionVersion = this.finiteCompletionVersion;
327
+ await this.resolveLocallyAvailableRetries(requestClaims);
328
+ let pendingIDs = await this.waitBeforeRemoteRequest(requestIDs, requestClaims, localCheckCompletionVersion);
329
+ if (pendingIDs.length === 0) return;
170
330
  const replicator = this.options.replicatorService.getActiveReplicator();
171
331
  if (!replicator) {
172
332
  Logger("No active replicator was found to request missing chunks.");
@@ -174,58 +334,46 @@ class ChunkFetcher {
174
334
  }
175
335
  if (!canFetchRemoteChunks(replicator)) {
176
336
  Logger("The active replicator does not support fetching individual remote chunks.");
177
- for (const chunkID of requestIDs) {
337
+ for (const chunkID of pendingIDs) {
178
338
  this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
179
339
  }
180
340
  return;
181
341
  }
182
- const fetched = await replicator.fetchRemoteChunks(requestIDs, false);
183
- this.touchClaims(requestIDs);
342
+ const requestCompletionVersion = this.finiteCompletionVersion;
343
+ const fetched = await replicator.fetchRemoteChunks(pendingIDs, false);
344
+ pendingIDs = this.getActiveClaimIds(pendingIDs, requestClaims);
345
+ this.touchClaims(pendingIDs, requestClaims);
184
346
  if (!fetched) {
185
- Logger(`No chunks were found for the following IDs: ${requestIDs.join(", ")}`);
186
- for (const chunkID of requestIDs) {
347
+ Logger(`No chunks were found for the following IDs: ${pendingIDs.join(", ")}`);
348
+ for (const chunkID of pendingIDs) {
187
349
  this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
188
350
  }
189
351
  return;
190
352
  }
191
- const chunks = fetched.filter((chunk) => isValidChunk2(chunk));
192
- if (chunks.length !== fetched.length) {
193
- Logger(
194
- `Some fetched chunks are invalid and will be ignored: (${fetched.length - chunks.length} / ${fetched.length}).`,
195
- LOG_LEVEL_VERBOSE
196
- );
197
- for (const chunk of fetched) {
198
- if (!isValidChunk2(chunk)) {
199
- Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
200
- }
201
- }
202
- }
203
- if (chunks.length === 0) {
204
- Logger(`No valid chunks were found for the following IDs: ${requestIDs.join(", ")}`);
205
- }
206
- const missingIDs = requestIDs.filter((id) => !chunks.some((chunk) => chunk._id === id));
353
+ const classification = classifyFetchedChunks(pendingIDs, fetched);
354
+ this.logFetchedChunks(pendingIDs, classification);
355
+ const { chunks, missingIds, invalidIds } = classification;
207
356
  try {
208
- if (chunks.length === 0) {
209
- return;
210
- }
211
- Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
212
- const result = await this.chunkManager.write(
213
- chunks,
214
- {
215
- skipCache: true,
216
- force: true
217
- // Force writing to ensure the chunks with existing _rev.
218
- },
219
- "ChunkFetcher"
220
- );
221
- this.touchClaims(requestIDs);
222
- if (result.result === true) {
223
- Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
224
- } else {
225
- Logger(
226
- `Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
227
- LOG_LEVEL_VERBOSE
357
+ if (chunks.length > 0) {
358
+ Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
359
+ const result = await this.chunkManager.write(
360
+ chunks,
361
+ {
362
+ skipCache: true,
363
+ force: true
364
+ // Force writing to ensure the chunks with existing _rev.
365
+ },
366
+ "ChunkFetcher"
228
367
  );
368
+ this.touchClaims(pendingIDs, requestClaims);
369
+ if (result.result === true) {
370
+ Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
371
+ } else {
372
+ Logger(
373
+ `Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
374
+ LOG_LEVEL_VERBOSE
375
+ );
376
+ }
229
377
  }
230
378
  } catch (error) {
231
379
  Logger(`An error occurred while storing fetched chunks!`, LOG_LEVEL_VERBOSE);
@@ -233,23 +381,24 @@ class ChunkFetcher {
233
381
  } finally {
234
382
  for (const chunk of chunks) {
235
383
  this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
384
+ this.settleClaim(chunk._id, requestClaims.get(chunk._id));
236
385
  }
237
- for (const chunkID of missingIDs) {
238
- this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
386
+ for (const invalidID of this.getActiveClaimIds(invalidIds, requestClaims)) {
387
+ this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, invalidID);
388
+ this.settleClaim(invalidID, requestClaims.get(invalidID));
239
389
  }
240
390
  }
391
+ this.applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion);
241
392
  } catch (error) {
242
393
  Logger("An error occurred while fetching remote chunks.", LOG_LEVEL_VERBOSE);
243
394
  Logger(error, LOG_LEVEL_VERBOSE);
244
395
  } finally {
245
- for (const id of requestIDs) {
246
- this.settleClaim(id);
396
+ for (const [id, pending] of requestClaims) {
397
+ if (pending.inFlight) this.settleClaim(id, pending);
247
398
  }
248
399
  this.currentProcessing--;
249
400
  this.previousRequestTime = Date.now();
250
- if (this.queue.length > 0) {
251
- compatGlobal.setTimeout(() => void this.requestMissingChunks(), 0);
252
- }
401
+ this.scheduleRequest();
253
402
  }
254
403
  }
255
404
  }
@@ -0,0 +1,21 @@
1
+ import type { DocumentID, EntryLeaf } from "../common/types.js";
2
+ type MissingChunkAction = {
3
+ kind: "missing";
4
+ } | {
5
+ kind: "retry";
6
+ missingResponses: number;
7
+ delayMs: number;
8
+ };
9
+ export declare function decideMissingChunkAction({ missingResponses, finiteReplicationActive, completedDuringRequest, }: {
10
+ missingResponses: number;
11
+ finiteReplicationActive: boolean;
12
+ completedDuringRequest: boolean;
13
+ }): MissingChunkAction;
14
+ export type FetchedChunkClassification = {
15
+ chunks: EntryLeaf[];
16
+ invalidChunks: unknown[];
17
+ missingIds: DocumentID[];
18
+ invalidIds: DocumentID[];
19
+ };
20
+ export declare function classifyFetchedChunks(requestedIds: readonly DocumentID[], fetched: readonly unknown[]): FetchedChunkClassification;
21
+ export {};
@@ -0,0 +1,43 @@
1
+ const REMOTE_MISSING_RETRY_STEP_MS = 2e3;
2
+ const REMOTE_MISSING_RETRY_MAX_DELAY_MS = 1e4;
3
+ function decideMissingChunkAction({
4
+ missingResponses,
5
+ finiteReplicationActive,
6
+ completedDuringRequest
7
+ }) {
8
+ if (missingResponses > 0 && !finiteReplicationActive && !completedDuringRequest) {
9
+ return { kind: "missing" };
10
+ }
11
+ const nextMissingResponses = missingResponses + 1;
12
+ return {
13
+ kind: "retry",
14
+ missingResponses: nextMissingResponses,
15
+ delayMs: completedDuringRequest ? 0 : Math.min(nextMissingResponses * REMOTE_MISSING_RETRY_STEP_MS, REMOTE_MISSING_RETRY_MAX_DELAY_MS)
16
+ };
17
+ }
18
+ function isValidChunk(chunk) {
19
+ const candidate = chunk;
20
+ return !!candidate && typeof candidate._id === "string" && typeof candidate.data === "string";
21
+ }
22
+ function classifyFetchedChunks(requestedIds, fetched) {
23
+ const chunks = [];
24
+ const invalidChunks = [];
25
+ const observedIds = /* @__PURE__ */ new Set();
26
+ for (const chunk of fetched) {
27
+ const candidate = chunk;
28
+ if (typeof candidate?._id === "string") observedIds.add(candidate._id);
29
+ if (isValidChunk(chunk)) chunks.push(chunk);
30
+ else invalidChunks.push(chunk);
31
+ }
32
+ const validIds = new Set(chunks.map((chunk) => chunk._id));
33
+ return {
34
+ chunks,
35
+ invalidChunks,
36
+ missingIds: requestedIds.filter((id) => !observedIds.has(id)),
37
+ invalidIds: requestedIds.filter((id) => observedIds.has(id) && !validIds.has(id))
38
+ };
39
+ }
40
+ export {
41
+ classifyFetchedChunks,
42
+ decideMissingChunkAction
43
+ };
@@ -10,6 +10,10 @@ export declare const lockStats: import("octagonal-wheels/dataobject/reactive_v2"
10
10
  count: number;
11
11
  }>;
12
12
  export declare const collectingChunks: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
13
+ export declare const chunkFetchCounts: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<{
14
+ initial: number;
15
+ retrying: number;
16
+ }>;
13
17
  export declare const pluginScanningCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
14
18
  export declare const hiddenFilesProcessingCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
15
19
  export declare const hiddenFilesEventCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
@@ -1,11 +1,13 @@
1
1
  import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
2
2
  const lockStats = reactiveSource({ pending: [], running: [], count: 0 });
3
3
  const collectingChunks = reactiveSource(0);
4
+ const chunkFetchCounts = reactiveSource({ initial: 0, retrying: 0 });
4
5
  const pluginScanningCount = reactiveSource(0);
5
6
  const hiddenFilesProcessingCount = reactiveSource(0);
6
7
  const hiddenFilesEventCount = reactiveSource(0);
7
8
  const logMessages = reactiveSource([]);
8
9
  export {
10
+ chunkFetchCounts,
9
11
  collectingChunks,
10
12
  hiddenFilesEventCount,
11
13
  hiddenFilesProcessingCount,
@@ -2,6 +2,7 @@ import { type ObsidianLiveSyncSettings, type SettingsMigrationState } from "../.
2
2
  import type { IAPIService, ISettingService } from "./IService.js";
3
3
  import { ServiceBase, type ServiceContext } from "./ServiceBase.js";
4
4
  import { createInstanceLogFunction } from "../lib/logUtils.js";
5
+ import { type PersistedSettings } from "../../common/models/setting.policy.js";
5
6
  export interface SettingServiceDependencies {
6
7
  APIService: IAPIService;
7
8
  /** Optional host hook for applying the loaded display language to its catalogue. */
@@ -17,7 +18,7 @@ export declare abstract class SettingService<T extends ServiceContext = ServiceC
17
18
  _settings: ObsidianLiveSyncSettings;
18
19
  get settings(): ObsidianLiveSyncSettings;
19
20
  set settings(value: ObsidianLiveSyncSettings);
20
- protected abstract saveData(setting: ObsidianLiveSyncSettings): Promise<void>;
21
+ protected abstract saveData(setting: PersistedSettings): Promise<void>;
21
22
  protected abstract loadData(): Promise<ObsidianLiveSyncSettings | undefined>;
22
23
  private _lastPersistedSettings?;
23
24
  private _settingsMigrationState?;