@vrtmrz/livesync-commonlib 0.1.33 → 0.1.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/models/setting.policy.d.ts +808 -0
- package/dist/common/models/setting.policy.js +298 -0
- package/dist/common/utils.patch.js +49 -30
- package/dist/managers/ChunkFetcher.d.ts +16 -0
- package/dist/managers/ChunkFetcher.js +231 -82
- package/dist/managers/chunkFetcher.helper.d.ts +21 -0
- package/dist/managers/chunkFetcher.helper.js +43 -0
- package/dist/mock_and_interop/stores.d.ts +4 -0
- package/dist/mock_and_interop/stores.js +2 -0
- package/dist/services/base/SettingService.d.ts +2 -1
- package/dist/services/base/SettingService.js +42 -63
- package/dist/services/implements/injectable/InjectableSettingService.d.ts +2 -1
- package/dist/settings.d.ts +1 -0
- package/dist/settings.js +10 -0
- package/docs/proven-in-use.md +7 -1
- package/docs/settings-lifecycle.md +13 -1
- package/package.json +1 -1
|
@@ -5,6 +5,11 @@ import { DEFAULT_SETTINGS } from "../common/types.js";
|
|
|
5
5
|
import {} from "./ChunkManager.js";
|
|
6
6
|
import { compatGlobal } from "../common/coreEnvFunctions.js";
|
|
7
7
|
import { DEFAULT_CHUNK_DELIVERY_STALL_TIMEOUT_MS } from "./ChunkDeliveryCoordinator.js";
|
|
8
|
+
import { chunkFetchCounts, collectingChunks } from "../mock_and_interop/stores.js";
|
|
9
|
+
import {
|
|
10
|
+
classifyFetchedChunks,
|
|
11
|
+
decideMissingChunkAction
|
|
12
|
+
} from "./chunkFetcher.helper.js";
|
|
8
13
|
const EVENT_MISSING_CHUNKS = "missingChunks";
|
|
9
14
|
const EVENT_MISSING_CHUNK_REMOTE = "missingChunkRemote";
|
|
10
15
|
const EVENT_CHUNK_FETCHED = "chunkFetched";
|
|
@@ -19,6 +24,13 @@ class ChunkFetcher {
|
|
|
19
24
|
}
|
|
20
25
|
queue = [];
|
|
21
26
|
pendingClaims = /* @__PURE__ */ new Map();
|
|
27
|
+
reportedPendingCount = 0;
|
|
28
|
+
reportedFetchCounts = { initial: 0, retrying: 0 };
|
|
29
|
+
requestTimer;
|
|
30
|
+
requestTimerDueAt = 0;
|
|
31
|
+
finiteActivityWasActive;
|
|
32
|
+
finiteCompletionVersion = 0;
|
|
33
|
+
stopObservingDelivery;
|
|
22
34
|
destroyed = false;
|
|
23
35
|
get interval() {
|
|
24
36
|
const settings = this.options.settingService.currentSettings();
|
|
@@ -31,6 +43,18 @@ class ChunkFetcher {
|
|
|
31
43
|
abort = new AbortController();
|
|
32
44
|
constructor(options) {
|
|
33
45
|
this.options = options;
|
|
46
|
+
this.finiteActivityWasActive = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
|
|
47
|
+
this.stopObservingDelivery = this.chunkManager.deliveryCoordinator.onChanged(() => {
|
|
48
|
+
const active = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
|
|
49
|
+
const completed = this.finiteActivityWasActive && !active;
|
|
50
|
+
this.finiteActivityWasActive = active;
|
|
51
|
+
if (!completed) return;
|
|
52
|
+
this.finiteCompletionVersion++;
|
|
53
|
+
for (const pending of this.pendingClaims.values()) {
|
|
54
|
+
if (pending.missingResponses > 0 && !pending.inFlight) pending.nextRequestAt = Date.now();
|
|
55
|
+
}
|
|
56
|
+
this.scheduleRequest();
|
|
57
|
+
});
|
|
34
58
|
this.chunkManager.addListener(EVENT_MISSING_CHUNKS, this.onEventHandler, {
|
|
35
59
|
signal: this.abort.signal
|
|
36
60
|
});
|
|
@@ -39,9 +63,12 @@ class ChunkFetcher {
|
|
|
39
63
|
if (this.destroyed) return;
|
|
40
64
|
this.destroyed = true;
|
|
41
65
|
this.abort.abort();
|
|
66
|
+
this.stopObservingDelivery();
|
|
67
|
+
this.clearRequestTimer();
|
|
42
68
|
this.queue = [];
|
|
43
|
-
const pendingDeliveries = new Set(this.pendingClaims.values());
|
|
69
|
+
const pendingDeliveries = new Set([...this.pendingClaims.values()].map((pending) => pending.delivery));
|
|
44
70
|
this.pendingClaims.clear();
|
|
71
|
+
this.updatePendingCount();
|
|
45
72
|
for (const pending of pendingDeliveries) {
|
|
46
73
|
pending.resolveActivityBoundary(false);
|
|
47
74
|
pending.claim.release();
|
|
@@ -52,9 +79,35 @@ class ChunkFetcher {
|
|
|
52
79
|
if (this.destroyed) return;
|
|
53
80
|
const claimedIds = this.ensureClaims(ids);
|
|
54
81
|
this.queue = unique([...this.queue, ...claimedIds]);
|
|
55
|
-
|
|
56
|
-
|
|
82
|
+
this.scheduleRequest(1);
|
|
83
|
+
}
|
|
84
|
+
clearRequestTimer() {
|
|
85
|
+
if (this.requestTimer !== void 0) compatGlobal.clearTimeout(this.requestTimer);
|
|
86
|
+
this.requestTimer = void 0;
|
|
87
|
+
}
|
|
88
|
+
scheduleRequest(minimumDelayMs = 0) {
|
|
89
|
+
if (this.destroyed || this.currentProcessing >= this.concurrency || this.queue.length === 0) {
|
|
90
|
+
this.clearRequestTimer();
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
let nextRequestAt = Infinity;
|
|
94
|
+
for (const id of this.queue) {
|
|
95
|
+
const pending = this.pendingClaims.get(id);
|
|
96
|
+
if (!pending?.inFlight) nextRequestAt = Math.min(nextRequestAt, pending?.nextRequestAt ?? 0);
|
|
57
97
|
}
|
|
98
|
+
if (nextRequestAt === Infinity) {
|
|
99
|
+
this.clearRequestTimer();
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const now = Date.now();
|
|
103
|
+
const dueAt = Math.max(nextRequestAt, now + minimumDelayMs);
|
|
104
|
+
if (this.requestTimer !== void 0 && this.requestTimerDueAt <= dueAt) return;
|
|
105
|
+
this.clearRequestTimer();
|
|
106
|
+
this.requestTimerDueAt = dueAt;
|
|
107
|
+
this.requestTimer = compatGlobal.setTimeout(() => {
|
|
108
|
+
this.requestTimer = void 0;
|
|
109
|
+
void this.requestMissingChunks();
|
|
110
|
+
}, dueAt - now);
|
|
58
111
|
}
|
|
59
112
|
ensureClaims(ids) {
|
|
60
113
|
const unclaimedIds = unique(ids.filter((id) => !this.pendingClaims.has(id)));
|
|
@@ -71,8 +124,9 @@ class ChunkFetcher {
|
|
|
71
124
|
resolveActivityBoundary: activityBoundary.resolve
|
|
72
125
|
};
|
|
73
126
|
for (const id of unclaimedIds) {
|
|
74
|
-
this.pendingClaims.set(id, pending);
|
|
127
|
+
this.pendingClaims.set(id, { delivery: pending, missingResponses: 0, nextRequestAt: 0, inFlight: false });
|
|
75
128
|
}
|
|
129
|
+
this.updatePendingCount();
|
|
76
130
|
void this.options.replicatorService.runBoundedRemoteActivity(
|
|
77
131
|
() => {
|
|
78
132
|
pending.claim.touch();
|
|
@@ -92,45 +146,142 @@ class ChunkFetcher {
|
|
|
92
146
|
}
|
|
93
147
|
removePendingDelivery(pending, ids) {
|
|
94
148
|
for (const id of ids) {
|
|
95
|
-
if (this.pendingClaims.get(id) === pending) {
|
|
149
|
+
if (this.pendingClaims.get(id)?.delivery === pending) {
|
|
96
150
|
this.pendingClaims.delete(id);
|
|
97
151
|
}
|
|
98
152
|
}
|
|
99
153
|
const removed = new Set(ids);
|
|
100
154
|
this.queue = this.queue.filter((id) => !(removed.has(id) && !this.pendingClaims.has(id)));
|
|
155
|
+
this.updatePendingCount();
|
|
156
|
+
this.scheduleRequest();
|
|
157
|
+
}
|
|
158
|
+
updatePendingCount() {
|
|
159
|
+
const nextCount = this.pendingClaims.size;
|
|
160
|
+
const difference = nextCount - this.reportedPendingCount;
|
|
161
|
+
this.reportedPendingCount = nextCount;
|
|
162
|
+
if (difference !== 0) collectingChunks.value += difference;
|
|
163
|
+
const nextFetchCounts = { initial: 0, retrying: 0 };
|
|
164
|
+
for (const pending of this.pendingClaims.values()) {
|
|
165
|
+
if (pending.missingResponses > 0) nextFetchCounts.retrying++;
|
|
166
|
+
else nextFetchCounts.initial++;
|
|
167
|
+
}
|
|
168
|
+
const initialDifference = nextFetchCounts.initial - this.reportedFetchCounts.initial;
|
|
169
|
+
const retryingDifference = nextFetchCounts.retrying - this.reportedFetchCounts.retrying;
|
|
170
|
+
this.reportedFetchCounts = nextFetchCounts;
|
|
171
|
+
if (initialDifference !== 0 || retryingDifference !== 0) {
|
|
172
|
+
chunkFetchCounts.value = {
|
|
173
|
+
initial: chunkFetchCounts.value.initial + initialDifference,
|
|
174
|
+
retrying: chunkFetchCounts.value.retrying + retryingDifference
|
|
175
|
+
};
|
|
176
|
+
}
|
|
101
177
|
}
|
|
102
178
|
onClaimStalled(pending, stalledIds) {
|
|
103
179
|
pending.resolveActivityBoundary(false);
|
|
104
180
|
this.removePendingDelivery(pending, stalledIds);
|
|
105
181
|
Logger(`Chunk delivery stalled for the following IDs: ${stalledIds.join(", ")}`, LOG_LEVEL_VERBOSE);
|
|
106
182
|
}
|
|
107
|
-
settleClaim(id) {
|
|
108
|
-
|
|
109
|
-
if (!pending) return;
|
|
183
|
+
settleClaim(id, pending) {
|
|
184
|
+
if (!pending || this.pendingClaims.get(id) !== pending) return;
|
|
110
185
|
this.pendingClaims.delete(id);
|
|
111
|
-
pending.claim.settle(id);
|
|
186
|
+
pending.delivery.claim.settle(id);
|
|
187
|
+
this.updatePendingCount();
|
|
112
188
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
189
|
+
getActiveClaimIds(ids, claims) {
|
|
190
|
+
return ids.filter((id) => claims.has(id) && this.pendingClaims.get(id) === claims.get(id));
|
|
191
|
+
}
|
|
192
|
+
touchClaims(ids, claims) {
|
|
193
|
+
const pendingDeliveries = new Set(this.getActiveClaimIds(ids, claims).map((id) => claims.get(id).delivery));
|
|
117
194
|
for (const pending of pendingDeliveries) {
|
|
118
195
|
pending.claim.touch();
|
|
119
196
|
}
|
|
120
197
|
}
|
|
121
|
-
async waitForActivityBoundary(
|
|
122
|
-
const pendingDeliveries = new Set(
|
|
123
|
-
ids.map((id) => this.pendingClaims.get(id)).filter((pending) => pending !== void 0)
|
|
124
|
-
);
|
|
198
|
+
async waitForActivityBoundary(claims) {
|
|
199
|
+
const pendingDeliveries = new Set([...claims.values()].map((pending) => pending.delivery));
|
|
125
200
|
const entered = new Map(
|
|
126
201
|
await Promise.all(
|
|
127
202
|
[...pendingDeliveries].map(async (pending) => [pending, await pending.activityBoundaryEntered])
|
|
128
203
|
)
|
|
129
204
|
);
|
|
130
|
-
return
|
|
131
|
-
|
|
132
|
-
|
|
205
|
+
return [...claims].flatMap(
|
|
206
|
+
([id, pending]) => this.pendingClaims.get(id) === pending && entered.get(pending.delivery) === true ? [id] : []
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
async waitBeforeRemoteRequest(ids, claims, localCheckCompletionVersion) {
|
|
210
|
+
this.touchClaims(ids, claims);
|
|
211
|
+
while (!this.destroyed) {
|
|
212
|
+
const activeIds = this.getActiveClaimIds(ids, claims);
|
|
213
|
+
if (activeIds.length === 0) return [];
|
|
214
|
+
if (localCheckCompletionVersion !== this.finiteCompletionVersion) {
|
|
215
|
+
localCheckCompletionVersion = this.finiteCompletionVersion;
|
|
216
|
+
await this.resolveLocallyAvailableRetries(claims);
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
const now = Date.now();
|
|
220
|
+
const timeToWait = this.previousRequestTime + this.interval - now;
|
|
221
|
+
if (timeToWait > 0) {
|
|
222
|
+
await delay(timeToWait);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
this.previousRequestTime = now;
|
|
226
|
+
this.touchClaims(activeIds, claims);
|
|
227
|
+
return activeIds;
|
|
228
|
+
}
|
|
229
|
+
return [];
|
|
230
|
+
}
|
|
231
|
+
async resolveLocallyAvailableRetries(claims) {
|
|
232
|
+
const retryIds = this.getActiveClaimIds([...claims.keys()], claims).filter(
|
|
233
|
+
(id) => claims.get(id).missingResponses > 0
|
|
234
|
+
);
|
|
235
|
+
if (retryIds.length === 0) return;
|
|
236
|
+
const localChunks = await this.chunkManager.read(retryIds, {
|
|
237
|
+
preventRemoteRequest: true,
|
|
238
|
+
skipCache: true,
|
|
239
|
+
waitForDelivery: false
|
|
133
240
|
});
|
|
241
|
+
for (const chunk of localChunks) {
|
|
242
|
+
if (!chunk) continue;
|
|
243
|
+
this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
|
|
244
|
+
this.settleClaim(chunk._id, claims.get(chunk._id));
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
logFetchedChunks(requestedIds, { chunks, invalidChunks }) {
|
|
248
|
+
if (invalidChunks.length > 0) {
|
|
249
|
+
Logger(
|
|
250
|
+
`Some fetched chunks are invalid and will be ignored: (${invalidChunks.length} / ${chunks.length + invalidChunks.length}).`,
|
|
251
|
+
LOG_LEVEL_VERBOSE
|
|
252
|
+
);
|
|
253
|
+
for (const chunk of invalidChunks) {
|
|
254
|
+
Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (chunks.length === 0) {
|
|
258
|
+
Logger(`No valid chunks were found for the following IDs: ${requestedIds.join(", ")}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion) {
|
|
262
|
+
for (const chunkID of this.getActiveClaimIds(missingIds, requestClaims)) {
|
|
263
|
+
const pending = requestClaims.get(chunkID);
|
|
264
|
+
const completedDuringRequest = requestCompletionVersion !== this.finiteCompletionVersion;
|
|
265
|
+
const action = decideMissingChunkAction({
|
|
266
|
+
missingResponses: pending.missingResponses,
|
|
267
|
+
finiteReplicationActive: this.chunkManager.deliveryCoordinator.isFiniteReplicationActive(),
|
|
268
|
+
completedDuringRequest
|
|
269
|
+
});
|
|
270
|
+
if (action.kind === "missing") {
|
|
271
|
+
this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
|
|
272
|
+
this.settleClaim(chunkID, pending);
|
|
273
|
+
} else {
|
|
274
|
+
pending.missingResponses = action.missingResponses;
|
|
275
|
+
pending.inFlight = false;
|
|
276
|
+
pending.nextRequestAt = Date.now() + action.delayMs;
|
|
277
|
+
this.queue.push(chunkID);
|
|
278
|
+
Logger(
|
|
279
|
+
`Remote chunk is not available yet; retrying ${chunkID} in ${action.delayMs} ms.`,
|
|
280
|
+
LOG_LEVEL_VERBOSE
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
this.updatePendingCount();
|
|
134
285
|
}
|
|
135
286
|
/**
|
|
136
287
|
* Processing requests
|
|
@@ -143,30 +294,39 @@ class ChunkFetcher {
|
|
|
143
294
|
*/
|
|
144
295
|
previousRequestTime = 0;
|
|
145
296
|
canRequestMore() {
|
|
146
|
-
return this.currentProcessing < this.concurrency && this.queue.
|
|
297
|
+
return this.currentProcessing < this.concurrency && this.queue.some((id) => this.isReady(id));
|
|
298
|
+
}
|
|
299
|
+
isReady(id) {
|
|
300
|
+
const pending = this.pendingClaims.get(id);
|
|
301
|
+
return !pending || !pending.inFlight && pending.nextRequestAt <= Date.now();
|
|
147
302
|
}
|
|
148
303
|
async requestMissingChunks() {
|
|
149
304
|
if (this.destroyed || !this.canRequestMore()) {
|
|
305
|
+
this.scheduleRequest();
|
|
150
306
|
return;
|
|
151
307
|
}
|
|
152
308
|
let requestIDs = [];
|
|
309
|
+
const requestClaims = /* @__PURE__ */ new Map();
|
|
153
310
|
try {
|
|
154
|
-
let isValidChunk2 = function(chunk) {
|
|
155
|
-
return chunk && typeof chunk?._id === "string" && typeof chunk?.data === "string";
|
|
156
|
-
};
|
|
157
|
-
var isValidChunk = isValidChunk2;
|
|
158
311
|
this.currentProcessing++;
|
|
159
|
-
requestIDs = this.queue.
|
|
312
|
+
requestIDs = unique(this.queue.filter((id) => this.isReady(id))).slice(0, BATCH_SIZE);
|
|
313
|
+
const selectedIds = new Set(requestIDs);
|
|
314
|
+
this.queue = this.queue.filter((id) => !selectedIds.has(id));
|
|
160
315
|
this.ensureClaims(requestIDs);
|
|
161
|
-
|
|
316
|
+
for (const id of requestIDs) {
|
|
317
|
+
const pending = this.pendingClaims.get(id);
|
|
318
|
+
if (pending) {
|
|
319
|
+
pending.inFlight = true;
|
|
320
|
+
requestClaims.set(id, pending);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
this.scheduleRequest();
|
|
324
|
+
requestIDs = await this.waitForActivityBoundary(requestClaims);
|
|
162
325
|
if (requestIDs.length === 0) return;
|
|
163
|
-
this.
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
const timeToWait = Math.max(this.interval - timeSinceLastRequest, 0);
|
|
168
|
-
if (timeToWait > 0) await delay(timeToWait);
|
|
169
|
-
this.touchClaims(requestIDs);
|
|
326
|
+
const localCheckCompletionVersion = this.finiteCompletionVersion;
|
|
327
|
+
await this.resolveLocallyAvailableRetries(requestClaims);
|
|
328
|
+
let pendingIDs = await this.waitBeforeRemoteRequest(requestIDs, requestClaims, localCheckCompletionVersion);
|
|
329
|
+
if (pendingIDs.length === 0) return;
|
|
170
330
|
const replicator = this.options.replicatorService.getActiveReplicator();
|
|
171
331
|
if (!replicator) {
|
|
172
332
|
Logger("No active replicator was found to request missing chunks.");
|
|
@@ -174,58 +334,46 @@ class ChunkFetcher {
|
|
|
174
334
|
}
|
|
175
335
|
if (!canFetchRemoteChunks(replicator)) {
|
|
176
336
|
Logger("The active replicator does not support fetching individual remote chunks.");
|
|
177
|
-
for (const chunkID of
|
|
337
|
+
for (const chunkID of pendingIDs) {
|
|
178
338
|
this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
|
|
179
339
|
}
|
|
180
340
|
return;
|
|
181
341
|
}
|
|
182
|
-
const
|
|
183
|
-
|
|
342
|
+
const requestCompletionVersion = this.finiteCompletionVersion;
|
|
343
|
+
const fetched = await replicator.fetchRemoteChunks(pendingIDs, false);
|
|
344
|
+
pendingIDs = this.getActiveClaimIds(pendingIDs, requestClaims);
|
|
345
|
+
this.touchClaims(pendingIDs, requestClaims);
|
|
184
346
|
if (!fetched) {
|
|
185
|
-
Logger(`No chunks were found for the following IDs: ${
|
|
186
|
-
for (const chunkID of
|
|
347
|
+
Logger(`No chunks were found for the following IDs: ${pendingIDs.join(", ")}`);
|
|
348
|
+
for (const chunkID of pendingIDs) {
|
|
187
349
|
this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
|
|
188
350
|
}
|
|
189
351
|
return;
|
|
190
352
|
}
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
`Some fetched chunks are invalid and will be ignored: (${fetched.length - chunks.length} / ${fetched.length}).`,
|
|
195
|
-
LOG_LEVEL_VERBOSE
|
|
196
|
-
);
|
|
197
|
-
for (const chunk of fetched) {
|
|
198
|
-
if (!isValidChunk2(chunk)) {
|
|
199
|
-
Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
if (chunks.length === 0) {
|
|
204
|
-
Logger(`No valid chunks were found for the following IDs: ${requestIDs.join(", ")}`);
|
|
205
|
-
}
|
|
206
|
-
const missingIDs = requestIDs.filter((id) => !chunks.some((chunk) => chunk._id === id));
|
|
353
|
+
const classification = classifyFetchedChunks(pendingIDs, fetched);
|
|
354
|
+
this.logFetchedChunks(pendingIDs, classification);
|
|
355
|
+
const { chunks, missingIds, invalidIds } = classification;
|
|
207
356
|
try {
|
|
208
|
-
if (chunks.length
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
},
|
|
219
|
-
"ChunkFetcher"
|
|
220
|
-
);
|
|
221
|
-
this.touchClaims(requestIDs);
|
|
222
|
-
if (result.result === true) {
|
|
223
|
-
Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
|
|
224
|
-
} else {
|
|
225
|
-
Logger(
|
|
226
|
-
`Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
|
|
227
|
-
LOG_LEVEL_VERBOSE
|
|
357
|
+
if (chunks.length > 0) {
|
|
358
|
+
Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
|
|
359
|
+
const result = await this.chunkManager.write(
|
|
360
|
+
chunks,
|
|
361
|
+
{
|
|
362
|
+
skipCache: true,
|
|
363
|
+
force: true
|
|
364
|
+
// Force writing to ensure the chunks with existing _rev.
|
|
365
|
+
},
|
|
366
|
+
"ChunkFetcher"
|
|
228
367
|
);
|
|
368
|
+
this.touchClaims(pendingIDs, requestClaims);
|
|
369
|
+
if (result.result === true) {
|
|
370
|
+
Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
|
|
371
|
+
} else {
|
|
372
|
+
Logger(
|
|
373
|
+
`Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
|
|
374
|
+
LOG_LEVEL_VERBOSE
|
|
375
|
+
);
|
|
376
|
+
}
|
|
229
377
|
}
|
|
230
378
|
} catch (error) {
|
|
231
379
|
Logger(`An error occurred while storing fetched chunks!`, LOG_LEVEL_VERBOSE);
|
|
@@ -233,23 +381,24 @@ class ChunkFetcher {
|
|
|
233
381
|
} finally {
|
|
234
382
|
for (const chunk of chunks) {
|
|
235
383
|
this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
|
|
384
|
+
this.settleClaim(chunk._id, requestClaims.get(chunk._id));
|
|
236
385
|
}
|
|
237
|
-
for (const
|
|
238
|
-
this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE,
|
|
386
|
+
for (const invalidID of this.getActiveClaimIds(invalidIds, requestClaims)) {
|
|
387
|
+
this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, invalidID);
|
|
388
|
+
this.settleClaim(invalidID, requestClaims.get(invalidID));
|
|
239
389
|
}
|
|
240
390
|
}
|
|
391
|
+
this.applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion);
|
|
241
392
|
} catch (error) {
|
|
242
393
|
Logger("An error occurred while fetching remote chunks.", LOG_LEVEL_VERBOSE);
|
|
243
394
|
Logger(error, LOG_LEVEL_VERBOSE);
|
|
244
395
|
} finally {
|
|
245
|
-
for (const id of
|
|
246
|
-
this.settleClaim(id);
|
|
396
|
+
for (const [id, pending] of requestClaims) {
|
|
397
|
+
if (pending.inFlight) this.settleClaim(id, pending);
|
|
247
398
|
}
|
|
248
399
|
this.currentProcessing--;
|
|
249
400
|
this.previousRequestTime = Date.now();
|
|
250
|
-
|
|
251
|
-
compatGlobal.setTimeout(() => void this.requestMissingChunks(), 0);
|
|
252
|
-
}
|
|
401
|
+
this.scheduleRequest();
|
|
253
402
|
}
|
|
254
403
|
}
|
|
255
404
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { DocumentID, EntryLeaf } from "../common/types.js";
|
|
2
|
+
type MissingChunkAction = {
|
|
3
|
+
kind: "missing";
|
|
4
|
+
} | {
|
|
5
|
+
kind: "retry";
|
|
6
|
+
missingResponses: number;
|
|
7
|
+
delayMs: number;
|
|
8
|
+
};
|
|
9
|
+
export declare function decideMissingChunkAction({ missingResponses, finiteReplicationActive, completedDuringRequest, }: {
|
|
10
|
+
missingResponses: number;
|
|
11
|
+
finiteReplicationActive: boolean;
|
|
12
|
+
completedDuringRequest: boolean;
|
|
13
|
+
}): MissingChunkAction;
|
|
14
|
+
export type FetchedChunkClassification = {
|
|
15
|
+
chunks: EntryLeaf[];
|
|
16
|
+
invalidChunks: unknown[];
|
|
17
|
+
missingIds: DocumentID[];
|
|
18
|
+
invalidIds: DocumentID[];
|
|
19
|
+
};
|
|
20
|
+
export declare function classifyFetchedChunks(requestedIds: readonly DocumentID[], fetched: readonly unknown[]): FetchedChunkClassification;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
const REMOTE_MISSING_RETRY_STEP_MS = 2e3;
|
|
2
|
+
const REMOTE_MISSING_RETRY_MAX_DELAY_MS = 1e4;
|
|
3
|
+
function decideMissingChunkAction({
|
|
4
|
+
missingResponses,
|
|
5
|
+
finiteReplicationActive,
|
|
6
|
+
completedDuringRequest
|
|
7
|
+
}) {
|
|
8
|
+
if (missingResponses > 0 && !finiteReplicationActive && !completedDuringRequest) {
|
|
9
|
+
return { kind: "missing" };
|
|
10
|
+
}
|
|
11
|
+
const nextMissingResponses = missingResponses + 1;
|
|
12
|
+
return {
|
|
13
|
+
kind: "retry",
|
|
14
|
+
missingResponses: nextMissingResponses,
|
|
15
|
+
delayMs: completedDuringRequest ? 0 : Math.min(nextMissingResponses * REMOTE_MISSING_RETRY_STEP_MS, REMOTE_MISSING_RETRY_MAX_DELAY_MS)
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function isValidChunk(chunk) {
|
|
19
|
+
const candidate = chunk;
|
|
20
|
+
return !!candidate && typeof candidate._id === "string" && typeof candidate.data === "string";
|
|
21
|
+
}
|
|
22
|
+
function classifyFetchedChunks(requestedIds, fetched) {
|
|
23
|
+
const chunks = [];
|
|
24
|
+
const invalidChunks = [];
|
|
25
|
+
const observedIds = /* @__PURE__ */ new Set();
|
|
26
|
+
for (const chunk of fetched) {
|
|
27
|
+
const candidate = chunk;
|
|
28
|
+
if (typeof candidate?._id === "string") observedIds.add(candidate._id);
|
|
29
|
+
if (isValidChunk(chunk)) chunks.push(chunk);
|
|
30
|
+
else invalidChunks.push(chunk);
|
|
31
|
+
}
|
|
32
|
+
const validIds = new Set(chunks.map((chunk) => chunk._id));
|
|
33
|
+
return {
|
|
34
|
+
chunks,
|
|
35
|
+
invalidChunks,
|
|
36
|
+
missingIds: requestedIds.filter((id) => !observedIds.has(id)),
|
|
37
|
+
invalidIds: requestedIds.filter((id) => observedIds.has(id) && !validIds.has(id))
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export {
|
|
41
|
+
classifyFetchedChunks,
|
|
42
|
+
decideMissingChunkAction
|
|
43
|
+
};
|
|
@@ -10,6 +10,10 @@ export declare const lockStats: import("octagonal-wheels/dataobject/reactive_v2"
|
|
|
10
10
|
count: number;
|
|
11
11
|
}>;
|
|
12
12
|
export declare const collectingChunks: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
|
|
13
|
+
export declare const chunkFetchCounts: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<{
|
|
14
|
+
initial: number;
|
|
15
|
+
retrying: number;
|
|
16
|
+
}>;
|
|
13
17
|
export declare const pluginScanningCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
|
|
14
18
|
export declare const hiddenFilesProcessingCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
|
|
15
19
|
export declare const hiddenFilesEventCount: import("octagonal-wheels/dataobject/reactive_v2").ReactiveSource<number>;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
|
|
2
2
|
const lockStats = reactiveSource({ pending: [], running: [], count: 0 });
|
|
3
3
|
const collectingChunks = reactiveSource(0);
|
|
4
|
+
const chunkFetchCounts = reactiveSource({ initial: 0, retrying: 0 });
|
|
4
5
|
const pluginScanningCount = reactiveSource(0);
|
|
5
6
|
const hiddenFilesProcessingCount = reactiveSource(0);
|
|
6
7
|
const hiddenFilesEventCount = reactiveSource(0);
|
|
7
8
|
const logMessages = reactiveSource([]);
|
|
8
9
|
export {
|
|
10
|
+
chunkFetchCounts,
|
|
9
11
|
collectingChunks,
|
|
10
12
|
hiddenFilesEventCount,
|
|
11
13
|
hiddenFilesProcessingCount,
|
|
@@ -2,6 +2,7 @@ import { type ObsidianLiveSyncSettings, type SettingsMigrationState } from "../.
|
|
|
2
2
|
import type { IAPIService, ISettingService } from "./IService.js";
|
|
3
3
|
import { ServiceBase, type ServiceContext } from "./ServiceBase.js";
|
|
4
4
|
import { createInstanceLogFunction } from "../lib/logUtils.js";
|
|
5
|
+
import { type PersistedSettings } from "../../common/models/setting.policy.js";
|
|
5
6
|
export interface SettingServiceDependencies {
|
|
6
7
|
APIService: IAPIService;
|
|
7
8
|
/** Optional host hook for applying the loaded display language to its catalogue. */
|
|
@@ -17,7 +18,7 @@ export declare abstract class SettingService<T extends ServiceContext = ServiceC
|
|
|
17
18
|
_settings: ObsidianLiveSyncSettings;
|
|
18
19
|
get settings(): ObsidianLiveSyncSettings;
|
|
19
20
|
set settings(value: ObsidianLiveSyncSettings);
|
|
20
|
-
protected abstract saveData(setting:
|
|
21
|
+
protected abstract saveData(setting: PersistedSettings): Promise<void>;
|
|
21
22
|
protected abstract loadData(): Promise<ObsidianLiveSyncSettings | undefined>;
|
|
22
23
|
private _lastPersistedSettings?;
|
|
23
24
|
private _settingsMigrationState?;
|