@indigoai-us/hq-cloud 6.18.51-alpha.1 → 6.18.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +0 -5
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +8 -44
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +5 -85
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +126 -1094
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +15 -36
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +20 -144
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/share.d.ts +0 -12
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +52 -91
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +7 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +30 -41
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +31 -3
- package/dist/cli/tombstones.d.ts.map +1 -1
- package/dist/cli/tombstones.js +142 -11
- package/dist/cli/tombstones.js.map +1 -1
- package/dist/ignore.d.ts +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +2 -2
- package/dist/ignore.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts +8 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +57 -27
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal.d.ts +5 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +31 -203
- package/dist/journal.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +2 -9
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +2 -34
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +76 -14
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/object-io.d.ts +19 -6
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +86 -15
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault-exclusions.d.ts +2 -2
- package/dist/personal-vault-exclusions.d.ts.map +1 -1
- package/dist/personal-vault-exclusions.js +3 -5
- package/dist/personal-vault-exclusions.js.map +1 -1
- package/dist/personal-vault.d.ts +0 -7
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +0 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/s3.d.ts +3 -7
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +10 -4
- package/dist/s3.js.map +1 -1
- package/dist/symlink-target-policy.d.ts +0 -11
- package/dist/symlink-target-policy.d.ts.map +1 -1
- package/dist/symlink-target-policy.js +0 -80
- package/dist/symlink-target-policy.js.map +1 -1
- package/dist/sync/mutation-session.d.ts.map +1 -1
- package/dist/sync/mutation-session.js +0 -4
- package/dist/sync/mutation-session.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +0 -29
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +16 -229
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +10 -32
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +1 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/watcher.d.ts +3 -16
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +21 -114
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/dist/bin/watcher-catchup-skip-store.d.ts +0 -60
- package/dist/bin/watcher-catchup-skip-store.d.ts.map +0 -1
- package/dist/bin/watcher-catchup-skip-store.js +0 -282
- package/dist/bin/watcher-catchup-skip-store.js.map +0 -1
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
5
4
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
6
5
|
import * as v8 from "node:v8";
|
|
7
6
|
import { performance } from "node:perf_hooks";
|
|
@@ -16,18 +15,13 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
|
16
15
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
17
16
|
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
18
17
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
19
|
-
import { isEphemeralPath } from "../cli/share.js";
|
|
20
|
-
import { isSafeRelativePath } from "../sync/push-event.js";
|
|
21
|
-
import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
|
|
22
|
-
import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
|
|
23
18
|
import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
|
|
24
19
|
import { createIgnoreFilter } from "../ignore.js";
|
|
25
|
-
import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
26
20
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
27
21
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
28
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync,
|
|
29
|
-
import { isPersonalVaultDeletionExcluded
|
|
30
|
-
import { NoopPushReceiver, ReceiverDispatchError,
|
|
22
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
|
|
23
|
+
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
24
|
+
import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
31
25
|
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
32
26
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
33
27
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
@@ -67,13 +61,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
67
61
|
*/
|
|
68
62
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
69
63
|
/**
|
|
70
|
-
*
|
|
64
|
+
* How many realtime executes may run during one chore hand-off.
|
|
71
65
|
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
72
|
-
* workers. One long
|
|
73
|
-
*
|
|
74
|
-
* work is never aborted
|
|
66
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
67
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
68
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
75
69
|
*/
|
|
76
|
-
export const REALTIME_WORKER_OCCUPANCY_CAP =
|
|
70
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
77
71
|
/** Objects at least one multipart part are sent through the chore hand-off. */
|
|
78
72
|
export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
|
|
79
73
|
let realtimeAdmissionDeadlineMsForTest;
|
|
@@ -97,29 +91,6 @@ function realtimeDrainBurstMs() {
|
|
|
97
91
|
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
98
92
|
*/
|
|
99
93
|
export const REALTIME_ADMISSION_RETRY_MS = 5_000;
|
|
100
|
-
/** Bound consecutive pull admissions while a watcher push is queued. */
|
|
101
|
-
export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
|
|
102
|
-
/** Bound synchronous warm-up candidate work between event-loop turns. */
|
|
103
|
-
export const WATCHER_WARMUP_YIELD_EVERY = 128;
|
|
104
|
-
export function shouldYieldWatcherWarmup(examinedPaths) {
|
|
105
|
-
return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
|
|
106
|
-
}
|
|
107
|
-
export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
|
|
108
|
-
return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
|
|
109
|
-
}
|
|
110
|
-
export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
|
|
111
|
-
// TreeWatcher marks this only when its production filter has already applied
|
|
112
|
-
// the same personal scope predicate to every collected candidate. Unrouted
|
|
113
|
-
// paths retain the runner check because the collector may include company paths.
|
|
114
|
-
return pushFiltersApplied && routeKind === "personal";
|
|
115
|
-
}
|
|
116
|
-
export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
|
|
117
|
-
const ignoreFilter = createIgnore(hqRoot);
|
|
118
|
-
const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
|
|
119
|
-
onPersonalExcluded(relativePath, match.id);
|
|
120
|
-
});
|
|
121
|
-
return { ignoreFilter, personalPushFilter };
|
|
122
|
-
}
|
|
123
94
|
/**
|
|
124
95
|
* In-process result of a scoped drain whose only shortfall was admission
|
|
125
96
|
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
@@ -220,32 +191,6 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
|
|
|
220
191
|
return explicitlyAllowed ||
|
|
221
192
|
(workClass === "realtime" && isFullPassDurableProgressEnabled());
|
|
222
193
|
}
|
|
223
|
-
export function resolveWatcherPushWaitStage(input) {
|
|
224
|
-
if (input.capYieldActive)
|
|
225
|
-
return "CPU/heap cap wait";
|
|
226
|
-
if (input.holders.some((holder) => holder.kind === "pull"))
|
|
227
|
-
return "pull-yield wait";
|
|
228
|
-
if (input.queueDepth > 0)
|
|
229
|
-
return "queued behind realtime work";
|
|
230
|
-
if (input.holders.some((holder) => holder.kind === "chore"))
|
|
231
|
-
return "chore holder wait";
|
|
232
|
-
if (input.holders.length > 0)
|
|
233
|
-
return "scheduler holder wait";
|
|
234
|
-
return "scheduler admission pending";
|
|
235
|
-
}
|
|
236
|
-
export function watcherPushWaitDiagnostics(input) {
|
|
237
|
-
return {
|
|
238
|
-
stage: input.admissionWaiting
|
|
239
|
-
? resolveWatcherPushWaitStage(input)
|
|
240
|
-
: input.stage,
|
|
241
|
-
waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
|
|
242
|
-
? null
|
|
243
|
-
: Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
|
|
244
|
-
schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
|
|
245
|
-
? null
|
|
246
|
-
: Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
194
|
/**
|
|
250
195
|
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
251
196
|
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
@@ -260,8 +205,6 @@ export class CooperativePassScheduler {
|
|
|
260
205
|
starvationCapMs;
|
|
261
206
|
options;
|
|
262
207
|
pending = [];
|
|
263
|
-
lastAnyAdmissionAtMonotonicMs = null;
|
|
264
|
-
nextEntryId = 0;
|
|
265
208
|
active = null;
|
|
266
209
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
267
210
|
realtimeOccupants = new Set();
|
|
@@ -298,58 +241,12 @@ export class CooperativePassScheduler {
|
|
|
298
241
|
/** Successful callers still applying results after the network hand-off. */
|
|
299
242
|
networkTransferCompletionCount = 0;
|
|
300
243
|
networkTransferApplyRelease = null;
|
|
301
|
-
consecutivePullAdmissions = 0;
|
|
302
244
|
resolveNetworkTransferApplyRelease = null;
|
|
303
245
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
304
246
|
this.now = now;
|
|
305
247
|
this.starvationCapMs = starvationCapMs;
|
|
306
248
|
this.options = options;
|
|
307
249
|
}
|
|
308
|
-
recordAdmission(entry) {
|
|
309
|
-
const monotonicMs = performance.now();
|
|
310
|
-
const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
|
|
311
|
-
? null
|
|
312
|
-
: Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
|
|
313
|
-
const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
314
|
-
const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
|
|
315
|
-
const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
|
|
316
|
-
const waitingSinceAdmissionRequestMs = oldestPush
|
|
317
|
-
? Math.max(0, this.now() - oldestPush.enqueuedAt)
|
|
318
|
-
: null;
|
|
319
|
-
if (waitingPush || admittedIsPush) {
|
|
320
|
-
this.options.onAdmission?.({
|
|
321
|
-
entryId: entry.entryId,
|
|
322
|
-
workClass: entry.workClass,
|
|
323
|
-
lane: entry.lane,
|
|
324
|
-
transferDirection: entry.transferDirection,
|
|
325
|
-
transferSizeClass: entry.transferSizeClass ?? "unknown",
|
|
326
|
-
label: entry.label.slice(0, 160),
|
|
327
|
-
admittedAt: new Date().toISOString(),
|
|
328
|
-
schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
|
|
329
|
-
realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
|
|
330
|
-
oldestWaitingWatcherPush: oldestPush
|
|
331
|
-
? {
|
|
332
|
-
entryId: oldestPush.entryId,
|
|
333
|
-
label: oldestPush.label.slice(0, 160),
|
|
334
|
-
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
335
|
-
}
|
|
336
|
-
: null,
|
|
337
|
-
passedOver: waitingPush && waitingPush !== entry
|
|
338
|
-
? {
|
|
339
|
-
entryId: waitingPush.entryId,
|
|
340
|
-
label: waitingPush.label.slice(0, 160),
|
|
341
|
-
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
342
|
-
reason: entry.transferDirection === "pull"
|
|
343
|
-
? "pull-admitted-ahead-of-waiting-push"
|
|
344
|
-
: this._handoffActive
|
|
345
|
-
? "handoff-selector-admitted-other-work"
|
|
346
|
-
: "scheduler-admitted-other-work",
|
|
347
|
-
}
|
|
348
|
-
: null,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
|
|
352
|
-
}
|
|
353
250
|
get handoffActive() {
|
|
354
251
|
return this._handoffActive;
|
|
355
252
|
}
|
|
@@ -366,40 +263,12 @@ export class CooperativePassScheduler {
|
|
|
366
263
|
get hasActiveSlowRealtime() {
|
|
367
264
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
368
265
|
}
|
|
369
|
-
get isInCapYield() {
|
|
370
|
-
return this.capYieldDepth > 0;
|
|
371
|
-
}
|
|
372
266
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
373
267
|
get activeRealtimeWorkerCount() {
|
|
374
268
|
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
375
269
|
(this.activeFastRealtime === null ? 0 : 1) +
|
|
376
270
|
this.realtimeOccupants.size);
|
|
377
271
|
}
|
|
378
|
-
get diagnosticAdmissionSnapshot() {
|
|
379
|
-
const active = [
|
|
380
|
-
this.active,
|
|
381
|
-
this.activeFastRealtime,
|
|
382
|
-
...this.realtimeOccupants,
|
|
383
|
-
this.detachedChore,
|
|
384
|
-
].filter((entry) => entry !== null);
|
|
385
|
-
const holders = active.map((entry) => ({
|
|
386
|
-
entryId: entry.entryId,
|
|
387
|
-
kind: entry.transferDirection === "pull"
|
|
388
|
-
? "pull"
|
|
389
|
-
: entry.workClass === "chore"
|
|
390
|
-
? "chore"
|
|
391
|
-
: "push",
|
|
392
|
-
scope: entry.label.slice(0, 160),
|
|
393
|
-
ageMs: entry.startedAtMonotonicMs === undefined
|
|
394
|
-
? 0
|
|
395
|
-
: Math.max(0, performance.now() - entry.startedAtMonotonicMs),
|
|
396
|
-
}));
|
|
397
|
-
return {
|
|
398
|
-
queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
|
|
399
|
-
holders: holders.slice(0, 20),
|
|
400
|
-
lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
|
|
401
|
-
};
|
|
402
|
-
}
|
|
403
272
|
pendingLaneCount(lane) {
|
|
404
273
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
405
274
|
}
|
|
@@ -414,7 +283,6 @@ export class CooperativePassScheduler {
|
|
|
414
283
|
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
415
284
|
}
|
|
416
285
|
const directEntry = {
|
|
417
|
-
entryId: `guarded-${++this.nextEntryId}`,
|
|
418
286
|
task,
|
|
419
287
|
workClass,
|
|
420
288
|
label,
|
|
@@ -438,8 +306,7 @@ export class CooperativePassScheduler {
|
|
|
438
306
|
if (workClass === "realtime" &&
|
|
439
307
|
this._handoffActive &&
|
|
440
308
|
directEntry.allowConcurrentHandoff &&
|
|
441
|
-
this.canAdmitRealtimeOccupant(directEntry)
|
|
442
|
-
!this.pullMustYieldForWaitingPush(directEntry)) {
|
|
309
|
+
this.canAdmitRealtimeOccupant(directEntry)) {
|
|
443
310
|
return new Promise((resolve, reject) => {
|
|
444
311
|
const entry = {
|
|
445
312
|
...directEntry,
|
|
@@ -588,21 +455,16 @@ export class CooperativePassScheduler {
|
|
|
588
455
|
this.admitHandoffBacklog();
|
|
589
456
|
return;
|
|
590
457
|
}
|
|
591
|
-
if (this.active !== null
|
|
458
|
+
if (this.active !== null) {
|
|
592
459
|
const fastRealtime = this.nextFastRealtime();
|
|
593
|
-
if (
|
|
460
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
594
461
|
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
595
462
|
this.startFastRealtime(fastRealtime);
|
|
596
|
-
return;
|
|
597
|
-
}
|
|
598
|
-
const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
|
|
599
|
-
if (peerIndex >= 0) {
|
|
600
|
-
const peer = this.pending.splice(peerIndex, 1)[0];
|
|
601
|
-
if (peer)
|
|
602
|
-
this.startRealtimeOccupant(peer);
|
|
603
463
|
}
|
|
604
464
|
return;
|
|
605
465
|
}
|
|
466
|
+
if (this.activeFastRealtime !== null)
|
|
467
|
+
return;
|
|
606
468
|
const next = this.takeNext();
|
|
607
469
|
if (next)
|
|
608
470
|
this.start(next);
|
|
@@ -621,29 +483,11 @@ export class CooperativePassScheduler {
|
|
|
621
483
|
return this.pending.splice(realtime, 1)[0];
|
|
622
484
|
return this.pending.shift();
|
|
623
485
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
628
|
-
}
|
|
629
|
-
nextRealtimeIndex(eligible = () => true) {
|
|
630
|
-
const waitingPush = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.transferDirection === "push" && eligible(entry));
|
|
631
|
-
if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
|
|
632
|
-
this.consecutivePullAdmissions = 0;
|
|
633
|
-
return waitingPush;
|
|
634
|
-
}
|
|
635
|
-
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow" && eligible(entry));
|
|
636
|
-
const next = slowRealtime >= 0
|
|
486
|
+
nextRealtimeIndex() {
|
|
487
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
488
|
+
return slowRealtime >= 0
|
|
637
489
|
? slowRealtime
|
|
638
|
-
: this.pending.findIndex((entry) => entry.workClass === "realtime"
|
|
639
|
-
if (next >= 0) {
|
|
640
|
-
const selected = this.pending[next];
|
|
641
|
-
if (selected?.transferDirection === "pull")
|
|
642
|
-
this.consecutivePullAdmissions += 1;
|
|
643
|
-
else if (selected?.transferDirection === "push")
|
|
644
|
-
this.consecutivePullAdmissions = 0;
|
|
645
|
-
}
|
|
646
|
-
return next;
|
|
490
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
647
491
|
}
|
|
648
492
|
nextFastRealtime() {
|
|
649
493
|
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
@@ -661,20 +505,8 @@ export class CooperativePassScheduler {
|
|
|
661
505
|
this.hasActiveSlowRealtime &&
|
|
662
506
|
!this.hasStarvedChore());
|
|
663
507
|
}
|
|
664
|
-
/** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
|
|
665
|
-
canAdmitRealtimePeer(entry) {
|
|
666
|
-
if (entry.workClass !== "realtime" ||
|
|
667
|
-
!entry.allowConcurrentHandoff ||
|
|
668
|
-
entry.transferDirection === undefined ||
|
|
669
|
-
this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
670
|
-
return false;
|
|
671
|
-
return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
|
|
672
|
-
active.transferDirection !== undefined &&
|
|
673
|
-
active.transferDirection !== entry.transferDirection);
|
|
674
|
-
}
|
|
675
508
|
start(entry) {
|
|
676
509
|
this.clearAdmission(entry);
|
|
677
|
-
this.recordAdmission(entry);
|
|
678
510
|
entry.startedAt = new Date().toISOString();
|
|
679
511
|
// Holder duration uses the process monotonic clock; do not consume the
|
|
680
512
|
// injected scheduler clock, whose reads define cadence-slot deadlines.
|
|
@@ -713,8 +545,6 @@ export class CooperativePassScheduler {
|
|
|
713
545
|
}
|
|
714
546
|
startFastRealtime(entry) {
|
|
715
547
|
this.clearAdmission(entry);
|
|
716
|
-
this.recordAdmission(entry);
|
|
717
|
-
entry.startedAtMonotonicMs = performance.now();
|
|
718
548
|
this.activeFastRealtime = entry;
|
|
719
549
|
let current;
|
|
720
550
|
try {
|
|
@@ -970,8 +800,6 @@ export class CooperativePassScheduler {
|
|
|
970
800
|
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
971
801
|
startRealtimeOccupant(entry) {
|
|
972
802
|
this.clearAdmission(entry);
|
|
973
|
-
this.recordAdmission(entry);
|
|
974
|
-
entry.startedAtMonotonicMs = performance.now();
|
|
975
803
|
this.realtimeOccupants.add(entry);
|
|
976
804
|
this.handoffAdmitted = true;
|
|
977
805
|
if (this.capYieldDepth > 0)
|
|
@@ -986,7 +814,6 @@ export class CooperativePassScheduler {
|
|
|
986
814
|
this.handoffBacklog = null;
|
|
987
815
|
entry.reject(error);
|
|
988
816
|
this.notifyOccupantChange();
|
|
989
|
-
this.drain();
|
|
990
817
|
return;
|
|
991
818
|
}
|
|
992
819
|
void running.then((outcome) => {
|
|
@@ -996,7 +823,6 @@ export class CooperativePassScheduler {
|
|
|
996
823
|
entry.resolve(outcome);
|
|
997
824
|
this.promoteOccupancyArrivals();
|
|
998
825
|
this.notifyOccupantChange();
|
|
999
|
-
this.drain();
|
|
1000
826
|
}, (error) => {
|
|
1001
827
|
this.realtimeOccupants.delete(entry);
|
|
1002
828
|
if (this.handoffBacklog === entry)
|
|
@@ -1004,7 +830,6 @@ export class CooperativePassScheduler {
|
|
|
1004
830
|
entry.reject(error);
|
|
1005
831
|
this.promoteOccupancyArrivals();
|
|
1006
832
|
this.notifyOccupantChange();
|
|
1007
|
-
this.drain();
|
|
1008
833
|
});
|
|
1009
834
|
}
|
|
1010
835
|
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
@@ -1013,7 +838,8 @@ export class CooperativePassScheduler {
|
|
|
1013
838
|
(!this.networkTransferActive && !this.withinHandoffAdmission()))
|
|
1014
839
|
return;
|
|
1015
840
|
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
1016
|
-
const index = this.
|
|
841
|
+
const index = this.pending.findIndex((pending) => pending.workClass === "realtime" &&
|
|
842
|
+
pending.occupancyArrival === true &&
|
|
1017
843
|
pending.allowConcurrentHandoff &&
|
|
1018
844
|
this.canAdmitRealtimeOccupant(pending));
|
|
1019
845
|
if (index < 0)
|
|
@@ -1032,7 +858,7 @@ export class CooperativePassScheduler {
|
|
|
1032
858
|
return;
|
|
1033
859
|
if (!this.withinHandoffAdmission())
|
|
1034
860
|
return;
|
|
1035
|
-
const realtimeIndex = this.nextRealtimeIndex(
|
|
861
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
1036
862
|
if (realtimeIndex < 0)
|
|
1037
863
|
return;
|
|
1038
864
|
const pending = this.pending[realtimeIndex];
|
|
@@ -1466,151 +1292,6 @@ export function isChoreLocalPush(paths, largeUpload) {
|
|
|
1466
1292
|
}
|
|
1467
1293
|
return count > 0;
|
|
1468
1294
|
}
|
|
1469
|
-
/** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
|
|
1470
|
-
export function watcherWarmupPathsByRoute(relativePaths) {
|
|
1471
|
-
const counts = new Map();
|
|
1472
|
-
for (const relativePath of relativePaths) {
|
|
1473
|
-
const route = routeChangeToTarget(relativePath);
|
|
1474
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1475
|
-
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1476
|
-
}
|
|
1477
|
-
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1478
|
-
const result = {};
|
|
1479
|
-
for (const [key, count] of ranked.slice(0, 20))
|
|
1480
|
-
result[key] = count;
|
|
1481
|
-
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1482
|
-
if (other > 0)
|
|
1483
|
-
result.other = other;
|
|
1484
|
-
return result;
|
|
1485
|
-
}
|
|
1486
|
-
export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
|
|
1487
|
-
const candidatePrefixes = new Map();
|
|
1488
|
-
const candidateCounts = new Map();
|
|
1489
|
-
for (const relativePath of candidateRelativePaths) {
|
|
1490
|
-
const route = routeChangeToTarget(relativePath);
|
|
1491
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1492
|
-
candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
|
|
1493
|
-
const routePath = route?.kind === "company"
|
|
1494
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1495
|
-
: relativePath;
|
|
1496
|
-
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1497
|
-
const routeCounts = candidatePrefixes.get(key) ?? new Map();
|
|
1498
|
-
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1499
|
-
candidatePrefixes.set(key, routeCounts);
|
|
1500
|
-
}
|
|
1501
|
-
const eligibleCounts = new Map();
|
|
1502
|
-
for (const relativePath of eligibleRelativePaths) {
|
|
1503
|
-
const route = routeChangeToTarget(relativePath);
|
|
1504
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1505
|
-
const routePath = route?.kind === "company"
|
|
1506
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1507
|
-
: relativePath;
|
|
1508
|
-
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1509
|
-
const routeCounts = eligibleCounts.get(key) ?? new Map();
|
|
1510
|
-
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1511
|
-
eligibleCounts.set(key, routeCounts);
|
|
1512
|
-
}
|
|
1513
|
-
const result = {};
|
|
1514
|
-
// Eligible routes are all retained; fill any remaining diagnostic slots
|
|
1515
|
-
// with the largest candidate routes. This keeps alphabetical one-path
|
|
1516
|
-
// stubs from crowding out a large route such as personal.
|
|
1517
|
-
const eligibleRoutes = [...eligibleCounts.keys()];
|
|
1518
|
-
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1519
|
-
const rankedCandidates = [...candidateCounts.entries()]
|
|
1520
|
-
.filter(([route]) => !eligibleCounts.has(route))
|
|
1521
|
-
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1522
|
-
const selectedRoutes = [
|
|
1523
|
-
...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
|
|
1524
|
-
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1525
|
-
];
|
|
1526
|
-
for (const route of selectedRoutes) {
|
|
1527
|
-
const topPrefixes = (counts) => {
|
|
1528
|
-
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1529
|
-
const top = {};
|
|
1530
|
-
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1531
|
-
top[prefix] = count;
|
|
1532
|
-
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1533
|
-
if (other > 0)
|
|
1534
|
-
top.other = other;
|
|
1535
|
-
return top;
|
|
1536
|
-
};
|
|
1537
|
-
result[route] = {
|
|
1538
|
-
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1539
|
-
eligible: topPrefixes(eligibleCounts.get(route)),
|
|
1540
|
-
};
|
|
1541
|
-
}
|
|
1542
|
-
const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
|
|
1543
|
-
if (otherRoutes.length > 0) {
|
|
1544
|
-
const merged = new Map();
|
|
1545
|
-
for (const [, routeCounts] of otherRoutes) {
|
|
1546
|
-
for (const [prefix, count] of routeCounts)
|
|
1547
|
-
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1548
|
-
}
|
|
1549
|
-
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1550
|
-
}
|
|
1551
|
-
return result;
|
|
1552
|
-
}
|
|
1553
|
-
function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
|
|
1554
|
-
counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
|
|
1555
|
-
const firstSlash = routePath.indexOf("/");
|
|
1556
|
-
const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
|
|
1557
|
-
const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
|
|
1558
|
-
let byPrefix = prefixes.get(routeName);
|
|
1559
|
-
if (!byPrefix) {
|
|
1560
|
-
byPrefix = new Map();
|
|
1561
|
-
prefixes.set(routeName, byPrefix);
|
|
1562
|
-
}
|
|
1563
|
-
byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
|
|
1564
|
-
}
|
|
1565
|
-
function rankedWarmupCounts(counts) {
|
|
1566
|
-
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1567
|
-
const result = {};
|
|
1568
|
-
for (const [key, count] of ranked.slice(0, 20))
|
|
1569
|
-
result[key] = count;
|
|
1570
|
-
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1571
|
-
if (other > 0)
|
|
1572
|
-
result.other = other;
|
|
1573
|
-
return result;
|
|
1574
|
-
}
|
|
1575
|
-
function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
|
|
1576
|
-
const topPrefixes = (counts) => {
|
|
1577
|
-
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1578
|
-
const top = {};
|
|
1579
|
-
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1580
|
-
top[prefix] = count;
|
|
1581
|
-
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1582
|
-
if (other > 0)
|
|
1583
|
-
top.other = other;
|
|
1584
|
-
return top;
|
|
1585
|
-
};
|
|
1586
|
-
const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
|
|
1587
|
-
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1588
|
-
const rankedCandidates = [...candidateCounts.entries()]
|
|
1589
|
-
.filter(([route]) => !eligibleCounts.has(route))
|
|
1590
|
-
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1591
|
-
const selectedRoutes = [
|
|
1592
|
-
...eligibleRoutes,
|
|
1593
|
-
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1594
|
-
];
|
|
1595
|
-
const result = {};
|
|
1596
|
-
for (const route of selectedRoutes) {
|
|
1597
|
-
result[route] = {
|
|
1598
|
-
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1599
|
-
eligible: topPrefixes(eligiblePrefixes.get(route)),
|
|
1600
|
-
};
|
|
1601
|
-
}
|
|
1602
|
-
const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
|
|
1603
|
-
if (otherRoutes.length > 0) {
|
|
1604
|
-
const merged = new Map();
|
|
1605
|
-
for (const route of otherRoutes) {
|
|
1606
|
-
for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
|
|
1607
|
-
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1608
|
-
}
|
|
1609
|
-
}
|
|
1610
|
-
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1611
|
-
}
|
|
1612
|
-
return result;
|
|
1613
|
-
}
|
|
1614
1295
|
/** A receiver event needs a known small upsert size before it can go realtime. */
|
|
1615
1296
|
export function isFastReceiverBatch(eventCount) {
|
|
1616
1297
|
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
@@ -2082,10 +1763,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2082
1763
|
let recycleRequested = null;
|
|
2083
1764
|
const pendingAppliedUpserts = new Map();
|
|
2084
1765
|
const pendingAppliedDeletes = new Set();
|
|
2085
|
-
const uploadedAwaitingPublish = new Set();
|
|
2086
|
-
const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
|
|
2087
|
-
const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
|
|
2088
|
-
let uploadedAwaitingPublishDropLogged = false;
|
|
2089
1766
|
const rememberAppliedUpsert = (relativePath, contentHash) => {
|
|
2090
1767
|
pendingAppliedDeletes.delete(relativePath);
|
|
2091
1768
|
if (eventSyncLive && eventSync) {
|
|
@@ -2116,23 +1793,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2116
1793
|
pendingAppliedDeletes.delete(oldest);
|
|
2117
1794
|
}
|
|
2118
1795
|
};
|
|
2119
|
-
const flushPendingUploadEvents = () => {
|
|
2120
|
-
if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
|
|
2121
|
-
return;
|
|
2122
|
-
while (uploadedAwaitingPublish.size > 0) {
|
|
2123
|
-
const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
|
|
2124
|
-
const paths = new Map(uploadedPaths.map((relativePath) => [
|
|
2125
|
-
localPathForVaultKey(hqRoot, relativePath),
|
|
2126
|
-
relativePath,
|
|
2127
|
-
]));
|
|
2128
|
-
const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
|
|
2129
|
-
if (expanded.size > 0)
|
|
2130
|
-
eventSync.publishBatch({ paths: expanded });
|
|
2131
|
-
for (const relativePath of uploadedPaths) {
|
|
2132
|
-
uploadedAwaitingPublish.delete(relativePath);
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2135
|
-
};
|
|
2136
1796
|
const flushPendingAppliedUpserts = () => {
|
|
2137
1797
|
if (!eventSyncLive || !eventSync)
|
|
2138
1798
|
return;
|
|
@@ -2144,7 +1804,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2144
1804
|
eventSync.markContentDeleted?.(relativePath);
|
|
2145
1805
|
}
|
|
2146
1806
|
pendingAppliedDeletes.clear();
|
|
2147
|
-
flushPendingUploadEvents();
|
|
2148
1807
|
};
|
|
2149
1808
|
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
2150
1809
|
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
@@ -2194,17 +1853,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2194
1853
|
}, 0);
|
|
2195
1854
|
refresh.unref?.();
|
|
2196
1855
|
}
|
|
2197
|
-
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
2198
|
-
|
|
2199
|
-
onAdmission: (event) => {
|
|
2200
|
-
process.stderr.write(`${JSON.stringify({
|
|
2201
|
-
type: "watcher-push-scheduler-admission",
|
|
2202
|
-
diagnostic: true,
|
|
2203
|
-
...event,
|
|
2204
|
-
})}\n`);
|
|
2205
|
-
},
|
|
2206
|
-
});
|
|
2207
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
|
|
1856
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
1857
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
|
|
2208
1858
|
const passCheckpoint = pullSignal || receiverSignal
|
|
2209
1859
|
? Object.assign(async () => checkpoint?.(), {
|
|
2210
1860
|
shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
|
|
@@ -2234,43 +1884,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2234
1884
|
}
|
|
2235
1885
|
};
|
|
2236
1886
|
const invokePass = () => {
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
const complete = () => {
|
|
2242
|
-
watcherPushStage?.("pass return");
|
|
2243
|
-
activePassCount = Math.max(0, activePassCount - 1);
|
|
2244
|
-
emitSyncWorkState(true);
|
|
2245
|
-
};
|
|
2246
|
-
// Observe settlement without returning a chained promise to the pass
|
|
2247
|
-
// scheduler. The scheduler's synchronous test seam and watcher wake
|
|
2248
|
-
// ordering depend on the original pass promise's completion turn.
|
|
2249
|
-
void pass.then(complete, complete);
|
|
2250
|
-
return pass;
|
|
2251
|
-
};
|
|
2252
|
-
try {
|
|
2253
|
-
if (!runPassSupportsCooperativeCheckpoint) {
|
|
2254
|
-
if (!runPassSupportsQmdLifecycle)
|
|
2255
|
-
return trackPass(runPass(passArgvForRun));
|
|
2256
|
-
return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
|
|
2257
|
-
}
|
|
2258
|
-
// This callback is emitted by the production runner after journal
|
|
2259
|
-
// repair/seed and before fanout. It starts ingress early, but never
|
|
2260
|
-
// admits a second durable writer: queued realtime still waits for the
|
|
2261
|
-
// existing scheduler's cooperative hand-off checkpoint.
|
|
2262
|
-
const onJournalsReady = () => activateEventPushSurfaces();
|
|
2263
|
-
if (!runPassSupportsQmdLifecycle) {
|
|
2264
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
|
|
2265
|
-
}
|
|
2266
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
|
|
1887
|
+
if (!runPassSupportsCooperativeCheckpoint) {
|
|
1888
|
+
if (!runPassSupportsQmdLifecycle)
|
|
1889
|
+
return runPass(passArgvForRun);
|
|
1890
|
+
return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
|
|
2267
1891
|
}
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
1892
|
+
// This callback is emitted by the production runner after journal
|
|
1893
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
1894
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
1895
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
1896
|
+
const onJournalsReady = () => activateEventPushSurfaces();
|
|
1897
|
+
if (!runPassSupportsQmdLifecycle) {
|
|
1898
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
|
|
2273
1899
|
}
|
|
1900
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
|
|
2274
1901
|
};
|
|
2275
1902
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
2276
1903
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -2285,11 +1912,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2285
1912
|
return;
|
|
2286
1913
|
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
2287
1914
|
recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
|
|
2288
|
-
rememberUploadedPaths(outcome);
|
|
2289
|
-
if (typeof outcome !== "number") {
|
|
2290
|
-
acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
|
|
2291
|
-
}
|
|
2292
|
-
flushPendingUploadEvents();
|
|
2293
1915
|
};
|
|
2294
1916
|
// Pre-admit boundary observation.
|
|
2295
1917
|
heapCheckpoint?.();
|
|
@@ -2540,25 +2162,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2540
2162
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2541
2163
|
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
|
|
2542
2164
|
};
|
|
2543
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass
|
|
2544
|
-
if (watcherPushStage) {
|
|
2545
|
-
watcherPushStage("guarded scheduler admission requested");
|
|
2546
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2547
|
-
if (passScheduler.isInCapYield)
|
|
2548
|
-
watcherPushStage("CPU/heap cap wait");
|
|
2549
|
-
else if (admission.holders.some((holder) => holder.kind === "pull")) {
|
|
2550
|
-
watcherPushStage("pull-yield wait");
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2165
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
|
|
2553
2166
|
telemetry?.mark("operationGate");
|
|
2554
2167
|
const admissionSignal = signal && receiverSignal
|
|
2555
2168
|
? AbortSignal.any([signal, receiverSignal])
|
|
2556
2169
|
: signal ?? receiverSignal;
|
|
2557
2170
|
return runGuardedTask((context) => {
|
|
2558
|
-
watcherPushStage?.("guarded scheduler admission granted");
|
|
2559
2171
|
telemetry?.mark("appendLock");
|
|
2560
2172
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2561
|
-
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal
|
|
2173
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2562
2174
|
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
|
|
2563
2175
|
};
|
|
2564
2176
|
/*
|
|
@@ -2680,7 +2292,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2680
2292
|
return {
|
|
2681
2293
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2682
2294
|
accepted,
|
|
2683
|
-
yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
|
|
2684
2295
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2685
2296
|
.filter((path) => path.status === "refused")
|
|
2686
2297
|
.map((path) => path.path)),
|
|
@@ -2878,222 +2489,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2878
2489
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
2879
2490
|
let drainedUntrackedDeletes = 0;
|
|
2880
2491
|
const pendingWatcherPaths = new Map();
|
|
2881
|
-
const durableWatcherPaths = new Map();
|
|
2882
|
-
const durableWatcherPathLimit = 10_000;
|
|
2883
|
-
let durableWatcherOverflowed = false;
|
|
2884
|
-
let durableWatcherPersistTimer;
|
|
2885
|
-
let durableWatcherPersistQueue = Promise.resolve();
|
|
2886
|
-
let durableWatcherPersistGeneration = 0;
|
|
2887
|
-
let durableWatcherPersistedGeneration = 0;
|
|
2888
|
-
const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
|
|
2889
|
-
const persistDurableWatcherPaths = () => {
|
|
2890
|
-
const generation = durableWatcherPersistGeneration;
|
|
2891
|
-
if (generation === durableWatcherPersistedGeneration)
|
|
2892
|
-
return durableWatcherPersistQueue;
|
|
2893
|
-
const snapshot = durableWatcherOverflowed
|
|
2894
|
-
? { format: 1, hqRoot, overflowed: true }
|
|
2895
|
-
: { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
|
|
2896
|
-
durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
|
|
2897
|
-
try {
|
|
2898
|
-
if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
|
|
2899
|
-
await fs.promises.rm(watcherPushStatePath, { force: true });
|
|
2900
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2901
|
-
return;
|
|
2902
|
-
}
|
|
2903
|
-
await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
|
|
2904
|
-
const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
|
|
2905
|
-
await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
|
|
2906
|
-
await fs.promises.rename(tempPath, watcherPushStatePath);
|
|
2907
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2908
|
-
}
|
|
2909
|
-
catch (err) {
|
|
2910
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2911
|
-
}
|
|
2912
|
-
}).catch((err) => {
|
|
2913
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2914
|
-
});
|
|
2915
|
-
return durableWatcherPersistQueue;
|
|
2916
|
-
};
|
|
2917
|
-
const scheduleDurableWatcherPersist = () => {
|
|
2918
|
-
durableWatcherPersistGeneration += 1;
|
|
2919
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
2920
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
2921
|
-
durableWatcherPersistTimer = setTimeout(() => {
|
|
2922
|
-
durableWatcherPersistTimer = undefined;
|
|
2923
|
-
void persistDurableWatcherPaths();
|
|
2924
|
-
}, 1_000);
|
|
2925
|
-
durableWatcherPersistTimer.unref?.();
|
|
2926
|
-
};
|
|
2927
|
-
const addDurableWatcherPath = (absolutePath, relativePath) => {
|
|
2928
|
-
if (durableWatcherOverflowed)
|
|
2929
|
-
return;
|
|
2930
|
-
if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
|
|
2931
|
-
durableWatcherPaths.clear();
|
|
2932
|
-
durableWatcherOverflowed = true;
|
|
2933
|
-
}
|
|
2934
|
-
else if (!durableWatcherOverflowed) {
|
|
2935
|
-
durableWatcherPaths.set(absolutePath, relativePath);
|
|
2936
|
-
}
|
|
2937
|
-
scheduleDurableWatcherPersist();
|
|
2938
|
-
};
|
|
2939
|
-
const clearDurableWatcherOverflow = () => {
|
|
2940
|
-
if (!durableWatcherOverflowed)
|
|
2941
|
-
return;
|
|
2942
|
-
durableWatcherOverflowed = false;
|
|
2943
|
-
durableWatcherPaths.clear();
|
|
2944
|
-
for (const [absolutePath, relativePath] of pendingWatcherPaths) {
|
|
2945
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
2946
|
-
}
|
|
2947
|
-
scheduleDurableWatcherPersist();
|
|
2948
|
-
};
|
|
2949
|
-
const acknowledgeDurableWatcherUploads = (relativePaths) => {
|
|
2950
|
-
let changed = false;
|
|
2951
|
-
const uploaded = new Set(relativePaths);
|
|
2952
|
-
for (const [absolutePath, relativePath] of durableWatcherPaths) {
|
|
2953
|
-
if (!uploaded.has(relativePath))
|
|
2954
|
-
continue;
|
|
2955
|
-
durableWatcherPaths.delete(absolutePath);
|
|
2956
|
-
changed = true;
|
|
2957
|
-
}
|
|
2958
|
-
if (changed)
|
|
2959
|
-
scheduleDurableWatcherPersist();
|
|
2960
|
-
};
|
|
2961
|
-
const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
|
|
2962
|
-
const stillPending = new Set(pendingWatcherPaths.values());
|
|
2963
|
-
acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
|
|
2964
|
-
};
|
|
2965
|
-
let activePassCount = 0;
|
|
2966
|
-
let inFlightWatcherPushes = 0;
|
|
2967
|
-
const watcherPushStages = new Map();
|
|
2968
|
-
let nextWatcherPushGroupId = 0;
|
|
2969
|
-
const setWatcherPushStage = (groupId, stage) => {
|
|
2970
|
-
const record = watcherPushStages.get(groupId);
|
|
2971
|
-
if (!record || record.stage === stage)
|
|
2972
|
-
return;
|
|
2973
|
-
record.stage = stage;
|
|
2974
|
-
record.stageEnteredAt = Date.now();
|
|
2975
|
-
if (stage === "guarded scheduler admission requested") {
|
|
2976
|
-
record.admissionRequestedAtMonotonicMs = performance.now();
|
|
2977
|
-
record.admissionWaiting = true;
|
|
2978
|
-
}
|
|
2979
|
-
else if (stage === "guarded scheduler admission granted") {
|
|
2980
|
-
record.admissionWaiting = false;
|
|
2981
|
-
}
|
|
2982
|
-
process.stderr.write(`${JSON.stringify({
|
|
2983
|
-
type: "watcher-push-group-stage",
|
|
2984
|
-
diagnostic: true,
|
|
2985
|
-
groupId: record.groupId,
|
|
2986
|
-
route: record.route,
|
|
2987
|
-
paths: record.paths,
|
|
2988
|
-
stage: record.stage,
|
|
2989
|
-
stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
|
|
2990
|
-
})}\n`);
|
|
2991
|
-
};
|
|
2992
|
-
const startWatcherPushStageTracking = (route, paths) => {
|
|
2993
|
-
const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
|
|
2994
|
-
watcherPushStages.set(groupId, {
|
|
2995
|
-
groupId,
|
|
2996
|
-
route,
|
|
2997
|
-
paths,
|
|
2998
|
-
stage: "",
|
|
2999
|
-
stageEnteredAt: Date.now(),
|
|
3000
|
-
admissionRequestedAtMonotonicMs: null,
|
|
3001
|
-
admissionWaiting: false,
|
|
3002
|
-
});
|
|
3003
|
-
return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
|
|
3004
|
-
};
|
|
3005
|
-
const emitWatcherPushInflightStages = () => {
|
|
3006
|
-
if (inFlightWatcherPushes <= 0)
|
|
3007
|
-
return;
|
|
3008
|
-
const now = Date.now();
|
|
3009
|
-
const entries = [...watcherPushStages.values()];
|
|
3010
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
3011
|
-
const coordinatorAdmission = coordinator?.inspect();
|
|
3012
|
-
for (const record of entries) {
|
|
3013
|
-
if (!record.admissionWaiting)
|
|
3014
|
-
continue;
|
|
3015
|
-
const liveStage = resolveWatcherPushWaitStage({
|
|
3016
|
-
capYieldActive: passScheduler.isInCapYield,
|
|
3017
|
-
queueDepth: admission.queueDepth,
|
|
3018
|
-
holders: admission.holders,
|
|
3019
|
-
});
|
|
3020
|
-
if (liveStage !== record.stage)
|
|
3021
|
-
setWatcherPushStage(record.groupId, liveStage);
|
|
3022
|
-
}
|
|
3023
|
-
process.stderr.write(`${JSON.stringify({
|
|
3024
|
-
type: "watcher-push-inflight-stages",
|
|
3025
|
-
diagnostic: true,
|
|
3026
|
-
groups: entries.slice(0, 20).map((record) => {
|
|
3027
|
-
const monotonicNow = performance.now();
|
|
3028
|
-
const current = watcherPushWaitDiagnostics({
|
|
3029
|
-
stage: record.stage,
|
|
3030
|
-
admissionWaiting: record.admissionWaiting,
|
|
3031
|
-
admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
|
|
3032
|
-
lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
|
|
3033
|
-
nowMonotonicMs: monotonicNow,
|
|
3034
|
-
capYieldActive: passScheduler.isInCapYield,
|
|
3035
|
-
queueDepth: admission.queueDepth,
|
|
3036
|
-
holders: admission.holders,
|
|
3037
|
-
});
|
|
3038
|
-
return {
|
|
3039
|
-
route: record.route,
|
|
3040
|
-
paths: record.paths,
|
|
3041
|
-
...current,
|
|
3042
|
-
ageMs: Math.max(0, now - record.stageEnteredAt),
|
|
3043
|
-
};
|
|
3044
|
-
}),
|
|
3045
|
-
other: Math.max(0, entries.length - 20),
|
|
3046
|
-
realtimeAdmissionQueueDepth: admission.queueDepth,
|
|
3047
|
-
holders: admission.holders,
|
|
3048
|
-
coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
|
|
3049
|
-
coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
|
|
3050
|
-
coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
|
|
3051
|
-
fast: null,
|
|
3052
|
-
slow: null,
|
|
3053
|
-
overflowPush: null,
|
|
3054
|
-
},
|
|
3055
|
-
coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
|
|
3056
|
-
coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
|
|
3057
|
-
coordinatorAdmission === undefined
|
|
3058
|
-
? null
|
|
3059
|
-
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
|
|
3060
|
-
coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
|
|
3061
|
-
coordinatorAdmission === undefined
|
|
3062
|
-
? null
|
|
3063
|
-
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
|
|
3064
|
-
})}\n`);
|
|
3065
|
-
};
|
|
3066
|
-
const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
|
|
3067
|
-
watcherPushStagesTimer.unref?.();
|
|
3068
|
-
let pendingReceivers = 0;
|
|
3069
|
-
let lastSyncWorkStateEmitAt = 0;
|
|
3070
|
-
let syncWorkStateTimer;
|
|
3071
|
-
const emitSyncWorkState = (force = false) => {
|
|
3072
|
-
const now = Date.now();
|
|
3073
|
-
const elapsed = now - lastSyncWorkStateEmitAt;
|
|
3074
|
-
if (force || elapsed >= 250) {
|
|
3075
|
-
if (syncWorkStateTimer !== undefined)
|
|
3076
|
-
clearTimeout(syncWorkStateTimer);
|
|
3077
|
-
syncWorkStateTimer = undefined;
|
|
3078
|
-
lastSyncWorkStateEmitAt = now;
|
|
3079
|
-
const snapshot = {
|
|
3080
|
-
passActive: activePassCount > 0,
|
|
3081
|
-
queuedWatcherPushes: pendingWatcherPaths.size,
|
|
3082
|
-
inFlightWatcherPushes,
|
|
3083
|
-
pendingReceivers,
|
|
3084
|
-
};
|
|
3085
|
-
deps.onSyncWorkStateSnapshot?.(snapshot);
|
|
3086
|
-
stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
|
|
3087
|
-
return;
|
|
3088
|
-
}
|
|
3089
|
-
if (syncWorkStateTimer !== undefined)
|
|
3090
|
-
return;
|
|
3091
|
-
syncWorkStateTimer = setTimeout(() => {
|
|
3092
|
-
syncWorkStateTimer = undefined;
|
|
3093
|
-
emitSyncWorkState(true);
|
|
3094
|
-
}, Math.max(1, 250 - elapsed));
|
|
3095
|
-
syncWorkStateTimer.unref?.();
|
|
3096
|
-
};
|
|
3097
2492
|
const droppedUnresolvedWatcherSlugs = new Set();
|
|
3098
2493
|
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
3099
2494
|
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
@@ -3161,11 +2556,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3161
2556
|
}
|
|
3162
2557
|
}
|
|
3163
2558
|
if (batch) {
|
|
3164
|
-
if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
|
|
3165
|
-
durableWatcherPaths.clear();
|
|
3166
|
-
durableWatcherOverflowed = true;
|
|
3167
|
-
scheduleDurableWatcherPersist();
|
|
3168
|
-
}
|
|
3169
2559
|
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
3170
2560
|
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
3171
2561
|
return previousSequence === undefined ||
|
|
@@ -3174,7 +2564,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3174
2564
|
});
|
|
3175
2565
|
if (entries.length === 0)
|
|
3176
2566
|
return false;
|
|
3177
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
3178
2567
|
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
3179
2568
|
pendingWatcherOriginalBatch =
|
|
3180
2569
|
pendingWatcherPaths.size === 0 &&
|
|
@@ -3186,13 +2575,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3186
2575
|
: null;
|
|
3187
2576
|
for (const [absolutePath, relativePath] of entries) {
|
|
3188
2577
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3189
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3190
2578
|
const change = batch.changes?.get(absolutePath);
|
|
3191
2579
|
if (change)
|
|
3192
2580
|
pendingWatcherChanges.set(absolutePath, change);
|
|
3193
2581
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3194
2582
|
}
|
|
3195
|
-
emitSyncWorkState(queueWasEmpty);
|
|
3196
2583
|
if (batch.overflowed) {
|
|
3197
2584
|
pendingWatcherOverflowed = true;
|
|
3198
2585
|
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
@@ -3216,20 +2603,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3216
2603
|
// counterpart must be re-derived through the codec — a raw join would
|
|
3217
2604
|
// point at the impossible colon path on Windows.
|
|
3218
2605
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
3219
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
3220
2606
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
3221
|
-
addDurableWatcherPath(absolutePath, changedRelPath);
|
|
3222
2607
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3223
2608
|
// A bare path signal carries no delete intent; drop any change record
|
|
3224
2609
|
// a prior batch left for this path so the stale intent cannot leak
|
|
3225
2610
|
// into the coalesced batch (#227).
|
|
3226
2611
|
pendingWatcherChanges.delete(absolutePath);
|
|
3227
|
-
emitSyncWorkState(queueWasEmpty);
|
|
3228
2612
|
return true;
|
|
3229
2613
|
}
|
|
3230
2614
|
pendingWatcherOriginalBatch = null;
|
|
3231
2615
|
pendingWatcherBareChange = true;
|
|
3232
|
-
emitSyncWorkState();
|
|
3233
2616
|
return true;
|
|
3234
2617
|
};
|
|
3235
2618
|
const takePendingWatcherChange = () => {
|
|
@@ -3274,7 +2657,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3274
2657
|
pendingWatcherDroppedRouteHints.clear();
|
|
3275
2658
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
3276
2659
|
pendingWatcherBatchGeneration = null;
|
|
3277
|
-
emitSyncWorkState();
|
|
3278
2660
|
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
3279
2661
|
};
|
|
3280
2662
|
const restorePendingWatcherPaths = (paths, generation, changes) => {
|
|
@@ -3288,54 +2670,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3288
2670
|
if (pendingWatcherChangeSequences.has(absolutePath))
|
|
3289
2671
|
continue;
|
|
3290
2672
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3291
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3292
2673
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3293
2674
|
const change = changes?.get(absolutePath);
|
|
3294
2675
|
if (change)
|
|
3295
2676
|
pendingWatcherChanges.set(absolutePath, change);
|
|
3296
2677
|
}
|
|
3297
|
-
emitSyncWorkState();
|
|
3298
2678
|
};
|
|
3299
|
-
if (eventPush) {
|
|
3300
|
-
try {
|
|
3301
|
-
const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
|
|
3302
|
-
if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
|
|
3303
|
-
(!Array.isArray(stored.paths) && stored.overflowed !== true)) {
|
|
3304
|
-
throw new Error("stored watcher push state has an unsupported shape or root");
|
|
3305
|
-
}
|
|
3306
|
-
if (stored.overflowed === true) {
|
|
3307
|
-
durableWatcherOverflowed = true;
|
|
3308
|
-
pendingWatcherOverflowed = true;
|
|
3309
|
-
pendingWatcherWarmupCatchup = true;
|
|
3310
|
-
pendingWatcherDroppedRoutesUnknown = true;
|
|
3311
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3312
|
-
emitSyncWorkState(true);
|
|
3313
|
-
process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
|
|
3314
|
-
}
|
|
3315
|
-
for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
|
|
3316
|
-
if (typeof relativePath !== "string" || relativePath.length === 0)
|
|
3317
|
-
continue;
|
|
3318
|
-
const route = routeChangeToTarget(relativePath);
|
|
3319
|
-
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3320
|
-
continue;
|
|
3321
|
-
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
3322
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3323
|
-
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3324
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3325
|
-
}
|
|
3326
|
-
if (pendingWatcherPaths.size > 0) {
|
|
3327
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3328
|
-
pendingWatcherWarmupCatchup = true;
|
|
3329
|
-
emitSyncWorkState(true);
|
|
3330
|
-
process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
|
|
3331
|
-
}
|
|
3332
|
-
}
|
|
3333
|
-
catch (err) {
|
|
3334
|
-
if (!(err instanceof Error && err.code === "ENOENT")) {
|
|
3335
|
-
process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
|
|
3336
|
-
}
|
|
3337
|
-
}
|
|
3338
|
-
}
|
|
3339
2679
|
const routePushFailureBackoff = new RoutePushFailureBackoff();
|
|
3340
2680
|
const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
|
|
3341
2681
|
const routePushRetryTimerHandles = new Map();
|
|
@@ -3916,12 +3256,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3916
3256
|
const relativePath = route.kind === "personal"
|
|
3917
3257
|
? key
|
|
3918
3258
|
: ["companies", route.slug, key].join("/");
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
if (
|
|
3259
|
+
const localPath = path.resolve(hqRoot, ...relativePath.split("/"));
|
|
3260
|
+
const fromRoot = path.relative(hqRoot, localPath);
|
|
3261
|
+
const parentPrefix = ["..", path.sep].join("");
|
|
3262
|
+
if (fromRoot === ".." ||
|
|
3263
|
+
fromRoot.startsWith(parentPrefix) ||
|
|
3264
|
+
path.isAbsolute(fromRoot)) {
|
|
3923
3265
|
continue;
|
|
3924
|
-
|
|
3266
|
+
}
|
|
3925
3267
|
if (shouldSync(localPath)) {
|
|
3926
3268
|
paths.push(relativePath);
|
|
3927
3269
|
continue;
|
|
@@ -4184,6 +3526,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4184
3526
|
// a file and still be refused as stale (a download changed the directory
|
|
4185
3527
|
// mid-pass); the retry then finds the file already in the cloud and uploads
|
|
4186
3528
|
// nothing, so without this the file is never announced to peers.
|
|
3529
|
+
const uploadedAwaitingPublish = new Set();
|
|
3530
|
+
const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
|
|
4187
3531
|
const rememberUploadedPaths = (result) => {
|
|
4188
3532
|
if (result === undefined || typeof result === "number")
|
|
4189
3533
|
return;
|
|
@@ -4196,14 +3540,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4196
3540
|
const oldest = uploadedAwaitingPublish.values().next().value;
|
|
4197
3541
|
if (oldest !== undefined)
|
|
4198
3542
|
uploadedAwaitingPublish.delete(oldest);
|
|
4199
|
-
if (!uploadedAwaitingPublishDropLogged) {
|
|
4200
|
-
uploadedAwaitingPublishDropLogged = true;
|
|
4201
|
-
process.stderr.write(`${JSON.stringify({
|
|
4202
|
-
type: "event-sync-upload-buffer-overflow",
|
|
4203
|
-
cap: UPLOADED_AWAITING_PUBLISH_CAP,
|
|
4204
|
-
action: "drop-oldest",
|
|
4205
|
-
})}\n`);
|
|
4206
|
-
}
|
|
4207
3543
|
}
|
|
4208
3544
|
}
|
|
4209
3545
|
};
|
|
@@ -4220,37 +3556,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4220
3556
|
};
|
|
4221
3557
|
let emitScopedPushFailureTelemetry = null;
|
|
4222
3558
|
const runScopedDrain = async (batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
4223
|
-
if (batch)
|
|
4224
|
-
|
|
4225
|
-
// before resolving delete snapshots or submitting work to the coordinator
|
|
4226
|
-
// so a quarantined key cannot acquire a new upload/delete intent.
|
|
4227
|
-
const pathsByJournal = new Map();
|
|
4228
|
-
let unresolvedCount = 0;
|
|
4229
|
-
for (const [absolutePath, relativePath] of [...batch.paths]) {
|
|
4230
|
-
const kind = batch.changes?.get(absolutePath)?.kind;
|
|
4231
|
-
const coordinates = journalCoordinates(relativePath, kind);
|
|
4232
|
-
if (!coordinates)
|
|
4233
|
-
continue;
|
|
4234
|
-
let quarantined = pathsByJournal.get(coordinates.slug);
|
|
4235
|
-
if (!quarantined) {
|
|
4236
|
-
quarantined = readQuarantinedJournalPaths(coordinates.slug);
|
|
4237
|
-
pathsByJournal.set(coordinates.slug, quarantined);
|
|
4238
|
-
}
|
|
4239
|
-
if (!quarantined.has(coordinates.key))
|
|
4240
|
-
continue;
|
|
4241
|
-
batch.paths.delete(absolutePath);
|
|
4242
|
-
batch.changes?.delete(absolutePath);
|
|
4243
|
-
unresolvedCount += 1;
|
|
4244
|
-
}
|
|
4245
|
-
if (unresolvedCount > 0) {
|
|
4246
|
-
(deps.stderr ?? process.stderr).write(`${JSON.stringify({
|
|
4247
|
-
type: "journal-quarantine-unresolved",
|
|
4248
|
-
count: unresolvedCount,
|
|
4249
|
-
})}\n`);
|
|
4250
|
-
}
|
|
4251
|
-
if (batch.paths.size > 0)
|
|
4252
|
-
await resolveDeleteSnapshotCaptures(batch);
|
|
4253
|
-
}
|
|
3559
|
+
if (batch)
|
|
3560
|
+
await resolveDeleteSnapshotCaptures(batch);
|
|
4254
3561
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
4255
3562
|
const skipCompanies = grouped ? resolveSkipCompanies() : null;
|
|
4256
3563
|
const isPausedRoute = (route) => route.kind === "company" &&
|
|
@@ -4347,48 +3654,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4347
3654
|
? localUploadSizeClass(uploadPaths)
|
|
4348
3655
|
: "small";
|
|
4349
3656
|
const largeUpload = sizeClass !== "small";
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
watcherPushStage("worker entry");
|
|
4364
|
-
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
4365
|
-
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
4366
|
-
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
4367
|
-
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
4368
|
-
? () => prepareWatcherChanges(admittedChanges)
|
|
4369
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
|
|
4370
|
-
// Watcher-originated pushes may share the guarded
|
|
4371
|
-
// realtime slot with an in-flight pull. Coordinator
|
|
4372
|
-
// fencing already serializes overlapping path epochs;
|
|
4373
|
-
// this must not depend on the optional full-pass flag.
|
|
4374
|
-
true, undefined, routeId, watcherPushStage);
|
|
4375
|
-
}, sizeClass)
|
|
4376
|
-
: (() => {
|
|
4377
|
-
watcherPushStage("worker entry");
|
|
4378
|
-
return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
4379
|
-
? () => prepareWatcherChanges(group.changes)
|
|
4380
|
-
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
|
|
4381
|
-
result,
|
|
4382
|
-
accepted: true,
|
|
4383
|
-
coordinatorRefusedPaths: [],
|
|
4384
|
-
}));
|
|
4385
|
-
})());
|
|
4386
|
-
}
|
|
4387
|
-
finally {
|
|
4388
|
-
watcherPushStages.delete(groupId);
|
|
4389
|
-
inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
|
|
4390
|
-
emitSyncWorkState(true);
|
|
4391
|
-
}
|
|
3657
|
+
const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
|
|
3658
|
+
const execution = parallelCoordinatorEnabled && !chorePush
|
|
3659
|
+
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
|
|
3660
|
+
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
3661
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
3662
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
3663
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
3664
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
3665
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
|
|
3666
|
+
}, sizeClass)
|
|
3667
|
+
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
3668
|
+
? () => prepareWatcherChanges(group.changes)
|
|
3669
|
+
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
4392
3670
|
const result = execution.result;
|
|
4393
3671
|
rememberUploadedPaths(result);
|
|
4394
3672
|
const pushExit = passExitCode(result);
|
|
@@ -4407,7 +3685,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4407
3685
|
}
|
|
4408
3686
|
const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
|
|
4409
3687
|
relativePath.startsWith(`${refusedPath}/`));
|
|
4410
|
-
acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
|
|
4411
3688
|
const publishPaths = new Map();
|
|
4412
3689
|
for (const [absolutePath, relativePath] of group.paths) {
|
|
4413
3690
|
if (isRefusedPath(relativePath))
|
|
@@ -4447,11 +3724,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4447
3724
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
4448
3725
|
}
|
|
4449
3726
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
4450
|
-
acknowledgeDurableWatcherUploads(group.relPaths);
|
|
4451
3727
|
routePushFailureBackoff.succeeded(routeId);
|
|
4452
3728
|
clearRoutePushRetry(routeId);
|
|
4453
3729
|
if (group.route.kind === "company") {
|
|
4454
|
-
deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
|
|
4455
3730
|
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
4456
3731
|
}
|
|
4457
3732
|
}
|
|
@@ -4513,7 +3788,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4513
3788
|
const execution = parallelCoordinatorEnabled
|
|
4514
3789
|
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
4515
3790
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4516
|
-
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced
|
|
3791
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
4517
3792
|
})
|
|
4518
3793
|
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
4519
3794
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
@@ -4565,7 +3840,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4565
3840
|
addRoute(hint);
|
|
4566
3841
|
const skipCompanies = resolveSkipCompanies();
|
|
4567
3842
|
if (parallelCoordinatorEnabled) {
|
|
4568
|
-
let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
|
|
4569
3843
|
for (const route of routes.values()) {
|
|
4570
3844
|
if (route.kind === "company" &&
|
|
4571
3845
|
skipCompanies.has(route.slug.toLowerCase())) {
|
|
@@ -4594,11 +3868,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4594
3868
|
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
4595
3869
|
interruptWait();
|
|
4596
3870
|
}
|
|
4597
|
-
else {
|
|
4598
|
-
routesRemaining -= 1;
|
|
4599
|
-
if (routesRemaining === 0)
|
|
4600
|
-
clearDurableWatcherOverflow();
|
|
4601
|
-
}
|
|
4602
3871
|
});
|
|
4603
3872
|
}
|
|
4604
3873
|
return 0;
|
|
@@ -4714,9 +3983,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4714
3983
|
// churn overflowed the 200-path batch and skipped publishBatch.
|
|
4715
3984
|
personalMode: true,
|
|
4716
3985
|
includeCompanyPaths: companiesMode || companyMode,
|
|
4717
|
-
// Inventory filtered candidates so the route's push filter remains
|
|
4718
|
-
// authoritative and can attribute exclusions in warm-up diagnostics.
|
|
4719
|
-
includeFilteredWarmupCandidates: true,
|
|
4720
3986
|
onDegraded: opts.onDegraded,
|
|
4721
3987
|
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
4722
3988
|
queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
|
|
@@ -4838,7 +4104,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4838
4104
|
}
|
|
4839
4105
|
}
|
|
4840
4106
|
watcher.start();
|
|
4841
|
-
let catchupPathStatWarningReported = false;
|
|
4842
4107
|
const reportWatcherReady = () => {
|
|
4843
4108
|
if (stopped || watcher === null)
|
|
4844
4109
|
return;
|
|
@@ -4879,202 +4144,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4879
4144
|
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
|
|
4880
4145
|
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
4881
4146
|
return;
|
|
4882
|
-
const
|
|
4883
|
-
const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
|
|
4884
|
-
const eligiblePaths = new Map();
|
|
4885
|
-
const eligibleChanges = new Map();
|
|
4886
|
-
const excludedByReason = {};
|
|
4887
|
-
const excludedByReasonByRoute = {};
|
|
4888
|
-
const personalFilterExclusions = new Map();
|
|
4889
|
-
const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
|
|
4890
|
-
const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
|
|
4891
|
-
const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
|
|
4892
|
-
const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
|
|
4893
|
-
const clearedResolvedCompanies = new Set();
|
|
4894
|
-
const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
|
|
4895
|
-
const candidateRouteCounts = new Map();
|
|
4896
|
-
const candidateRoutePrefixes = new Map();
|
|
4897
|
-
const eligibleRouteCounts = new Map();
|
|
4898
|
-
const eligibleRoutePrefixes = new Map();
|
|
4899
|
-
const backupClass = (relativePath) => {
|
|
4900
|
-
if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
|
|
4901
|
-
return "attioCleanup";
|
|
4902
|
-
if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
|
|
4903
|
-
return "readModelRollout";
|
|
4904
|
-
return undefined;
|
|
4905
|
-
};
|
|
4906
|
-
const exclude = (reason, routeName) => {
|
|
4907
|
-
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
|
|
4908
|
-
const routeReasons = excludedByReasonByRoute[routeName] ??= {};
|
|
4909
|
-
routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
|
|
4910
|
-
};
|
|
4911
|
-
let filteredCandidateCount = 0;
|
|
4912
|
-
for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
|
|
4913
|
-
filteredCandidateCount += count;
|
|
4914
|
-
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
|
|
4915
|
-
const routeReasons = excludedByReasonByRoute.filtered ??= {};
|
|
4916
|
-
routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
|
|
4917
|
-
}
|
|
4918
|
-
if (filteredCandidateCount > 0)
|
|
4919
|
-
candidateRouteCounts.set("filtered", filteredCandidateCount);
|
|
4920
|
-
let examinedWarmupPaths = 0;
|
|
4921
|
-
for (const [absolutePath, relativePath] of catchup.batch.paths) {
|
|
4922
|
-
examinedWarmupPaths += 1;
|
|
4923
|
-
const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
|
|
4924
|
-
if (warmupYield)
|
|
4925
|
-
await warmupYield;
|
|
4926
|
-
if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
|
|
4927
|
-
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
|
|
4928
|
-
exclude("unsafe-path", "unrouted");
|
|
4929
|
-
continue;
|
|
4930
|
-
}
|
|
4931
|
-
const route = routeChangeToTarget(relativePath);
|
|
4932
|
-
const routeName = route ? routeKey(route) : "unrouted";
|
|
4933
|
-
const summaryRoutePath = route?.kind === "company"
|
|
4934
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4935
|
-
: relativePath;
|
|
4936
|
-
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
|
|
4937
|
-
const ontologyBackupClass = backupClass(relativePath);
|
|
4938
|
-
if (ontologyBackupClass) {
|
|
4939
|
-
exclude("local-rule:ontology-backup", routeName);
|
|
4940
|
-
ontologyBackupPaths[ontologyBackupClass] += 1;
|
|
4941
|
-
continue;
|
|
4942
|
-
}
|
|
4943
|
-
if (route?.kind === "company") {
|
|
4944
|
-
const resolvedNow = cloudCompanySlugs.has(route.slug) ||
|
|
4945
|
-
skipStore?.hasCurrentGrantCompany(route.slug) ||
|
|
4946
|
-
skipStore?.hasCurrentGrantVersion(route.slug) ||
|
|
4947
|
-
(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
|
|
4948
|
-
if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
|
|
4949
|
-
skipStore?.clearUnresolvedCompany(route.slug);
|
|
4950
|
-
clearedResolvedCompanies.add(route.slug);
|
|
4951
|
-
}
|
|
4952
|
-
if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
|
|
4953
|
-
exclude("unresolved-company", routeName);
|
|
4954
|
-
continue;
|
|
4955
|
-
}
|
|
4956
|
-
}
|
|
4957
|
-
if (route?.kind === "company" &&
|
|
4958
|
-
!cloudCompanySlugs.has(route.slug) &&
|
|
4959
|
-
!skipStore?.hasCurrentGrantCompany(route.slug) &&
|
|
4960
|
-
!skipStore?.hasCurrentGrantVersion(route.slug) &&
|
|
4961
|
-
!(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
|
|
4962
|
-
exclude("local-only-company", routeName);
|
|
4963
|
-
continue;
|
|
4964
|
-
}
|
|
4965
|
-
const company = route?.kind === "company" ? route.slug : "personal";
|
|
4966
|
-
const routePath = route?.kind === "company"
|
|
4967
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4968
|
-
: relativePath;
|
|
4969
|
-
if (isEphemeralPath(routePath)) {
|
|
4970
|
-
exclude("push-filter:conflict-mirror", routeName);
|
|
4971
|
-
continue;
|
|
4972
|
-
}
|
|
4973
|
-
// The production TreeWatcher collector has already applied this
|
|
4974
|
-
// shared filter stack and reports rejected files as reason counts.
|
|
4975
|
-
// Injected/custom collectors without that proof still evaluate the
|
|
4976
|
-
// route filter here, with personal defaults first for `.env` attribution.
|
|
4977
|
-
const personalDefaultExclusion = catchup.pushFiltersApplied === true
|
|
4978
|
-
? undefined
|
|
4979
|
-
: route?.kind === "personal"
|
|
4980
|
-
? matchPersonalVaultExclusion(relativePath, false)
|
|
4981
|
-
: undefined;
|
|
4982
|
-
const isPersonalRoute = route?.kind !== "company";
|
|
4983
|
-
const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
|
|
4984
|
-
? personalPushFilter(absolutePath, false, relativePath)
|
|
4985
|
-
: ignoreFilter(absolutePath, false, relativePath)));
|
|
4986
|
-
if (!acceptedByPushFilter) {
|
|
4987
|
-
const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
|
|
4988
|
-
? personalFilterExclusions.get(relativePath)
|
|
4989
|
-
: undefined);
|
|
4990
|
-
exclude(exclusionId
|
|
4991
|
-
? `push-filter:personal:${exclusionId}`
|
|
4992
|
-
: "push-filter:ignore", routeName);
|
|
4993
|
-
continue;
|
|
4994
|
-
}
|
|
4995
|
-
if (isPersonalRoute &&
|
|
4996
|
-
!shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
|
|
4997
|
-
isPersonalVaultScopeExcluded(relativePath, {
|
|
4998
|
-
continuityPaths,
|
|
4999
|
-
hasAgencyCarveout,
|
|
5000
|
-
hasSessionLogsCarveout,
|
|
5001
|
-
})) {
|
|
5002
|
-
exclude("push-filter:personal:excluded-top-level", routeName);
|
|
5003
|
-
continue;
|
|
5004
|
-
}
|
|
5005
|
-
if (route && isLocallyIneligibleForPush(routePath, {
|
|
5006
|
-
boardClientOwned: true,
|
|
5007
|
-
ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
|
|
5008
|
-
})) {
|
|
5009
|
-
const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
|
|
5010
|
-
exclude(`local-rule:${pathClass}`, routeName);
|
|
5011
|
-
continue;
|
|
5012
|
-
}
|
|
5013
|
-
const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
|
|
5014
|
-
const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
|
|
5015
|
-
try {
|
|
5016
|
-
const stat = !catchup.symlinkPaths || hasSkipRecord
|
|
5017
|
-
? fs.lstatSync(absolutePath)
|
|
5018
|
-
: undefined;
|
|
5019
|
-
if (route && (knownSymlink || stat?.isSymbolicLink())) {
|
|
5020
|
-
const scopeRoot = route.kind === "company"
|
|
5021
|
-
? path.join(hqRoot, "companies", route.slug)
|
|
5022
|
-
: relativePath.startsWith("personal/")
|
|
5023
|
-
? path.join(hqRoot, "personal")
|
|
5024
|
-
: hqRoot;
|
|
5025
|
-
const inspection = inspectLocalSymlinkScope({
|
|
5026
|
-
absoluteLinkPath: absolutePath,
|
|
5027
|
-
linkPath: relativePath,
|
|
5028
|
-
scopeRoot,
|
|
5029
|
-
scope: route.kind === "company"
|
|
5030
|
-
? { kind: "company", companySlug: route.slug }
|
|
5031
|
-
: { kind: "personal" },
|
|
5032
|
-
});
|
|
5033
|
-
if (inspection.refused) {
|
|
5034
|
-
exclude("symlink", routeName);
|
|
5035
|
-
continue;
|
|
5036
|
-
}
|
|
5037
|
-
}
|
|
5038
|
-
if (hasSkipRecord && stat &&
|
|
5039
|
-
skipStore?.matches(company, routePath, stat)) {
|
|
5040
|
-
exclude("record-match", routeName);
|
|
5041
|
-
continue;
|
|
5042
|
-
}
|
|
5043
|
-
}
|
|
5044
|
-
catch (error) {
|
|
5045
|
-
// Fail open: unreadable state or a filesystem race must not suppress a candidate.
|
|
5046
|
-
if (!catchupPathStatWarningReported) {
|
|
5047
|
-
catchupPathStatWarningReported = true;
|
|
5048
|
-
process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
|
|
5049
|
-
}
|
|
5050
|
-
}
|
|
5051
|
-
eligiblePaths.set(absolutePath, relativePath);
|
|
5052
|
-
recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
|
|
5053
|
-
const change = catchup.batch.changes?.get(absolutePath);
|
|
5054
|
-
if (change)
|
|
5055
|
-
eligibleChanges.set(absolutePath, change);
|
|
5056
|
-
}
|
|
5057
|
-
const queued = addPendingWatcherChange(undefined, {
|
|
5058
|
-
paths: eligiblePaths,
|
|
5059
|
-
changes: eligibleChanges,
|
|
5060
|
-
}, {
|
|
4147
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
5061
4148
|
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
5062
4149
|
});
|
|
5063
4150
|
pendingWatcherWarmupCatchup ||= queued;
|
|
5064
|
-
const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
|
|
5065
4151
|
process.stderr.write(JSON.stringify({
|
|
5066
4152
|
type: "watcher-warmup-catchup",
|
|
5067
4153
|
reason: "watch-set-ready",
|
|
5068
|
-
pathsFound:
|
|
5069
|
-
candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
|
|
5070
|
-
pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
|
|
5071
|
-
pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
|
|
5072
|
-
excludedByReason: {
|
|
5073
|
-
...excludedByReason,
|
|
5074
|
-
...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
|
|
5075
|
-
},
|
|
5076
|
-
excludedByReasonByRoute,
|
|
5077
|
-
ontologyBackupPaths,
|
|
4154
|
+
pathsFound: catchup.batch.paths.size,
|
|
5078
4155
|
scannedPaths: catchup.scannedPaths,
|
|
5079
4156
|
elapsedMs: Date.now() - startedAt,
|
|
5080
4157
|
queued,
|
|
@@ -5141,16 +4218,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5141
4218
|
const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
5142
4219
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
5143
4220
|
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
5144
|
-
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal
|
|
4221
|
+
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
5145
4222
|
});
|
|
5146
4223
|
const pathCount = events.length;
|
|
5147
4224
|
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
5148
4225
|
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
5149
4226
|
if (execution.admissionTimeout)
|
|
5150
4227
|
throw execution.admissionTimeout;
|
|
5151
|
-
if (execution.yieldedToPush) {
|
|
5152
|
-
throw new RealtimePushYieldError();
|
|
5153
|
-
}
|
|
5154
4228
|
const exitCode = passExitCode(execution.result);
|
|
5155
4229
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
5156
4230
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -5172,82 +4246,74 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5172
4246
|
const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
|
|
5173
4247
|
if (stopped)
|
|
5174
4248
|
return;
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
if (lane) {
|
|
5192
|
-
receiverLaneByBatch.delete(events);
|
|
5193
|
-
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5194
|
-
return;
|
|
5195
|
-
}
|
|
5196
|
-
// Direct embedders may invoke the engine without first going through
|
|
5197
|
-
// SqsPushReceiver. Preserve that seam while applying the same
|
|
5198
|
-
// delivery-wide partition the receiver would expose.
|
|
5199
|
-
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5200
|
-
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5201
|
-
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4249
|
+
// The injectable receiver seam and the production SQS receiver both
|
|
4250
|
+
// arrive here only after a successful receive. Keep the cadence tight
|
|
4251
|
+
// until the receiver later reports an empty outer long-poll.
|
|
4252
|
+
onReceiveActivity(true);
|
|
4253
|
+
// A received wake/high-water signal means there is bounded work now.
|
|
4254
|
+
// It wakes the scheduler but is still funneled through runGuarded below,
|
|
4255
|
+
// so it cannot overlap the pass that may currently own the operation
|
|
4256
|
+
// lock.
|
|
4257
|
+
// Apply the caps once across the whole delivery, then retain the residue
|
|
4258
|
+
// in the normal lane. The receiver already coalesces to one latest event
|
|
4259
|
+
// per path, so the partition cannot split revisions of one path.
|
|
4260
|
+
if (parallelCoordinatorEnabled) {
|
|
4261
|
+
const lane = receiverLaneByBatch.get(events);
|
|
4262
|
+
if (lane) {
|
|
4263
|
+
receiverLaneByBatch.delete(events);
|
|
4264
|
+
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5202
4265
|
return;
|
|
5203
4266
|
}
|
|
4267
|
+
// Direct embedders may invoke the engine without first going through
|
|
4268
|
+
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4269
|
+
// delivery-wide partition the receiver would expose.
|
|
5204
4270
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
}
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
}
|
|
5234
|
-
if (exitCode !== 0) {
|
|
4271
|
+
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4272
|
+
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4273
|
+
return;
|
|
4274
|
+
}
|
|
4275
|
+
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4276
|
+
const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
|
|
4277
|
+
const eventsByRoute = new Map();
|
|
4278
|
+
for (const event of laneEvents) {
|
|
4279
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
4280
|
+
// The receiver is an independent ingress from the local watcher.
|
|
4281
|
+
// Apply the same route-selection boundary before it can enqueue a
|
|
4282
|
+
// fast pull: a --skip-personal runner must acknowledge, not pull,
|
|
4283
|
+
// a personal-namespace delivery.
|
|
4284
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
4285
|
+
continue;
|
|
4286
|
+
const key = routeKey(route);
|
|
4287
|
+
const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
|
|
4288
|
+
group.events.push(event);
|
|
4289
|
+
if (event.kind === "upsert")
|
|
4290
|
+
group.upsertPaths.push(event.relativePath);
|
|
4291
|
+
eventsByRoute.set(key, group);
|
|
4292
|
+
}
|
|
4293
|
+
for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
|
|
4294
|
+
const paths = routeEvents.map((event) => event.relativePath);
|
|
4295
|
+
const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
|
|
4296
|
+
const exitCode = passExitCode(result);
|
|
4297
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
4298
|
+
if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
|
|
5235
4299
|
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
5236
4300
|
}
|
|
4301
|
+
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
5237
4302
|
await settleEvents?.(routeEvents);
|
|
4303
|
+
continue;
|
|
5238
4304
|
}
|
|
5239
|
-
|
|
5240
|
-
|
|
5241
|
-
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
|
|
4305
|
+
if (exitCode !== 0) {
|
|
4306
|
+
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4307
|
+
}
|
|
4308
|
+
await settleEvents?.(routeEvents);
|
|
4309
|
+
}
|
|
4310
|
+
};
|
|
4311
|
+
// Dispatch realtime work before awaiting its bulk residue. This lets an
|
|
4312
|
+
// established slow pass continue while the small scoped pull uses its
|
|
4313
|
+
// ref-counted lock carve-out; the residue remains serialized and cannot
|
|
4314
|
+
// starve the slow lane.
|
|
4315
|
+
await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
|
|
4316
|
+
await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
|
|
5251
4317
|
};
|
|
5252
4318
|
receiverSyncBatchFn.partition = (events, maxPaths) => {
|
|
5253
4319
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
@@ -5404,14 +4470,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5404
4470
|
if (stopped)
|
|
5405
4471
|
return shutdownPromise ?? Promise.resolve();
|
|
5406
4472
|
stopped = true;
|
|
5407
|
-
clearInterval(watcherPushStagesTimer);
|
|
5408
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
5409
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
5410
|
-
durableWatcherPersistTimer = undefined;
|
|
5411
|
-
if (syncWorkStateTimer !== undefined)
|
|
5412
|
-
clearTimeout(syncWorkStateTimer);
|
|
5413
|
-
syncWorkStateTimer = undefined;
|
|
5414
|
-
emitSyncWorkState(true);
|
|
5415
4473
|
for (const route of routePushRetryTimerHandles.keys()) {
|
|
5416
4474
|
clearRoutePushRetry(route);
|
|
5417
4475
|
}
|
|
@@ -5441,7 +4499,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5441
4499
|
coordinatorDisposal,
|
|
5442
4500
|
boundedEventSyncStartup,
|
|
5443
4501
|
eventSyncDisposal,
|
|
5444
|
-
persistDurableWatcherPaths(),
|
|
5445
4502
|
]).then(() => undefined);
|
|
5446
4503
|
try {
|
|
5447
4504
|
void receiver?.dispose();
|
|
@@ -5859,14 +4916,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5859
4916
|
lastCompletedFullReconcileWallAt = Date.now();
|
|
5860
4917
|
fullPassJournalHealthy = execution.accepted &&
|
|
5861
4918
|
isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
|
|
5862
|
-
if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
|
|
5863
|
-
clearDurableWatcherOverflow();
|
|
5864
|
-
}
|
|
5865
|
-
else if (fullPassJournalHealthy && pending.batch !== null) {
|
|
5866
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
5867
|
-
...pending.batch.paths.values(),
|
|
5868
|
-
]);
|
|
5869
|
-
}
|
|
5870
4919
|
if (fullPassJournalHealthy) {
|
|
5871
4920
|
fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
|
|
5872
4921
|
}
|
|
@@ -5924,11 +4973,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5924
4973
|
const catchupBatch = workingBatch;
|
|
5925
4974
|
const catchupGeneration = pending.generation;
|
|
5926
4975
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
5927
|
-
if (exit === 0)
|
|
5928
|
-
clearDurableWatcherOverflow();
|
|
5929
|
-
return;
|
|
5930
|
-
}
|
|
5931
|
-
if (exit === SCOPED_DRAIN_DEFERRED)
|
|
4976
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
5932
4977
|
return;
|
|
5933
4978
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
5934
4979
|
interruptWait();
|
|
@@ -6036,19 +5081,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
6036
5081
|
? exitCode === 0
|
|
6037
5082
|
: hasCompletedRunnerPass(result));
|
|
6038
5083
|
if (completedPostRepairPass) {
|
|
6039
|
-
if (!parallelCoordinatorEnabled &&
|
|
6040
|
-
pending.batch?.overflowed === true &&
|
|
6041
|
-
((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
|
|
6042
|
-
clearDurableWatcherOverflow();
|
|
6043
|
-
}
|
|
6044
|
-
else if (!parallelCoordinatorEnabled &&
|
|
6045
|
-
ranFullReconcile &&
|
|
6046
|
-
fullPassJournalHealthy &&
|
|
6047
|
-
pending.batch !== null) {
|
|
6048
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
6049
|
-
...pending.batch.paths.values(),
|
|
6050
|
-
]);
|
|
6051
|
-
}
|
|
6052
5084
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
6053
5085
|
// which follows journal repair. Its numeric seam has the same
|
|
6054
5086
|
// contract: zero means the injected pass completed. This preserves
|