@indigoai-us/hq-cloud 6.18.49 → 6.18.50-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +5 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +24 -2
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +85 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +1062 -123
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +30 -11
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +133 -15
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/watcher-catchup-skip-store.d.ts +60 -0
- package/dist/bin/watcher-catchup-skip-store.d.ts.map +1 -0
- package/dist/bin/watcher-catchup-skip-store.js +282 -0
- package/dist/bin/watcher-catchup-skip-store.js.map +1 -0
- package/dist/cli/share.d.ts +12 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +57 -4
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +3 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +27 -7
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts.map +1 -1
- package/dist/cli/tombstones.js +7 -2
- package/dist/cli/tombstones.js.map +1 -1
- package/dist/ignore.d.ts +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +2 -2
- package/dist/ignore.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +9 -2
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +34 -2
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/local-path-codec.d.ts.map +1 -1
- package/dist/local-path-codec.js +3 -2
- package/dist/local-path-codec.js.map +1 -1
- package/dist/personal-vault-exclusions.d.ts +2 -2
- package/dist/personal-vault-exclusions.d.ts.map +1 -1
- package/dist/personal-vault-exclusions.js +5 -3
- package/dist/personal-vault-exclusions.js.map +1 -1
- package/dist/personal-vault.d.ts +7 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/symlink-target-policy.d.ts +11 -0
- package/dist/symlink-target-policy.d.ts.map +1 -1
- package/dist/symlink-target-policy.js +80 -0
- package/dist/symlink-target-policy.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +29 -0
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +229 -16
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +5 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +32 -10
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/watcher.d.ts +16 -3
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +114 -21
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
4
5
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
5
6
|
import * as v8 from "node:v8";
|
|
6
7
|
import { performance } from "node:perf_hooks";
|
|
@@ -15,13 +16,18 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
|
15
16
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
16
17
|
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
17
18
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
19
|
+
import { isEphemeralPath } from "../cli/share.js";
|
|
20
|
+
import { isSafeRelativePath } from "../sync/push-event.js";
|
|
21
|
+
import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
|
|
22
|
+
import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
|
|
18
23
|
import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
|
|
19
24
|
import { createIgnoreFilter } from "../ignore.js";
|
|
25
|
+
import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
20
26
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
21
27
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
22
28
|
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
|
|
23
|
-
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
24
|
-
import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
29
|
+
import { isPersonalVaultDeletionExcluded, readCloudBackedSlugsFromManifest, computeContinuityPointerPaths, computeAgencySyncPaths, computeSessionLogsSyncPaths, isPersonalVaultScopeExcluded, } from "../personal-vault.js";
|
|
30
|
+
import { NoopPushReceiver, ReceiverDispatchError, RealtimePushYieldError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
25
31
|
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
26
32
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
27
33
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
@@ -61,13 +67,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
61
67
|
*/
|
|
62
68
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
63
69
|
/**
|
|
64
|
-
*
|
|
70
|
+
* Maximum guarded realtime executes, including the reserved watcher push peer.
|
|
65
71
|
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
66
|
-
* workers. One long
|
|
67
|
-
*
|
|
68
|
-
*
|
|
72
|
+
* workers. One long pull and one long push may each retain a normal worker;
|
|
73
|
+
* one additional watcher push can still make bounded progress. In-flight
|
|
74
|
+
* work is never aborted for this admission.
|
|
69
75
|
*/
|
|
70
|
-
export const REALTIME_WORKER_OCCUPANCY_CAP =
|
|
76
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 3;
|
|
71
77
|
/** Objects at least one multipart part are sent through the chore hand-off. */
|
|
72
78
|
export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
|
|
73
79
|
let realtimeAdmissionDeadlineMsForTest;
|
|
@@ -91,6 +97,29 @@ function realtimeDrainBurstMs() {
|
|
|
91
97
|
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
92
98
|
*/
|
|
93
99
|
export const REALTIME_ADMISSION_RETRY_MS = 5_000;
|
|
100
|
+
/** Bound consecutive pull admissions while a watcher push is queued. */
|
|
101
|
+
export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
|
|
102
|
+
/** Bound synchronous warm-up candidate work between event-loop turns. */
|
|
103
|
+
export const WATCHER_WARMUP_YIELD_EVERY = 128;
|
|
104
|
+
export function shouldYieldWatcherWarmup(examinedPaths) {
|
|
105
|
+
return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
|
|
106
|
+
}
|
|
107
|
+
export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
|
|
108
|
+
return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
|
|
109
|
+
}
|
|
110
|
+
export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
|
|
111
|
+
// TreeWatcher marks this only when its production filter has already applied
|
|
112
|
+
// the same personal scope predicate to every collected candidate. Unrouted
|
|
113
|
+
// paths retain the runner check because the collector may include company paths.
|
|
114
|
+
return pushFiltersApplied && routeKind === "personal";
|
|
115
|
+
}
|
|
116
|
+
export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
|
|
117
|
+
const ignoreFilter = createIgnore(hqRoot);
|
|
118
|
+
const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
|
|
119
|
+
onPersonalExcluded(relativePath, match.id);
|
|
120
|
+
});
|
|
121
|
+
return { ignoreFilter, personalPushFilter };
|
|
122
|
+
}
|
|
94
123
|
/**
|
|
95
124
|
* In-process result of a scoped drain whose only shortfall was admission
|
|
96
125
|
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
@@ -191,6 +220,32 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
|
|
|
191
220
|
return explicitlyAllowed ||
|
|
192
221
|
(workClass === "realtime" && isFullPassDurableProgressEnabled());
|
|
193
222
|
}
|
|
223
|
+
export function resolveWatcherPushWaitStage(input) {
|
|
224
|
+
if (input.capYieldActive)
|
|
225
|
+
return "CPU/heap cap wait";
|
|
226
|
+
if (input.holders.some((holder) => holder.kind === "pull"))
|
|
227
|
+
return "pull-yield wait";
|
|
228
|
+
if (input.queueDepth > 0)
|
|
229
|
+
return "queued behind realtime work";
|
|
230
|
+
if (input.holders.some((holder) => holder.kind === "chore"))
|
|
231
|
+
return "chore holder wait";
|
|
232
|
+
if (input.holders.length > 0)
|
|
233
|
+
return "scheduler holder wait";
|
|
234
|
+
return "scheduler admission pending";
|
|
235
|
+
}
|
|
236
|
+
export function watcherPushWaitDiagnostics(input) {
|
|
237
|
+
return {
|
|
238
|
+
stage: input.admissionWaiting
|
|
239
|
+
? resolveWatcherPushWaitStage(input)
|
|
240
|
+
: input.stage,
|
|
241
|
+
waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
|
|
242
|
+
? null
|
|
243
|
+
: Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
|
|
244
|
+
schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
|
|
245
|
+
? null
|
|
246
|
+
: Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
194
249
|
/**
|
|
195
250
|
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
196
251
|
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
@@ -205,6 +260,8 @@ export class CooperativePassScheduler {
|
|
|
205
260
|
starvationCapMs;
|
|
206
261
|
options;
|
|
207
262
|
pending = [];
|
|
263
|
+
lastAnyAdmissionAtMonotonicMs = null;
|
|
264
|
+
nextEntryId = 0;
|
|
208
265
|
active = null;
|
|
209
266
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
210
267
|
realtimeOccupants = new Set();
|
|
@@ -241,12 +298,58 @@ export class CooperativePassScheduler {
|
|
|
241
298
|
/** Successful callers still applying results after the network hand-off. */
|
|
242
299
|
networkTransferCompletionCount = 0;
|
|
243
300
|
networkTransferApplyRelease = null;
|
|
301
|
+
consecutivePullAdmissions = 0;
|
|
244
302
|
resolveNetworkTransferApplyRelease = null;
|
|
245
303
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
246
304
|
this.now = now;
|
|
247
305
|
this.starvationCapMs = starvationCapMs;
|
|
248
306
|
this.options = options;
|
|
249
307
|
}
|
|
308
|
+
recordAdmission(entry) {
|
|
309
|
+
const monotonicMs = performance.now();
|
|
310
|
+
const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
|
|
311
|
+
? null
|
|
312
|
+
: Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
|
|
313
|
+
const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
314
|
+
const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
|
|
315
|
+
const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
|
|
316
|
+
const waitingSinceAdmissionRequestMs = oldestPush
|
|
317
|
+
? Math.max(0, this.now() - oldestPush.enqueuedAt)
|
|
318
|
+
: null;
|
|
319
|
+
if (waitingPush || admittedIsPush) {
|
|
320
|
+
this.options.onAdmission?.({
|
|
321
|
+
entryId: entry.entryId,
|
|
322
|
+
workClass: entry.workClass,
|
|
323
|
+
lane: entry.lane,
|
|
324
|
+
transferDirection: entry.transferDirection,
|
|
325
|
+
transferSizeClass: entry.transferSizeClass ?? "unknown",
|
|
326
|
+
label: entry.label.slice(0, 160),
|
|
327
|
+
admittedAt: new Date().toISOString(),
|
|
328
|
+
schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
|
|
329
|
+
realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
|
|
330
|
+
oldestWaitingWatcherPush: oldestPush
|
|
331
|
+
? {
|
|
332
|
+
entryId: oldestPush.entryId,
|
|
333
|
+
label: oldestPush.label.slice(0, 160),
|
|
334
|
+
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
335
|
+
}
|
|
336
|
+
: null,
|
|
337
|
+
passedOver: waitingPush && waitingPush !== entry
|
|
338
|
+
? {
|
|
339
|
+
entryId: waitingPush.entryId,
|
|
340
|
+
label: waitingPush.label.slice(0, 160),
|
|
341
|
+
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
342
|
+
reason: entry.transferDirection === "pull"
|
|
343
|
+
? "pull-admitted-ahead-of-waiting-push"
|
|
344
|
+
: this._handoffActive
|
|
345
|
+
? "handoff-selector-admitted-other-work"
|
|
346
|
+
: "scheduler-admitted-other-work",
|
|
347
|
+
}
|
|
348
|
+
: null,
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
|
|
352
|
+
}
|
|
250
353
|
get handoffActive() {
|
|
251
354
|
return this._handoffActive;
|
|
252
355
|
}
|
|
@@ -263,12 +366,40 @@ export class CooperativePassScheduler {
|
|
|
263
366
|
get hasActiveSlowRealtime() {
|
|
264
367
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
265
368
|
}
|
|
369
|
+
get isInCapYield() {
|
|
370
|
+
return this.capYieldDepth > 0;
|
|
371
|
+
}
|
|
266
372
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
267
373
|
get activeRealtimeWorkerCount() {
|
|
268
374
|
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
269
375
|
(this.activeFastRealtime === null ? 0 : 1) +
|
|
270
376
|
this.realtimeOccupants.size);
|
|
271
377
|
}
|
|
378
|
+
get diagnosticAdmissionSnapshot() {
|
|
379
|
+
const active = [
|
|
380
|
+
this.active,
|
|
381
|
+
this.activeFastRealtime,
|
|
382
|
+
...this.realtimeOccupants,
|
|
383
|
+
this.detachedChore,
|
|
384
|
+
].filter((entry) => entry !== null);
|
|
385
|
+
const holders = active.map((entry) => ({
|
|
386
|
+
entryId: entry.entryId,
|
|
387
|
+
kind: entry.transferDirection === "pull"
|
|
388
|
+
? "pull"
|
|
389
|
+
: entry.workClass === "chore"
|
|
390
|
+
? "chore"
|
|
391
|
+
: "push",
|
|
392
|
+
scope: entry.label.slice(0, 160),
|
|
393
|
+
ageMs: entry.startedAtMonotonicMs === undefined
|
|
394
|
+
? 0
|
|
395
|
+
: Math.max(0, performance.now() - entry.startedAtMonotonicMs),
|
|
396
|
+
}));
|
|
397
|
+
return {
|
|
398
|
+
queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
|
|
399
|
+
holders: holders.slice(0, 20),
|
|
400
|
+
lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
272
403
|
pendingLaneCount(lane) {
|
|
273
404
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
274
405
|
}
|
|
@@ -283,6 +414,7 @@ export class CooperativePassScheduler {
|
|
|
283
414
|
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
284
415
|
}
|
|
285
416
|
const directEntry = {
|
|
417
|
+
entryId: `guarded-${++this.nextEntryId}`,
|
|
286
418
|
task,
|
|
287
419
|
workClass,
|
|
288
420
|
label,
|
|
@@ -306,7 +438,8 @@ export class CooperativePassScheduler {
|
|
|
306
438
|
if (workClass === "realtime" &&
|
|
307
439
|
this._handoffActive &&
|
|
308
440
|
directEntry.allowConcurrentHandoff &&
|
|
309
|
-
this.canAdmitRealtimeOccupant(directEntry)
|
|
441
|
+
this.canAdmitRealtimeOccupant(directEntry) &&
|
|
442
|
+
!this.pullMustYieldForWaitingPush(directEntry)) {
|
|
310
443
|
return new Promise((resolve, reject) => {
|
|
311
444
|
const entry = {
|
|
312
445
|
...directEntry,
|
|
@@ -455,16 +588,21 @@ export class CooperativePassScheduler {
|
|
|
455
588
|
this.admitHandoffBacklog();
|
|
456
589
|
return;
|
|
457
590
|
}
|
|
458
|
-
if (this.active !== null) {
|
|
591
|
+
if (this.active !== null || this.activeFastRealtime !== null || this.realtimeOccupants.size > 0) {
|
|
459
592
|
const fastRealtime = this.nextFastRealtime();
|
|
460
|
-
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
593
|
+
if (this.active !== null && fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
461
594
|
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
462
595
|
this.startFastRealtime(fastRealtime);
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
|
|
599
|
+
if (peerIndex >= 0) {
|
|
600
|
+
const peer = this.pending.splice(peerIndex, 1)[0];
|
|
601
|
+
if (peer)
|
|
602
|
+
this.startRealtimeOccupant(peer);
|
|
463
603
|
}
|
|
464
604
|
return;
|
|
465
605
|
}
|
|
466
|
-
if (this.activeFastRealtime !== null)
|
|
467
|
-
return;
|
|
468
606
|
const next = this.takeNext();
|
|
469
607
|
if (next)
|
|
470
608
|
this.start(next);
|
|
@@ -483,11 +621,29 @@ export class CooperativePassScheduler {
|
|
|
483
621
|
return this.pending.splice(realtime, 1)[0];
|
|
484
622
|
return this.pending.shift();
|
|
485
623
|
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
624
|
+
pullMustYieldForWaitingPush(entry) {
|
|
625
|
+
return entry.transferDirection === "pull" &&
|
|
626
|
+
this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH &&
|
|
627
|
+
this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
628
|
+
}
|
|
629
|
+
nextRealtimeIndex(eligible = () => true) {
|
|
630
|
+
const waitingPush = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.transferDirection === "push" && eligible(entry));
|
|
631
|
+
if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
|
|
632
|
+
this.consecutivePullAdmissions = 0;
|
|
633
|
+
return waitingPush;
|
|
634
|
+
}
|
|
635
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow" && eligible(entry));
|
|
636
|
+
const next = slowRealtime >= 0
|
|
489
637
|
? slowRealtime
|
|
490
|
-
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
638
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime" && eligible(entry));
|
|
639
|
+
if (next >= 0) {
|
|
640
|
+
const selected = this.pending[next];
|
|
641
|
+
if (selected?.transferDirection === "pull")
|
|
642
|
+
this.consecutivePullAdmissions += 1;
|
|
643
|
+
else if (selected?.transferDirection === "push")
|
|
644
|
+
this.consecutivePullAdmissions = 0;
|
|
645
|
+
}
|
|
646
|
+
return next;
|
|
491
647
|
}
|
|
492
648
|
nextFastRealtime() {
|
|
493
649
|
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
@@ -505,8 +661,20 @@ export class CooperativePassScheduler {
|
|
|
505
661
|
this.hasActiveSlowRealtime &&
|
|
506
662
|
!this.hasStarvedChore());
|
|
507
663
|
}
|
|
664
|
+
/** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
|
|
665
|
+
canAdmitRealtimePeer(entry) {
|
|
666
|
+
if (entry.workClass !== "realtime" ||
|
|
667
|
+
!entry.allowConcurrentHandoff ||
|
|
668
|
+
entry.transferDirection === undefined ||
|
|
669
|
+
this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
670
|
+
return false;
|
|
671
|
+
return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
|
|
672
|
+
active.transferDirection !== undefined &&
|
|
673
|
+
active.transferDirection !== entry.transferDirection);
|
|
674
|
+
}
|
|
508
675
|
start(entry) {
|
|
509
676
|
this.clearAdmission(entry);
|
|
677
|
+
this.recordAdmission(entry);
|
|
510
678
|
entry.startedAt = new Date().toISOString();
|
|
511
679
|
// Holder duration uses the process monotonic clock; do not consume the
|
|
512
680
|
// injected scheduler clock, whose reads define cadence-slot deadlines.
|
|
@@ -545,6 +713,8 @@ export class CooperativePassScheduler {
|
|
|
545
713
|
}
|
|
546
714
|
startFastRealtime(entry) {
|
|
547
715
|
this.clearAdmission(entry);
|
|
716
|
+
this.recordAdmission(entry);
|
|
717
|
+
entry.startedAtMonotonicMs = performance.now();
|
|
548
718
|
this.activeFastRealtime = entry;
|
|
549
719
|
let current;
|
|
550
720
|
try {
|
|
@@ -800,6 +970,8 @@ export class CooperativePassScheduler {
|
|
|
800
970
|
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
801
971
|
startRealtimeOccupant(entry) {
|
|
802
972
|
this.clearAdmission(entry);
|
|
973
|
+
this.recordAdmission(entry);
|
|
974
|
+
entry.startedAtMonotonicMs = performance.now();
|
|
803
975
|
this.realtimeOccupants.add(entry);
|
|
804
976
|
this.handoffAdmitted = true;
|
|
805
977
|
if (this.capYieldDepth > 0)
|
|
@@ -814,6 +986,7 @@ export class CooperativePassScheduler {
|
|
|
814
986
|
this.handoffBacklog = null;
|
|
815
987
|
entry.reject(error);
|
|
816
988
|
this.notifyOccupantChange();
|
|
989
|
+
this.drain();
|
|
817
990
|
return;
|
|
818
991
|
}
|
|
819
992
|
void running.then((outcome) => {
|
|
@@ -823,6 +996,7 @@ export class CooperativePassScheduler {
|
|
|
823
996
|
entry.resolve(outcome);
|
|
824
997
|
this.promoteOccupancyArrivals();
|
|
825
998
|
this.notifyOccupantChange();
|
|
999
|
+
this.drain();
|
|
826
1000
|
}, (error) => {
|
|
827
1001
|
this.realtimeOccupants.delete(entry);
|
|
828
1002
|
if (this.handoffBacklog === entry)
|
|
@@ -830,6 +1004,7 @@ export class CooperativePassScheduler {
|
|
|
830
1004
|
entry.reject(error);
|
|
831
1005
|
this.promoteOccupancyArrivals();
|
|
832
1006
|
this.notifyOccupantChange();
|
|
1007
|
+
this.drain();
|
|
833
1008
|
});
|
|
834
1009
|
}
|
|
835
1010
|
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
@@ -838,8 +1013,7 @@ export class CooperativePassScheduler {
|
|
|
838
1013
|
(!this.networkTransferActive && !this.withinHandoffAdmission()))
|
|
839
1014
|
return;
|
|
840
1015
|
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
841
|
-
const index = this.
|
|
842
|
-
pending.occupancyArrival === true &&
|
|
1016
|
+
const index = this.nextRealtimeIndex((pending) => pending.occupancyArrival === true &&
|
|
843
1017
|
pending.allowConcurrentHandoff &&
|
|
844
1018
|
this.canAdmitRealtimeOccupant(pending));
|
|
845
1019
|
if (index < 0)
|
|
@@ -858,7 +1032,7 @@ export class CooperativePassScheduler {
|
|
|
858
1032
|
return;
|
|
859
1033
|
if (!this.withinHandoffAdmission())
|
|
860
1034
|
return;
|
|
861
|
-
const realtimeIndex = this.nextRealtimeIndex();
|
|
1035
|
+
const realtimeIndex = this.nextRealtimeIndex((pending) => !pending.occupancyArrival && this.canAdmitRealtimeOccupant(pending));
|
|
862
1036
|
if (realtimeIndex < 0)
|
|
863
1037
|
return;
|
|
864
1038
|
const pending = this.pending[realtimeIndex];
|
|
@@ -1292,6 +1466,151 @@ export function isChoreLocalPush(paths, largeUpload) {
|
|
|
1292
1466
|
}
|
|
1293
1467
|
return count > 0;
|
|
1294
1468
|
}
|
|
1469
|
+
/** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
|
|
1470
|
+
export function watcherWarmupPathsByRoute(relativePaths) {
|
|
1471
|
+
const counts = new Map();
|
|
1472
|
+
for (const relativePath of relativePaths) {
|
|
1473
|
+
const route = routeChangeToTarget(relativePath);
|
|
1474
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1475
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1476
|
+
}
|
|
1477
|
+
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1478
|
+
const result = {};
|
|
1479
|
+
for (const [key, count] of ranked.slice(0, 20))
|
|
1480
|
+
result[key] = count;
|
|
1481
|
+
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1482
|
+
if (other > 0)
|
|
1483
|
+
result.other = other;
|
|
1484
|
+
return result;
|
|
1485
|
+
}
|
|
1486
|
+
export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
|
|
1487
|
+
const candidatePrefixes = new Map();
|
|
1488
|
+
const candidateCounts = new Map();
|
|
1489
|
+
for (const relativePath of candidateRelativePaths) {
|
|
1490
|
+
const route = routeChangeToTarget(relativePath);
|
|
1491
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1492
|
+
candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
|
|
1493
|
+
const routePath = route?.kind === "company"
|
|
1494
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1495
|
+
: relativePath;
|
|
1496
|
+
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1497
|
+
const routeCounts = candidatePrefixes.get(key) ?? new Map();
|
|
1498
|
+
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1499
|
+
candidatePrefixes.set(key, routeCounts);
|
|
1500
|
+
}
|
|
1501
|
+
const eligibleCounts = new Map();
|
|
1502
|
+
for (const relativePath of eligibleRelativePaths) {
|
|
1503
|
+
const route = routeChangeToTarget(relativePath);
|
|
1504
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1505
|
+
const routePath = route?.kind === "company"
|
|
1506
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1507
|
+
: relativePath;
|
|
1508
|
+
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1509
|
+
const routeCounts = eligibleCounts.get(key) ?? new Map();
|
|
1510
|
+
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1511
|
+
eligibleCounts.set(key, routeCounts);
|
|
1512
|
+
}
|
|
1513
|
+
const result = {};
|
|
1514
|
+
// Eligible routes are all retained; fill any remaining diagnostic slots
|
|
1515
|
+
// with the largest candidate routes. This keeps alphabetical one-path
|
|
1516
|
+
// stubs from crowding out a large route such as personal.
|
|
1517
|
+
const eligibleRoutes = [...eligibleCounts.keys()];
|
|
1518
|
+
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1519
|
+
const rankedCandidates = [...candidateCounts.entries()]
|
|
1520
|
+
.filter(([route]) => !eligibleCounts.has(route))
|
|
1521
|
+
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1522
|
+
const selectedRoutes = [
|
|
1523
|
+
...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
|
|
1524
|
+
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1525
|
+
];
|
|
1526
|
+
for (const route of selectedRoutes) {
|
|
1527
|
+
const topPrefixes = (counts) => {
|
|
1528
|
+
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1529
|
+
const top = {};
|
|
1530
|
+
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1531
|
+
top[prefix] = count;
|
|
1532
|
+
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1533
|
+
if (other > 0)
|
|
1534
|
+
top.other = other;
|
|
1535
|
+
return top;
|
|
1536
|
+
};
|
|
1537
|
+
result[route] = {
|
|
1538
|
+
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1539
|
+
eligible: topPrefixes(eligibleCounts.get(route)),
|
|
1540
|
+
};
|
|
1541
|
+
}
|
|
1542
|
+
const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
|
|
1543
|
+
if (otherRoutes.length > 0) {
|
|
1544
|
+
const merged = new Map();
|
|
1545
|
+
for (const [, routeCounts] of otherRoutes) {
|
|
1546
|
+
for (const [prefix, count] of routeCounts)
|
|
1547
|
+
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1548
|
+
}
|
|
1549
|
+
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1550
|
+
}
|
|
1551
|
+
return result;
|
|
1552
|
+
}
|
|
1553
|
+
function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
|
|
1554
|
+
counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
|
|
1555
|
+
const firstSlash = routePath.indexOf("/");
|
|
1556
|
+
const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
|
|
1557
|
+
const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
|
|
1558
|
+
let byPrefix = prefixes.get(routeName);
|
|
1559
|
+
if (!byPrefix) {
|
|
1560
|
+
byPrefix = new Map();
|
|
1561
|
+
prefixes.set(routeName, byPrefix);
|
|
1562
|
+
}
|
|
1563
|
+
byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
|
|
1564
|
+
}
|
|
1565
|
+
function rankedWarmupCounts(counts) {
|
|
1566
|
+
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1567
|
+
const result = {};
|
|
1568
|
+
for (const [key, count] of ranked.slice(0, 20))
|
|
1569
|
+
result[key] = count;
|
|
1570
|
+
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1571
|
+
if (other > 0)
|
|
1572
|
+
result.other = other;
|
|
1573
|
+
return result;
|
|
1574
|
+
}
|
|
1575
|
+
function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
|
|
1576
|
+
const topPrefixes = (counts) => {
|
|
1577
|
+
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1578
|
+
const top = {};
|
|
1579
|
+
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1580
|
+
top[prefix] = count;
|
|
1581
|
+
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1582
|
+
if (other > 0)
|
|
1583
|
+
top.other = other;
|
|
1584
|
+
return top;
|
|
1585
|
+
};
|
|
1586
|
+
const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
|
|
1587
|
+
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1588
|
+
const rankedCandidates = [...candidateCounts.entries()]
|
|
1589
|
+
.filter(([route]) => !eligibleCounts.has(route))
|
|
1590
|
+
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1591
|
+
const selectedRoutes = [
|
|
1592
|
+
...eligibleRoutes,
|
|
1593
|
+
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1594
|
+
];
|
|
1595
|
+
const result = {};
|
|
1596
|
+
for (const route of selectedRoutes) {
|
|
1597
|
+
result[route] = {
|
|
1598
|
+
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1599
|
+
eligible: topPrefixes(eligiblePrefixes.get(route)),
|
|
1600
|
+
};
|
|
1601
|
+
}
|
|
1602
|
+
const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
|
|
1603
|
+
if (otherRoutes.length > 0) {
|
|
1604
|
+
const merged = new Map();
|
|
1605
|
+
for (const route of otherRoutes) {
|
|
1606
|
+
for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
|
|
1607
|
+
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1611
|
+
}
|
|
1612
|
+
return result;
|
|
1613
|
+
}
|
|
1295
1614
|
/** A receiver event needs a known small upsert size before it can go realtime. */
|
|
1296
1615
|
export function isFastReceiverBatch(eventCount) {
|
|
1297
1616
|
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
@@ -1763,6 +2082,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1763
2082
|
let recycleRequested = null;
|
|
1764
2083
|
const pendingAppliedUpserts = new Map();
|
|
1765
2084
|
const pendingAppliedDeletes = new Set();
|
|
2085
|
+
const uploadedAwaitingPublish = new Set();
|
|
2086
|
+
const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
|
|
2087
|
+
const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
|
|
2088
|
+
let uploadedAwaitingPublishDropLogged = false;
|
|
1766
2089
|
const rememberAppliedUpsert = (relativePath, contentHash) => {
|
|
1767
2090
|
pendingAppliedDeletes.delete(relativePath);
|
|
1768
2091
|
if (eventSyncLive && eventSync) {
|
|
@@ -1793,6 +2116,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1793
2116
|
pendingAppliedDeletes.delete(oldest);
|
|
1794
2117
|
}
|
|
1795
2118
|
};
|
|
2119
|
+
const flushPendingUploadEvents = () => {
|
|
2120
|
+
if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
|
|
2121
|
+
return;
|
|
2122
|
+
while (uploadedAwaitingPublish.size > 0) {
|
|
2123
|
+
const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
|
|
2124
|
+
const paths = new Map(uploadedPaths.map((relativePath) => [
|
|
2125
|
+
localPathForVaultKey(hqRoot, relativePath),
|
|
2126
|
+
relativePath,
|
|
2127
|
+
]));
|
|
2128
|
+
const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
|
|
2129
|
+
if (expanded.size > 0)
|
|
2130
|
+
eventSync.publishBatch({ paths: expanded });
|
|
2131
|
+
for (const relativePath of uploadedPaths) {
|
|
2132
|
+
uploadedAwaitingPublish.delete(relativePath);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
};
|
|
1796
2136
|
const flushPendingAppliedUpserts = () => {
|
|
1797
2137
|
if (!eventSyncLive || !eventSync)
|
|
1798
2138
|
return;
|
|
@@ -1804,6 +2144,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1804
2144
|
eventSync.markContentDeleted?.(relativePath);
|
|
1805
2145
|
}
|
|
1806
2146
|
pendingAppliedDeletes.clear();
|
|
2147
|
+
flushPendingUploadEvents();
|
|
1807
2148
|
};
|
|
1808
2149
|
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
1809
2150
|
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
@@ -1853,8 +2194,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1853
2194
|
}, 0);
|
|
1854
2195
|
refresh.unref?.();
|
|
1855
2196
|
}
|
|
1856
|
-
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
1857
|
-
|
|
2197
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
2198
|
+
caps: fullPassCaps,
|
|
2199
|
+
onAdmission: (event) => {
|
|
2200
|
+
process.stderr.write(`${JSON.stringify({
|
|
2201
|
+
type: "watcher-push-scheduler-admission",
|
|
2202
|
+
diagnostic: true,
|
|
2203
|
+
...event,
|
|
2204
|
+
})}\n`);
|
|
2205
|
+
},
|
|
2206
|
+
});
|
|
2207
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
|
|
1858
2208
|
const passCheckpoint = pullSignal || receiverSignal
|
|
1859
2209
|
? Object.assign(async () => checkpoint?.(), {
|
|
1860
2210
|
shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
|
|
@@ -1884,20 +2234,43 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1884
2234
|
}
|
|
1885
2235
|
};
|
|
1886
2236
|
const invokePass = () => {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
2237
|
+
watcherPushStage?.("pass invocation");
|
|
2238
|
+
activePassCount += 1;
|
|
2239
|
+
emitSyncWorkState(true);
|
|
2240
|
+
const trackPass = (pass) => {
|
|
2241
|
+
const complete = () => {
|
|
2242
|
+
watcherPushStage?.("pass return");
|
|
2243
|
+
activePassCount = Math.max(0, activePassCount - 1);
|
|
2244
|
+
emitSyncWorkState(true);
|
|
2245
|
+
};
|
|
2246
|
+
// Observe settlement without returning a chained promise to the pass
|
|
2247
|
+
// scheduler. The scheduler's synchronous test seam and watcher wake
|
|
2248
|
+
// ordering depend on the original pass promise's completion turn.
|
|
2249
|
+
void pass.then(complete, complete);
|
|
2250
|
+
return pass;
|
|
2251
|
+
};
|
|
2252
|
+
try {
|
|
2253
|
+
if (!runPassSupportsCooperativeCheckpoint) {
|
|
2254
|
+
if (!runPassSupportsQmdLifecycle)
|
|
2255
|
+
return trackPass(runPass(passArgvForRun));
|
|
2256
|
+
return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
|
|
2257
|
+
}
|
|
2258
|
+
// This callback is emitted by the production runner after journal
|
|
2259
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
2260
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
2261
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
2262
|
+
const onJournalsReady = () => activateEventPushSurfaces();
|
|
2263
|
+
if (!runPassSupportsQmdLifecycle) {
|
|
2264
|
+
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
|
|
2265
|
+
}
|
|
2266
|
+
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
|
|
1891
2267
|
}
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
if (!runPassSupportsQmdLifecycle) {
|
|
1898
|
-
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
|
|
2268
|
+
catch (error) {
|
|
2269
|
+
watcherPushStage?.("pass return");
|
|
2270
|
+
activePassCount = Math.max(0, activePassCount - 1);
|
|
2271
|
+
emitSyncWorkState(true);
|
|
2272
|
+
throw error;
|
|
1899
2273
|
}
|
|
1900
|
-
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
|
|
1901
2274
|
};
|
|
1902
2275
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1903
2276
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -1912,6 +2285,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1912
2285
|
return;
|
|
1913
2286
|
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
1914
2287
|
recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
|
|
2288
|
+
rememberUploadedPaths(outcome);
|
|
2289
|
+
if (typeof outcome !== "number") {
|
|
2290
|
+
acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
|
|
2291
|
+
}
|
|
2292
|
+
flushPendingUploadEvents();
|
|
1915
2293
|
};
|
|
1916
2294
|
// Pre-admit boundary observation.
|
|
1917
2295
|
heapCheckpoint?.();
|
|
@@ -2162,15 +2540,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2162
2540
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2163
2541
|
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
|
|
2164
2542
|
};
|
|
2165
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
|
|
2543
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass, watcherPushStage) => {
|
|
2544
|
+
if (watcherPushStage) {
|
|
2545
|
+
watcherPushStage("guarded scheduler admission requested");
|
|
2546
|
+
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2547
|
+
if (passScheduler.isInCapYield)
|
|
2548
|
+
watcherPushStage("CPU/heap cap wait");
|
|
2549
|
+
else if (admission.holders.some((holder) => holder.kind === "pull")) {
|
|
2550
|
+
watcherPushStage("pull-yield wait");
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2166
2553
|
telemetry?.mark("operationGate");
|
|
2167
2554
|
const admissionSignal = signal && receiverSignal
|
|
2168
2555
|
? AbortSignal.any([signal, receiverSignal])
|
|
2169
2556
|
: signal ?? receiverSignal;
|
|
2170
2557
|
return runGuardedTask((context) => {
|
|
2558
|
+
watcherPushStage?.("guarded scheduler admission granted");
|
|
2171
2559
|
telemetry?.mark("appendLock");
|
|
2172
2560
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2173
|
-
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2561
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal, watcherPushStage);
|
|
2174
2562
|
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
|
|
2175
2563
|
};
|
|
2176
2564
|
/*
|
|
@@ -2292,6 +2680,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2292
2680
|
return {
|
|
2293
2681
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2294
2682
|
accepted,
|
|
2683
|
+
yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
|
|
2295
2684
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2296
2685
|
.filter((path) => path.status === "refused")
|
|
2297
2686
|
.map((path) => path.path)),
|
|
@@ -2489,6 +2878,222 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2489
2878
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
2490
2879
|
let drainedUntrackedDeletes = 0;
|
|
2491
2880
|
const pendingWatcherPaths = new Map();
|
|
2881
|
+
const durableWatcherPaths = new Map();
|
|
2882
|
+
const durableWatcherPathLimit = 10_000;
|
|
2883
|
+
let durableWatcherOverflowed = false;
|
|
2884
|
+
let durableWatcherPersistTimer;
|
|
2885
|
+
let durableWatcherPersistQueue = Promise.resolve();
|
|
2886
|
+
let durableWatcherPersistGeneration = 0;
|
|
2887
|
+
let durableWatcherPersistedGeneration = 0;
|
|
2888
|
+
const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
|
|
2889
|
+
const persistDurableWatcherPaths = () => {
|
|
2890
|
+
const generation = durableWatcherPersistGeneration;
|
|
2891
|
+
if (generation === durableWatcherPersistedGeneration)
|
|
2892
|
+
return durableWatcherPersistQueue;
|
|
2893
|
+
const snapshot = durableWatcherOverflowed
|
|
2894
|
+
? { format: 1, hqRoot, overflowed: true }
|
|
2895
|
+
: { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
|
|
2896
|
+
durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
|
|
2897
|
+
try {
|
|
2898
|
+
if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
|
|
2899
|
+
await fs.promises.rm(watcherPushStatePath, { force: true });
|
|
2900
|
+
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
|
|
2904
|
+
const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
|
|
2905
|
+
await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
|
|
2906
|
+
await fs.promises.rename(tempPath, watcherPushStatePath);
|
|
2907
|
+
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2908
|
+
}
|
|
2909
|
+
catch (err) {
|
|
2910
|
+
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2911
|
+
}
|
|
2912
|
+
}).catch((err) => {
|
|
2913
|
+
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2914
|
+
});
|
|
2915
|
+
return durableWatcherPersistQueue;
|
|
2916
|
+
};
|
|
2917
|
+
const scheduleDurableWatcherPersist = () => {
|
|
2918
|
+
durableWatcherPersistGeneration += 1;
|
|
2919
|
+
if (durableWatcherPersistTimer !== undefined)
|
|
2920
|
+
clearTimeout(durableWatcherPersistTimer);
|
|
2921
|
+
durableWatcherPersistTimer = setTimeout(() => {
|
|
2922
|
+
durableWatcherPersistTimer = undefined;
|
|
2923
|
+
void persistDurableWatcherPaths();
|
|
2924
|
+
}, 1_000);
|
|
2925
|
+
durableWatcherPersistTimer.unref?.();
|
|
2926
|
+
};
|
|
2927
|
+
const addDurableWatcherPath = (absolutePath, relativePath) => {
|
|
2928
|
+
if (durableWatcherOverflowed)
|
|
2929
|
+
return;
|
|
2930
|
+
if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
|
|
2931
|
+
durableWatcherPaths.clear();
|
|
2932
|
+
durableWatcherOverflowed = true;
|
|
2933
|
+
}
|
|
2934
|
+
else if (!durableWatcherOverflowed) {
|
|
2935
|
+
durableWatcherPaths.set(absolutePath, relativePath);
|
|
2936
|
+
}
|
|
2937
|
+
scheduleDurableWatcherPersist();
|
|
2938
|
+
};
|
|
2939
|
+
const clearDurableWatcherOverflow = () => {
|
|
2940
|
+
if (!durableWatcherOverflowed)
|
|
2941
|
+
return;
|
|
2942
|
+
durableWatcherOverflowed = false;
|
|
2943
|
+
durableWatcherPaths.clear();
|
|
2944
|
+
for (const [absolutePath, relativePath] of pendingWatcherPaths) {
|
|
2945
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
2946
|
+
}
|
|
2947
|
+
scheduleDurableWatcherPersist();
|
|
2948
|
+
};
|
|
2949
|
+
const acknowledgeDurableWatcherUploads = (relativePaths) => {
|
|
2950
|
+
let changed = false;
|
|
2951
|
+
const uploaded = new Set(relativePaths);
|
|
2952
|
+
for (const [absolutePath, relativePath] of durableWatcherPaths) {
|
|
2953
|
+
if (!uploaded.has(relativePath))
|
|
2954
|
+
continue;
|
|
2955
|
+
durableWatcherPaths.delete(absolutePath);
|
|
2956
|
+
changed = true;
|
|
2957
|
+
}
|
|
2958
|
+
if (changed)
|
|
2959
|
+
scheduleDurableWatcherPersist();
|
|
2960
|
+
};
|
|
2961
|
+
const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
|
|
2962
|
+
const stillPending = new Set(pendingWatcherPaths.values());
|
|
2963
|
+
acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
|
|
2964
|
+
};
|
|
2965
|
+
let activePassCount = 0;
|
|
2966
|
+
let inFlightWatcherPushes = 0;
|
|
2967
|
+
const watcherPushStages = new Map();
|
|
2968
|
+
let nextWatcherPushGroupId = 0;
|
|
2969
|
+
const setWatcherPushStage = (groupId, stage) => {
|
|
2970
|
+
const record = watcherPushStages.get(groupId);
|
|
2971
|
+
if (!record || record.stage === stage)
|
|
2972
|
+
return;
|
|
2973
|
+
record.stage = stage;
|
|
2974
|
+
record.stageEnteredAt = Date.now();
|
|
2975
|
+
if (stage === "guarded scheduler admission requested") {
|
|
2976
|
+
record.admissionRequestedAtMonotonicMs = performance.now();
|
|
2977
|
+
record.admissionWaiting = true;
|
|
2978
|
+
}
|
|
2979
|
+
else if (stage === "guarded scheduler admission granted") {
|
|
2980
|
+
record.admissionWaiting = false;
|
|
2981
|
+
}
|
|
2982
|
+
process.stderr.write(`${JSON.stringify({
|
|
2983
|
+
type: "watcher-push-group-stage",
|
|
2984
|
+
diagnostic: true,
|
|
2985
|
+
groupId: record.groupId,
|
|
2986
|
+
route: record.route,
|
|
2987
|
+
paths: record.paths,
|
|
2988
|
+
stage: record.stage,
|
|
2989
|
+
stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
|
|
2990
|
+
})}\n`);
|
|
2991
|
+
};
|
|
2992
|
+
const startWatcherPushStageTracking = (route, paths) => {
|
|
2993
|
+
const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
|
|
2994
|
+
watcherPushStages.set(groupId, {
|
|
2995
|
+
groupId,
|
|
2996
|
+
route,
|
|
2997
|
+
paths,
|
|
2998
|
+
stage: "",
|
|
2999
|
+
stageEnteredAt: Date.now(),
|
|
3000
|
+
admissionRequestedAtMonotonicMs: null,
|
|
3001
|
+
admissionWaiting: false,
|
|
3002
|
+
});
|
|
3003
|
+
return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
|
|
3004
|
+
};
|
|
3005
|
+
const emitWatcherPushInflightStages = () => {
|
|
3006
|
+
if (inFlightWatcherPushes <= 0)
|
|
3007
|
+
return;
|
|
3008
|
+
const now = Date.now();
|
|
3009
|
+
const entries = [...watcherPushStages.values()];
|
|
3010
|
+
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
3011
|
+
const coordinatorAdmission = coordinator?.inspect();
|
|
3012
|
+
for (const record of entries) {
|
|
3013
|
+
if (!record.admissionWaiting)
|
|
3014
|
+
continue;
|
|
3015
|
+
const liveStage = resolveWatcherPushWaitStage({
|
|
3016
|
+
capYieldActive: passScheduler.isInCapYield,
|
|
3017
|
+
queueDepth: admission.queueDepth,
|
|
3018
|
+
holders: admission.holders,
|
|
3019
|
+
});
|
|
3020
|
+
if (liveStage !== record.stage)
|
|
3021
|
+
setWatcherPushStage(record.groupId, liveStage);
|
|
3022
|
+
}
|
|
3023
|
+
process.stderr.write(`${JSON.stringify({
|
|
3024
|
+
type: "watcher-push-inflight-stages",
|
|
3025
|
+
diagnostic: true,
|
|
3026
|
+
groups: entries.slice(0, 20).map((record) => {
|
|
3027
|
+
const monotonicNow = performance.now();
|
|
3028
|
+
const current = watcherPushWaitDiagnostics({
|
|
3029
|
+
stage: record.stage,
|
|
3030
|
+
admissionWaiting: record.admissionWaiting,
|
|
3031
|
+
admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
|
|
3032
|
+
lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
|
|
3033
|
+
nowMonotonicMs: monotonicNow,
|
|
3034
|
+
capYieldActive: passScheduler.isInCapYield,
|
|
3035
|
+
queueDepth: admission.queueDepth,
|
|
3036
|
+
holders: admission.holders,
|
|
3037
|
+
});
|
|
3038
|
+
return {
|
|
3039
|
+
route: record.route,
|
|
3040
|
+
paths: record.paths,
|
|
3041
|
+
...current,
|
|
3042
|
+
ageMs: Math.max(0, now - record.stageEnteredAt),
|
|
3043
|
+
};
|
|
3044
|
+
}),
|
|
3045
|
+
other: Math.max(0, entries.length - 20),
|
|
3046
|
+
realtimeAdmissionQueueDepth: admission.queueDepth,
|
|
3047
|
+
holders: admission.holders,
|
|
3048
|
+
coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
|
|
3049
|
+
coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
|
|
3050
|
+
coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
|
|
3051
|
+
fast: null,
|
|
3052
|
+
slow: null,
|
|
3053
|
+
overflowPush: null,
|
|
3054
|
+
},
|
|
3055
|
+
coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
|
|
3056
|
+
coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
|
|
3057
|
+
coordinatorAdmission === undefined
|
|
3058
|
+
? null
|
|
3059
|
+
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
|
|
3060
|
+
coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
|
|
3061
|
+
coordinatorAdmission === undefined
|
|
3062
|
+
? null
|
|
3063
|
+
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
|
|
3064
|
+
})}\n`);
|
|
3065
|
+
};
|
|
3066
|
+
const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
|
|
3067
|
+
watcherPushStagesTimer.unref?.();
|
|
3068
|
+
let pendingReceivers = 0;
|
|
3069
|
+
let lastSyncWorkStateEmitAt = 0;
|
|
3070
|
+
let syncWorkStateTimer;
|
|
3071
|
+
const emitSyncWorkState = (force = false) => {
|
|
3072
|
+
const now = Date.now();
|
|
3073
|
+
const elapsed = now - lastSyncWorkStateEmitAt;
|
|
3074
|
+
if (force || elapsed >= 250) {
|
|
3075
|
+
if (syncWorkStateTimer !== undefined)
|
|
3076
|
+
clearTimeout(syncWorkStateTimer);
|
|
3077
|
+
syncWorkStateTimer = undefined;
|
|
3078
|
+
lastSyncWorkStateEmitAt = now;
|
|
3079
|
+
const snapshot = {
|
|
3080
|
+
passActive: activePassCount > 0,
|
|
3081
|
+
queuedWatcherPushes: pendingWatcherPaths.size,
|
|
3082
|
+
inFlightWatcherPushes,
|
|
3083
|
+
pendingReceivers,
|
|
3084
|
+
};
|
|
3085
|
+
deps.onSyncWorkStateSnapshot?.(snapshot);
|
|
3086
|
+
stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
|
|
3087
|
+
return;
|
|
3088
|
+
}
|
|
3089
|
+
if (syncWorkStateTimer !== undefined)
|
|
3090
|
+
return;
|
|
3091
|
+
syncWorkStateTimer = setTimeout(() => {
|
|
3092
|
+
syncWorkStateTimer = undefined;
|
|
3093
|
+
emitSyncWorkState(true);
|
|
3094
|
+
}, Math.max(1, 250 - elapsed));
|
|
3095
|
+
syncWorkStateTimer.unref?.();
|
|
3096
|
+
};
|
|
2492
3097
|
const droppedUnresolvedWatcherSlugs = new Set();
|
|
2493
3098
|
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
2494
3099
|
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
@@ -2556,6 +3161,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2556
3161
|
}
|
|
2557
3162
|
}
|
|
2558
3163
|
if (batch) {
|
|
3164
|
+
if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
|
|
3165
|
+
durableWatcherPaths.clear();
|
|
3166
|
+
durableWatcherOverflowed = true;
|
|
3167
|
+
scheduleDurableWatcherPersist();
|
|
3168
|
+
}
|
|
2559
3169
|
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
2560
3170
|
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
2561
3171
|
return previousSequence === undefined ||
|
|
@@ -2564,6 +3174,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2564
3174
|
});
|
|
2565
3175
|
if (entries.length === 0)
|
|
2566
3176
|
return false;
|
|
3177
|
+
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2567
3178
|
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
2568
3179
|
pendingWatcherOriginalBatch =
|
|
2569
3180
|
pendingWatcherPaths.size === 0 &&
|
|
@@ -2575,11 +3186,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2575
3186
|
: null;
|
|
2576
3187
|
for (const [absolutePath, relativePath] of entries) {
|
|
2577
3188
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3189
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
2578
3190
|
const change = batch.changes?.get(absolutePath);
|
|
2579
3191
|
if (change)
|
|
2580
3192
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2581
3193
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2582
3194
|
}
|
|
3195
|
+
emitSyncWorkState(queueWasEmpty);
|
|
2583
3196
|
if (batch.overflowed) {
|
|
2584
3197
|
pendingWatcherOverflowed = true;
|
|
2585
3198
|
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
@@ -2603,16 +3216,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2603
3216
|
// counterpart must be re-derived through the codec — a raw join would
|
|
2604
3217
|
// point at the impossible colon path on Windows.
|
|
2605
3218
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
3219
|
+
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2606
3220
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
3221
|
+
addDurableWatcherPath(absolutePath, changedRelPath);
|
|
2607
3222
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2608
3223
|
// A bare path signal carries no delete intent; drop any change record
|
|
2609
3224
|
// a prior batch left for this path so the stale intent cannot leak
|
|
2610
3225
|
// into the coalesced batch (#227).
|
|
2611
3226
|
pendingWatcherChanges.delete(absolutePath);
|
|
3227
|
+
emitSyncWorkState(queueWasEmpty);
|
|
2612
3228
|
return true;
|
|
2613
3229
|
}
|
|
2614
3230
|
pendingWatcherOriginalBatch = null;
|
|
2615
3231
|
pendingWatcherBareChange = true;
|
|
3232
|
+
emitSyncWorkState();
|
|
2616
3233
|
return true;
|
|
2617
3234
|
};
|
|
2618
3235
|
const takePendingWatcherChange = () => {
|
|
@@ -2657,6 +3274,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2657
3274
|
pendingWatcherDroppedRouteHints.clear();
|
|
2658
3275
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
2659
3276
|
pendingWatcherBatchGeneration = null;
|
|
3277
|
+
emitSyncWorkState();
|
|
2660
3278
|
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
2661
3279
|
};
|
|
2662
3280
|
const restorePendingWatcherPaths = (paths, generation, changes) => {
|
|
@@ -2670,12 +3288,54 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2670
3288
|
if (pendingWatcherChangeSequences.has(absolutePath))
|
|
2671
3289
|
continue;
|
|
2672
3290
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3291
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
2673
3292
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2674
3293
|
const change = changes?.get(absolutePath);
|
|
2675
3294
|
if (change)
|
|
2676
3295
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2677
3296
|
}
|
|
3297
|
+
emitSyncWorkState();
|
|
2678
3298
|
};
|
|
3299
|
+
if (eventPush) {
|
|
3300
|
+
try {
|
|
3301
|
+
const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
|
|
3302
|
+
if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
|
|
3303
|
+
(!Array.isArray(stored.paths) && stored.overflowed !== true)) {
|
|
3304
|
+
throw new Error("stored watcher push state has an unsupported shape or root");
|
|
3305
|
+
}
|
|
3306
|
+
if (stored.overflowed === true) {
|
|
3307
|
+
durableWatcherOverflowed = true;
|
|
3308
|
+
pendingWatcherOverflowed = true;
|
|
3309
|
+
pendingWatcherWarmupCatchup = true;
|
|
3310
|
+
pendingWatcherDroppedRoutesUnknown = true;
|
|
3311
|
+
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3312
|
+
emitSyncWorkState(true);
|
|
3313
|
+
process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
|
|
3314
|
+
}
|
|
3315
|
+
for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
|
|
3316
|
+
if (typeof relativePath !== "string" || relativePath.length === 0)
|
|
3317
|
+
continue;
|
|
3318
|
+
const route = routeChangeToTarget(relativePath);
|
|
3319
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3320
|
+
continue;
|
|
3321
|
+
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
3322
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
3323
|
+
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3324
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3325
|
+
}
|
|
3326
|
+
if (pendingWatcherPaths.size > 0) {
|
|
3327
|
+
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3328
|
+
pendingWatcherWarmupCatchup = true;
|
|
3329
|
+
emitSyncWorkState(true);
|
|
3330
|
+
process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
catch (err) {
|
|
3334
|
+
if (!(err instanceof Error && err.code === "ENOENT")) {
|
|
3335
|
+
process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
|
|
3336
|
+
}
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
2679
3339
|
const routePushFailureBackoff = new RoutePushFailureBackoff();
|
|
2680
3340
|
const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
|
|
2681
3341
|
const routePushRetryTimerHandles = new Map();
|
|
@@ -3256,14 +3916,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3256
3916
|
const relativePath = route.kind === "personal"
|
|
3257
3917
|
? key
|
|
3258
3918
|
: ["companies", route.slug, key].join("/");
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
if (
|
|
3263
|
-
fromRoot.startsWith(parentPrefix) ||
|
|
3264
|
-
path.isAbsolute(fromRoot)) {
|
|
3919
|
+
// Journal keys are POSIX relative paths. Reject unsafe segments with
|
|
3920
|
+
// a string-only check before building the absolute path; path.resolve
|
|
3921
|
+
// plus path.relative allocated and normalized every journal row.
|
|
3922
|
+
if (!isSafeRelativePath(relativePath))
|
|
3265
3923
|
continue;
|
|
3266
|
-
|
|
3924
|
+
const localPath = path.join(hqRoot, relativePath);
|
|
3267
3925
|
if (shouldSync(localPath)) {
|
|
3268
3926
|
paths.push(relativePath);
|
|
3269
3927
|
continue;
|
|
@@ -3526,8 +4184,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3526
4184
|
// a file and still be refused as stale (a download changed the directory
|
|
3527
4185
|
// mid-pass); the retry then finds the file already in the cloud and uploads
|
|
3528
4186
|
// nothing, so without this the file is never announced to peers.
|
|
3529
|
-
const uploadedAwaitingPublish = new Set();
|
|
3530
|
-
const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
|
|
3531
4187
|
const rememberUploadedPaths = (result) => {
|
|
3532
4188
|
if (result === undefined || typeof result === "number")
|
|
3533
4189
|
return;
|
|
@@ -3540,6 +4196,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3540
4196
|
const oldest = uploadedAwaitingPublish.values().next().value;
|
|
3541
4197
|
if (oldest !== undefined)
|
|
3542
4198
|
uploadedAwaitingPublish.delete(oldest);
|
|
4199
|
+
if (!uploadedAwaitingPublishDropLogged) {
|
|
4200
|
+
uploadedAwaitingPublishDropLogged = true;
|
|
4201
|
+
process.stderr.write(`${JSON.stringify({
|
|
4202
|
+
type: "event-sync-upload-buffer-overflow",
|
|
4203
|
+
cap: UPLOADED_AWAITING_PUBLISH_CAP,
|
|
4204
|
+
action: "drop-oldest",
|
|
4205
|
+
})}\n`);
|
|
4206
|
+
}
|
|
3543
4207
|
}
|
|
3544
4208
|
}
|
|
3545
4209
|
};
|
|
@@ -3654,19 +4318,48 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3654
4318
|
? localUploadSizeClass(uploadPaths)
|
|
3655
4319
|
: "small";
|
|
3656
4320
|
const largeUpload = sizeClass !== "small";
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
4321
|
+
// With coordinator fencing enabled, even large watcher batches can
|
|
4322
|
+
// use bounded realtime admission. Keeping them as chores would make
|
|
4323
|
+
// them wait behind a long-lived receiver pull's slot indefinitely.
|
|
4324
|
+
const chorePush = isChoreLocalPush(group.relPaths, largeUpload && !parallelCoordinatorEnabled);
|
|
4325
|
+
const { groupId, setStage: watcherPushStage } = startWatcherPushStageTracking(routeId, group.relPaths.length);
|
|
4326
|
+
watcherPushStage("coordinator submit");
|
|
4327
|
+
inFlightWatcherPushes += group.relPaths.length;
|
|
4328
|
+
emitSyncWorkState(true);
|
|
4329
|
+
emitWatcherPushInflightStages();
|
|
4330
|
+
let execution;
|
|
4331
|
+
try {
|
|
4332
|
+
execution = await (parallelCoordinatorEnabled && !chorePush
|
|
4333
|
+
? runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
|
|
4334
|
+
watcherPushStage("worker entry");
|
|
4335
|
+
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
4336
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
4337
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
4338
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
4339
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
4340
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
|
|
4341
|
+
// Watcher-originated pushes may share the guarded
|
|
4342
|
+
// realtime slot with an in-flight pull. Coordinator
|
|
4343
|
+
// fencing already serializes overlapping path epochs;
|
|
4344
|
+
// this must not depend on the optional full-pass flag.
|
|
4345
|
+
true, undefined, routeId, watcherPushStage);
|
|
4346
|
+
}, sizeClass)
|
|
4347
|
+
: (() => {
|
|
4348
|
+
watcherPushStage("worker entry");
|
|
4349
|
+
return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
4350
|
+
? () => prepareWatcherChanges(group.changes)
|
|
4351
|
+
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
|
|
4352
|
+
result,
|
|
4353
|
+
accepted: true,
|
|
4354
|
+
coordinatorRefusedPaths: [],
|
|
4355
|
+
}));
|
|
4356
|
+
})());
|
|
4357
|
+
}
|
|
4358
|
+
finally {
|
|
4359
|
+
watcherPushStages.delete(groupId);
|
|
4360
|
+
inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
|
|
4361
|
+
emitSyncWorkState(true);
|
|
4362
|
+
}
|
|
3670
4363
|
const result = execution.result;
|
|
3671
4364
|
rememberUploadedPaths(result);
|
|
3672
4365
|
const pushExit = passExitCode(result);
|
|
@@ -3685,6 +4378,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3685
4378
|
}
|
|
3686
4379
|
const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
|
|
3687
4380
|
relativePath.startsWith(`${refusedPath}/`));
|
|
4381
|
+
acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
|
|
3688
4382
|
const publishPaths = new Map();
|
|
3689
4383
|
for (const [absolutePath, relativePath] of group.paths) {
|
|
3690
4384
|
if (isRefusedPath(relativePath))
|
|
@@ -3724,9 +4418,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3724
4418
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
3725
4419
|
}
|
|
3726
4420
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
4421
|
+
acknowledgeDurableWatcherUploads(group.relPaths);
|
|
3727
4422
|
routePushFailureBackoff.succeeded(routeId);
|
|
3728
4423
|
clearRoutePushRetry(routeId);
|
|
3729
4424
|
if (group.route.kind === "company") {
|
|
4425
|
+
deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
|
|
3730
4426
|
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
3731
4427
|
}
|
|
3732
4428
|
}
|
|
@@ -3788,7 +4484,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3788
4484
|
const execution = parallelCoordinatorEnabled
|
|
3789
4485
|
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
3790
4486
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
3791
|
-
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
4487
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route));
|
|
3792
4488
|
})
|
|
3793
4489
|
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
3794
4490
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
@@ -3840,6 +4536,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3840
4536
|
addRoute(hint);
|
|
3841
4537
|
const skipCompanies = resolveSkipCompanies();
|
|
3842
4538
|
if (parallelCoordinatorEnabled) {
|
|
4539
|
+
let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
|
|
3843
4540
|
for (const route of routes.values()) {
|
|
3844
4541
|
if (route.kind === "company" &&
|
|
3845
4542
|
skipCompanies.has(route.slug.toLowerCase())) {
|
|
@@ -3868,6 +4565,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3868
4565
|
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
3869
4566
|
interruptWait();
|
|
3870
4567
|
}
|
|
4568
|
+
else {
|
|
4569
|
+
routesRemaining -= 1;
|
|
4570
|
+
if (routesRemaining === 0)
|
|
4571
|
+
clearDurableWatcherOverflow();
|
|
4572
|
+
}
|
|
3871
4573
|
});
|
|
3872
4574
|
}
|
|
3873
4575
|
return 0;
|
|
@@ -3983,6 +4685,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3983
4685
|
// churn overflowed the 200-path batch and skipped publishBatch.
|
|
3984
4686
|
personalMode: true,
|
|
3985
4687
|
includeCompanyPaths: companiesMode || companyMode,
|
|
4688
|
+
// Inventory filtered candidates so the route's push filter remains
|
|
4689
|
+
// authoritative and can attribute exclusions in warm-up diagnostics.
|
|
4690
|
+
includeFilteredWarmupCandidates: true,
|
|
3986
4691
|
onDegraded: opts.onDegraded,
|
|
3987
4692
|
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
3988
4693
|
queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
|
|
@@ -4104,6 +4809,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4104
4809
|
}
|
|
4105
4810
|
}
|
|
4106
4811
|
watcher.start();
|
|
4812
|
+
let catchupPathStatWarningReported = false;
|
|
4107
4813
|
const reportWatcherReady = () => {
|
|
4108
4814
|
if (stopped || watcher === null)
|
|
4109
4815
|
return;
|
|
@@ -4144,14 +4850,202 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4144
4850
|
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
|
|
4145
4851
|
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
4146
4852
|
return;
|
|
4147
|
-
const
|
|
4853
|
+
const skipStore = deps.watcherCatchupSkipStore;
|
|
4854
|
+
const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
|
|
4855
|
+
const eligiblePaths = new Map();
|
|
4856
|
+
const eligibleChanges = new Map();
|
|
4857
|
+
const excludedByReason = {};
|
|
4858
|
+
const excludedByReasonByRoute = {};
|
|
4859
|
+
const personalFilterExclusions = new Map();
|
|
4860
|
+
const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
|
|
4861
|
+
const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
|
|
4862
|
+
const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
|
|
4863
|
+
const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
|
|
4864
|
+
const clearedResolvedCompanies = new Set();
|
|
4865
|
+
const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
|
|
4866
|
+
const candidateRouteCounts = new Map();
|
|
4867
|
+
const candidateRoutePrefixes = new Map();
|
|
4868
|
+
const eligibleRouteCounts = new Map();
|
|
4869
|
+
const eligibleRoutePrefixes = new Map();
|
|
4870
|
+
const backupClass = (relativePath) => {
|
|
4871
|
+
if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
|
|
4872
|
+
return "attioCleanup";
|
|
4873
|
+
if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
|
|
4874
|
+
return "readModelRollout";
|
|
4875
|
+
return undefined;
|
|
4876
|
+
};
|
|
4877
|
+
const exclude = (reason, routeName) => {
|
|
4878
|
+
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
|
|
4879
|
+
const routeReasons = excludedByReasonByRoute[routeName] ??= {};
|
|
4880
|
+
routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
|
|
4881
|
+
};
|
|
4882
|
+
let filteredCandidateCount = 0;
|
|
4883
|
+
for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
|
|
4884
|
+
filteredCandidateCount += count;
|
|
4885
|
+
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
|
|
4886
|
+
const routeReasons = excludedByReasonByRoute.filtered ??= {};
|
|
4887
|
+
routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
|
|
4888
|
+
}
|
|
4889
|
+
if (filteredCandidateCount > 0)
|
|
4890
|
+
candidateRouteCounts.set("filtered", filteredCandidateCount);
|
|
4891
|
+
let examinedWarmupPaths = 0;
|
|
4892
|
+
for (const [absolutePath, relativePath] of catchup.batch.paths) {
|
|
4893
|
+
examinedWarmupPaths += 1;
|
|
4894
|
+
const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
|
|
4895
|
+
if (warmupYield)
|
|
4896
|
+
await warmupYield;
|
|
4897
|
+
if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
|
|
4898
|
+
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
|
|
4899
|
+
exclude("unsafe-path", "unrouted");
|
|
4900
|
+
continue;
|
|
4901
|
+
}
|
|
4902
|
+
const route = routeChangeToTarget(relativePath);
|
|
4903
|
+
const routeName = route ? routeKey(route) : "unrouted";
|
|
4904
|
+
const summaryRoutePath = route?.kind === "company"
|
|
4905
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4906
|
+
: relativePath;
|
|
4907
|
+
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
|
|
4908
|
+
const ontologyBackupClass = backupClass(relativePath);
|
|
4909
|
+
if (ontologyBackupClass) {
|
|
4910
|
+
exclude("local-rule:ontology-backup", routeName);
|
|
4911
|
+
ontologyBackupPaths[ontologyBackupClass] += 1;
|
|
4912
|
+
continue;
|
|
4913
|
+
}
|
|
4914
|
+
if (route?.kind === "company") {
|
|
4915
|
+
const resolvedNow = cloudCompanySlugs.has(route.slug) ||
|
|
4916
|
+
skipStore?.hasCurrentGrantCompany(route.slug) ||
|
|
4917
|
+
skipStore?.hasCurrentGrantVersion(route.slug) ||
|
|
4918
|
+
(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
|
|
4919
|
+
if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
|
|
4920
|
+
skipStore?.clearUnresolvedCompany(route.slug);
|
|
4921
|
+
clearedResolvedCompanies.add(route.slug);
|
|
4922
|
+
}
|
|
4923
|
+
if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
|
|
4924
|
+
exclude("unresolved-company", routeName);
|
|
4925
|
+
continue;
|
|
4926
|
+
}
|
|
4927
|
+
}
|
|
4928
|
+
if (route?.kind === "company" &&
|
|
4929
|
+
!cloudCompanySlugs.has(route.slug) &&
|
|
4930
|
+
!skipStore?.hasCurrentGrantCompany(route.slug) &&
|
|
4931
|
+
!skipStore?.hasCurrentGrantVersion(route.slug) &&
|
|
4932
|
+
!(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
|
|
4933
|
+
exclude("local-only-company", routeName);
|
|
4934
|
+
continue;
|
|
4935
|
+
}
|
|
4936
|
+
const company = route?.kind === "company" ? route.slug : "personal";
|
|
4937
|
+
const routePath = route?.kind === "company"
|
|
4938
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4939
|
+
: relativePath;
|
|
4940
|
+
if (isEphemeralPath(routePath)) {
|
|
4941
|
+
exclude("push-filter:conflict-mirror", routeName);
|
|
4942
|
+
continue;
|
|
4943
|
+
}
|
|
4944
|
+
// The production TreeWatcher collector has already applied this
|
|
4945
|
+
// shared filter stack and reports rejected files as reason counts.
|
|
4946
|
+
// Injected/custom collectors without that proof still evaluate the
|
|
4947
|
+
// route filter here, with personal defaults first for `.env` attribution.
|
|
4948
|
+
const personalDefaultExclusion = catchup.pushFiltersApplied === true
|
|
4949
|
+
? undefined
|
|
4950
|
+
: route?.kind === "personal"
|
|
4951
|
+
? matchPersonalVaultExclusion(relativePath, false)
|
|
4952
|
+
: undefined;
|
|
4953
|
+
const isPersonalRoute = route?.kind !== "company";
|
|
4954
|
+
const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
|
|
4955
|
+
? personalPushFilter(absolutePath, false, relativePath)
|
|
4956
|
+
: ignoreFilter(absolutePath, false, relativePath)));
|
|
4957
|
+
if (!acceptedByPushFilter) {
|
|
4958
|
+
const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
|
|
4959
|
+
? personalFilterExclusions.get(relativePath)
|
|
4960
|
+
: undefined);
|
|
4961
|
+
exclude(exclusionId
|
|
4962
|
+
? `push-filter:personal:${exclusionId}`
|
|
4963
|
+
: "push-filter:ignore", routeName);
|
|
4964
|
+
continue;
|
|
4965
|
+
}
|
|
4966
|
+
if (isPersonalRoute &&
|
|
4967
|
+
!shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
|
|
4968
|
+
isPersonalVaultScopeExcluded(relativePath, {
|
|
4969
|
+
continuityPaths,
|
|
4970
|
+
hasAgencyCarveout,
|
|
4971
|
+
hasSessionLogsCarveout,
|
|
4972
|
+
})) {
|
|
4973
|
+
exclude("push-filter:personal:excluded-top-level", routeName);
|
|
4974
|
+
continue;
|
|
4975
|
+
}
|
|
4976
|
+
if (route && isLocallyIneligibleForPush(routePath, {
|
|
4977
|
+
boardClientOwned: true,
|
|
4978
|
+
ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
|
|
4979
|
+
})) {
|
|
4980
|
+
const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
|
|
4981
|
+
exclude(`local-rule:${pathClass}`, routeName);
|
|
4982
|
+
continue;
|
|
4983
|
+
}
|
|
4984
|
+
const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
|
|
4985
|
+
const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
|
|
4986
|
+
try {
|
|
4987
|
+
const stat = !catchup.symlinkPaths || hasSkipRecord
|
|
4988
|
+
? fs.lstatSync(absolutePath)
|
|
4989
|
+
: undefined;
|
|
4990
|
+
if (route && (knownSymlink || stat?.isSymbolicLink())) {
|
|
4991
|
+
const scopeRoot = route.kind === "company"
|
|
4992
|
+
? path.join(hqRoot, "companies", route.slug)
|
|
4993
|
+
: relativePath.startsWith("personal/")
|
|
4994
|
+
? path.join(hqRoot, "personal")
|
|
4995
|
+
: hqRoot;
|
|
4996
|
+
const inspection = inspectLocalSymlinkScope({
|
|
4997
|
+
absoluteLinkPath: absolutePath,
|
|
4998
|
+
linkPath: relativePath,
|
|
4999
|
+
scopeRoot,
|
|
5000
|
+
scope: route.kind === "company"
|
|
5001
|
+
? { kind: "company", companySlug: route.slug }
|
|
5002
|
+
: { kind: "personal" },
|
|
5003
|
+
});
|
|
5004
|
+
if (inspection.refused) {
|
|
5005
|
+
exclude("symlink", routeName);
|
|
5006
|
+
continue;
|
|
5007
|
+
}
|
|
5008
|
+
}
|
|
5009
|
+
if (hasSkipRecord && stat &&
|
|
5010
|
+
skipStore?.matches(company, routePath, stat)) {
|
|
5011
|
+
exclude("record-match", routeName);
|
|
5012
|
+
continue;
|
|
5013
|
+
}
|
|
5014
|
+
}
|
|
5015
|
+
catch (error) {
|
|
5016
|
+
// Fail open: unreadable state or a filesystem race must not suppress a candidate.
|
|
5017
|
+
if (!catchupPathStatWarningReported) {
|
|
5018
|
+
catchupPathStatWarningReported = true;
|
|
5019
|
+
process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
|
|
5020
|
+
}
|
|
5021
|
+
}
|
|
5022
|
+
eligiblePaths.set(absolutePath, relativePath);
|
|
5023
|
+
recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
|
|
5024
|
+
const change = catchup.batch.changes?.get(absolutePath);
|
|
5025
|
+
if (change)
|
|
5026
|
+
eligibleChanges.set(absolutePath, change);
|
|
5027
|
+
}
|
|
5028
|
+
const queued = addPendingWatcherChange(undefined, {
|
|
5029
|
+
paths: eligiblePaths,
|
|
5030
|
+
changes: eligibleChanges,
|
|
5031
|
+
}, {
|
|
4148
5032
|
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
4149
5033
|
});
|
|
4150
5034
|
pendingWatcherWarmupCatchup ||= queued;
|
|
5035
|
+
const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
|
|
4151
5036
|
process.stderr.write(JSON.stringify({
|
|
4152
5037
|
type: "watcher-warmup-catchup",
|
|
4153
5038
|
reason: "watch-set-ready",
|
|
4154
|
-
pathsFound:
|
|
5039
|
+
pathsFound: eligiblePaths.size,
|
|
5040
|
+
candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
|
|
5041
|
+
pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
|
|
5042
|
+
pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
|
|
5043
|
+
excludedByReason: {
|
|
5044
|
+
...excludedByReason,
|
|
5045
|
+
...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
|
|
5046
|
+
},
|
|
5047
|
+
excludedByReasonByRoute,
|
|
5048
|
+
ontologyBackupPaths,
|
|
4155
5049
|
scannedPaths: catchup.scannedPaths,
|
|
4156
5050
|
elapsedMs: Date.now() - startedAt,
|
|
4157
5051
|
queued,
|
|
@@ -4218,13 +5112,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4218
5112
|
const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
4219
5113
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4220
5114
|
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
4221
|
-
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
5115
|
+
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal, routeKey(route)));
|
|
4222
5116
|
});
|
|
4223
5117
|
const pathCount = events.length;
|
|
4224
5118
|
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
4225
5119
|
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
4226
5120
|
if (execution.admissionTimeout)
|
|
4227
5121
|
throw execution.admissionTimeout;
|
|
5122
|
+
if (execution.yieldedToPush) {
|
|
5123
|
+
throw new RealtimePushYieldError();
|
|
5124
|
+
}
|
|
4228
5125
|
const exitCode = passExitCode(execution.result);
|
|
4229
5126
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
4230
5127
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -4246,74 +5143,82 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4246
5143
|
const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
|
|
4247
5144
|
if (stopped)
|
|
4248
5145
|
return;
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
5146
|
+
try {
|
|
5147
|
+
// The injectable receiver seam and the production SQS receiver both
|
|
5148
|
+
// arrive here only after a successful receive. Keep the cadence tight
|
|
5149
|
+
// until the receiver later reports an empty outer long-poll.
|
|
5150
|
+
onReceiveActivity(true);
|
|
5151
|
+
pendingReceivers += events.length;
|
|
5152
|
+
emitSyncWorkState();
|
|
5153
|
+
// A received wake/high-water signal means there is bounded work now.
|
|
5154
|
+
// It wakes the scheduler but is still funneled through runGuarded below,
|
|
5155
|
+
// so it cannot overlap the pass that may currently own the operation
|
|
5156
|
+
// lock.
|
|
5157
|
+
// Apply the caps once across the whole delivery, then retain the residue
|
|
5158
|
+
// in the normal lane. The receiver already coalesces to one latest event
|
|
5159
|
+
// per path, so the partition cannot split revisions of one path.
|
|
5160
|
+
if (parallelCoordinatorEnabled) {
|
|
5161
|
+
const lane = receiverLaneByBatch.get(events);
|
|
5162
|
+
if (lane) {
|
|
5163
|
+
receiverLaneByBatch.delete(events);
|
|
5164
|
+
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5165
|
+
return;
|
|
5166
|
+
}
|
|
5167
|
+
// Direct embedders may invoke the engine without first going through
|
|
5168
|
+
// SqsPushReceiver. Preserve that seam while applying the same
|
|
5169
|
+
// delivery-wide partition the receiver would expose.
|
|
5170
|
+
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5171
|
+
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5172
|
+
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4265
5173
|
return;
|
|
4266
5174
|
}
|
|
4267
|
-
// Direct embedders may invoke the engine without first going through
|
|
4268
|
-
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4269
|
-
// delivery-wide partition the receiver would expose.
|
|
4270
5175
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
5176
|
+
const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
|
|
5177
|
+
const eventsByRoute = new Map();
|
|
5178
|
+
for (const event of laneEvents) {
|
|
5179
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
5180
|
+
// The receiver is an independent ingress from the local watcher.
|
|
5181
|
+
// Apply the same route-selection boundary before it can enqueue a
|
|
5182
|
+
// fast pull: a --skip-personal runner must acknowledge, not pull,
|
|
5183
|
+
// a personal-namespace delivery.
|
|
5184
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
5185
|
+
continue;
|
|
5186
|
+
const key = routeKey(route);
|
|
5187
|
+
const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
|
|
5188
|
+
group.events.push(event);
|
|
5189
|
+
if (event.kind === "upsert")
|
|
5190
|
+
group.upsertPaths.push(event.relativePath);
|
|
5191
|
+
eventsByRoute.set(key, group);
|
|
5192
|
+
}
|
|
5193
|
+
for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
|
|
5194
|
+
const paths = routeEvents.map((event) => event.relativePath);
|
|
5195
|
+
const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
|
|
5196
|
+
const exitCode = passExitCode(result);
|
|
5197
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
5198
|
+
if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
|
|
5199
|
+
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
5200
|
+
}
|
|
5201
|
+
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
5202
|
+
await settleEvents?.(routeEvents);
|
|
5203
|
+
continue;
|
|
5204
|
+
}
|
|
5205
|
+
if (exitCode !== 0) {
|
|
4299
5206
|
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4300
5207
|
}
|
|
4301
|
-
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
4302
5208
|
await settleEvents?.(routeEvents);
|
|
4303
|
-
continue;
|
|
4304
|
-
}
|
|
4305
|
-
if (exitCode !== 0) {
|
|
4306
|
-
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4307
5209
|
}
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
5210
|
+
};
|
|
5211
|
+
// Dispatch realtime work before awaiting its bulk residue. This lets an
|
|
5212
|
+
// established slow pass continue while the small scoped pull uses its
|
|
5213
|
+
// ref-counted lock carve-out; the residue remains serialized and cannot
|
|
5214
|
+
// starve the slow lane.
|
|
5215
|
+
await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
|
|
5216
|
+
await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
|
|
5217
|
+
}
|
|
5218
|
+
finally {
|
|
5219
|
+
pendingReceivers = Math.max(0, pendingReceivers - events.length);
|
|
5220
|
+
emitSyncWorkState();
|
|
5221
|
+
}
|
|
4317
5222
|
};
|
|
4318
5223
|
receiverSyncBatchFn.partition = (events, maxPaths) => {
|
|
4319
5224
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
@@ -4470,6 +5375,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4470
5375
|
if (stopped)
|
|
4471
5376
|
return shutdownPromise ?? Promise.resolve();
|
|
4472
5377
|
stopped = true;
|
|
5378
|
+
clearInterval(watcherPushStagesTimer);
|
|
5379
|
+
if (durableWatcherPersistTimer !== undefined)
|
|
5380
|
+
clearTimeout(durableWatcherPersistTimer);
|
|
5381
|
+
durableWatcherPersistTimer = undefined;
|
|
5382
|
+
if (syncWorkStateTimer !== undefined)
|
|
5383
|
+
clearTimeout(syncWorkStateTimer);
|
|
5384
|
+
syncWorkStateTimer = undefined;
|
|
5385
|
+
emitSyncWorkState(true);
|
|
4473
5386
|
for (const route of routePushRetryTimerHandles.keys()) {
|
|
4474
5387
|
clearRoutePushRetry(route);
|
|
4475
5388
|
}
|
|
@@ -4499,6 +5412,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4499
5412
|
coordinatorDisposal,
|
|
4500
5413
|
boundedEventSyncStartup,
|
|
4501
5414
|
eventSyncDisposal,
|
|
5415
|
+
persistDurableWatcherPaths(),
|
|
4502
5416
|
]).then(() => undefined);
|
|
4503
5417
|
try {
|
|
4504
5418
|
void receiver?.dispose();
|
|
@@ -4916,6 +5830,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4916
5830
|
lastCompletedFullReconcileWallAt = Date.now();
|
|
4917
5831
|
fullPassJournalHealthy = execution.accepted &&
|
|
4918
5832
|
isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
|
|
5833
|
+
if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
|
|
5834
|
+
clearDurableWatcherOverflow();
|
|
5835
|
+
}
|
|
5836
|
+
else if (fullPassJournalHealthy && pending.batch !== null) {
|
|
5837
|
+
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
5838
|
+
...pending.batch.paths.values(),
|
|
5839
|
+
]);
|
|
5840
|
+
}
|
|
4919
5841
|
if (fullPassJournalHealthy) {
|
|
4920
5842
|
fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
|
|
4921
5843
|
}
|
|
@@ -4973,7 +5895,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4973
5895
|
const catchupBatch = workingBatch;
|
|
4974
5896
|
const catchupGeneration = pending.generation;
|
|
4975
5897
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
4976
|
-
if (exit === 0
|
|
5898
|
+
if (exit === 0) {
|
|
5899
|
+
clearDurableWatcherOverflow();
|
|
5900
|
+
return;
|
|
5901
|
+
}
|
|
5902
|
+
if (exit === SCOPED_DRAIN_DEFERRED)
|
|
4977
5903
|
return;
|
|
4978
5904
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
4979
5905
|
interruptWait();
|
|
@@ -5081,6 +6007,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5081
6007
|
? exitCode === 0
|
|
5082
6008
|
: hasCompletedRunnerPass(result));
|
|
5083
6009
|
if (completedPostRepairPass) {
|
|
6010
|
+
if (!parallelCoordinatorEnabled &&
|
|
6011
|
+
pending.batch?.overflowed === true &&
|
|
6012
|
+
((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
|
|
6013
|
+
clearDurableWatcherOverflow();
|
|
6014
|
+
}
|
|
6015
|
+
else if (!parallelCoordinatorEnabled &&
|
|
6016
|
+
ranFullReconcile &&
|
|
6017
|
+
fullPassJournalHealthy &&
|
|
6018
|
+
pending.batch !== null) {
|
|
6019
|
+
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
6020
|
+
...pending.batch.paths.values(),
|
|
6021
|
+
]);
|
|
6022
|
+
}
|
|
5084
6023
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
5085
6024
|
// which follows journal repair. Its numeric seam has the same
|
|
5086
6025
|
// contract: zero means the injected pass completed. This preserves
|