@indigoai-us/hq-cloud 6.18.51-alpha.0 → 6.18.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +0 -5
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +0 -19
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +5 -85
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +123 -1062
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +11 -30
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +15 -133
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/share.d.ts +0 -12
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +4 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/tombstones.d.ts +31 -3
- package/dist/cli/tombstones.d.ts.map +1 -1
- package/dist/cli/tombstones.js +142 -11
- package/dist/cli/tombstones.js.map +1 -1
- package/dist/ignore.d.ts +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +2 -2
- package/dist/ignore.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +3 -9
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +2 -9
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +2 -34
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/personal-vault-exclusions.d.ts +2 -2
- package/dist/personal-vault-exclusions.d.ts.map +1 -1
- package/dist/personal-vault-exclusions.js +3 -5
- package/dist/personal-vault-exclusions.js.map +1 -1
- package/dist/personal-vault.d.ts +0 -7
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +0 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/symlink-target-policy.d.ts +0 -11
- package/dist/symlink-target-policy.d.ts.map +1 -1
- package/dist/symlink-target-policy.js +0 -80
- package/dist/symlink-target-policy.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +0 -29
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +16 -229
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +10 -32
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/watcher.d.ts +3 -16
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +21 -114
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/dist/bin/watcher-catchup-skip-store.d.ts +0 -60
- package/dist/bin/watcher-catchup-skip-store.d.ts.map +0 -1
- package/dist/bin/watcher-catchup-skip-store.js +0 -282
- package/dist/bin/watcher-catchup-skip-store.js.map +0 -1
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
5
4
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
6
5
|
import * as v8 from "node:v8";
|
|
7
6
|
import { performance } from "node:perf_hooks";
|
|
@@ -16,18 +15,13 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
|
16
15
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
17
16
|
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
18
17
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
19
|
-
import { isEphemeralPath } from "../cli/share.js";
|
|
20
|
-
import { isSafeRelativePath } from "../sync/push-event.js";
|
|
21
|
-
import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
|
|
22
|
-
import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
|
|
23
18
|
import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
|
|
24
19
|
import { createIgnoreFilter } from "../ignore.js";
|
|
25
|
-
import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
26
20
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
27
21
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
28
22
|
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
|
|
29
|
-
import { isPersonalVaultDeletionExcluded
|
|
30
|
-
import { NoopPushReceiver, ReceiverDispatchError,
|
|
23
|
+
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
24
|
+
import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
31
25
|
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
32
26
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
33
27
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
@@ -67,13 +61,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
67
61
|
*/
|
|
68
62
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
69
63
|
/**
|
|
70
|
-
*
|
|
64
|
+
* How many realtime executes may run during one chore hand-off.
|
|
71
65
|
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
72
|
-
* workers. One long
|
|
73
|
-
*
|
|
74
|
-
* work is never aborted
|
|
66
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
67
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
68
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
75
69
|
*/
|
|
76
|
-
export const REALTIME_WORKER_OCCUPANCY_CAP =
|
|
70
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
77
71
|
/** Objects at least one multipart part are sent through the chore hand-off. */
|
|
78
72
|
export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
|
|
79
73
|
let realtimeAdmissionDeadlineMsForTest;
|
|
@@ -97,29 +91,6 @@ function realtimeDrainBurstMs() {
|
|
|
97
91
|
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
98
92
|
*/
|
|
99
93
|
export const REALTIME_ADMISSION_RETRY_MS = 5_000;
|
|
100
|
-
/** Bound consecutive pull admissions while a watcher push is queued. */
|
|
101
|
-
export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
|
|
102
|
-
/** Bound synchronous warm-up candidate work between event-loop turns. */
|
|
103
|
-
export const WATCHER_WARMUP_YIELD_EVERY = 128;
|
|
104
|
-
export function shouldYieldWatcherWarmup(examinedPaths) {
|
|
105
|
-
return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
|
|
106
|
-
}
|
|
107
|
-
export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
|
|
108
|
-
return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
|
|
109
|
-
}
|
|
110
|
-
export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
|
|
111
|
-
// TreeWatcher marks this only when its production filter has already applied
|
|
112
|
-
// the same personal scope predicate to every collected candidate. Unrouted
|
|
113
|
-
// paths retain the runner check because the collector may include company paths.
|
|
114
|
-
return pushFiltersApplied && routeKind === "personal";
|
|
115
|
-
}
|
|
116
|
-
export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
|
|
117
|
-
const ignoreFilter = createIgnore(hqRoot);
|
|
118
|
-
const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
|
|
119
|
-
onPersonalExcluded(relativePath, match.id);
|
|
120
|
-
});
|
|
121
|
-
return { ignoreFilter, personalPushFilter };
|
|
122
|
-
}
|
|
123
94
|
/**
|
|
124
95
|
* In-process result of a scoped drain whose only shortfall was admission
|
|
125
96
|
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
@@ -220,32 +191,6 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
|
|
|
220
191
|
return explicitlyAllowed ||
|
|
221
192
|
(workClass === "realtime" && isFullPassDurableProgressEnabled());
|
|
222
193
|
}
|
|
223
|
-
export function resolveWatcherPushWaitStage(input) {
|
|
224
|
-
if (input.capYieldActive)
|
|
225
|
-
return "CPU/heap cap wait";
|
|
226
|
-
if (input.holders.some((holder) => holder.kind === "pull"))
|
|
227
|
-
return "pull-yield wait";
|
|
228
|
-
if (input.queueDepth > 0)
|
|
229
|
-
return "queued behind realtime work";
|
|
230
|
-
if (input.holders.some((holder) => holder.kind === "chore"))
|
|
231
|
-
return "chore holder wait";
|
|
232
|
-
if (input.holders.length > 0)
|
|
233
|
-
return "scheduler holder wait";
|
|
234
|
-
return "scheduler admission pending";
|
|
235
|
-
}
|
|
236
|
-
export function watcherPushWaitDiagnostics(input) {
|
|
237
|
-
return {
|
|
238
|
-
stage: input.admissionWaiting
|
|
239
|
-
? resolveWatcherPushWaitStage(input)
|
|
240
|
-
: input.stage,
|
|
241
|
-
waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
|
|
242
|
-
? null
|
|
243
|
-
: Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
|
|
244
|
-
schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
|
|
245
|
-
? null
|
|
246
|
-
: Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
194
|
/**
|
|
250
195
|
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
251
196
|
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
@@ -260,8 +205,6 @@ export class CooperativePassScheduler {
|
|
|
260
205
|
starvationCapMs;
|
|
261
206
|
options;
|
|
262
207
|
pending = [];
|
|
263
|
-
lastAnyAdmissionAtMonotonicMs = null;
|
|
264
|
-
nextEntryId = 0;
|
|
265
208
|
active = null;
|
|
266
209
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
267
210
|
realtimeOccupants = new Set();
|
|
@@ -298,58 +241,12 @@ export class CooperativePassScheduler {
|
|
|
298
241
|
/** Successful callers still applying results after the network hand-off. */
|
|
299
242
|
networkTransferCompletionCount = 0;
|
|
300
243
|
networkTransferApplyRelease = null;
|
|
301
|
-
consecutivePullAdmissions = 0;
|
|
302
244
|
resolveNetworkTransferApplyRelease = null;
|
|
303
245
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
304
246
|
this.now = now;
|
|
305
247
|
this.starvationCapMs = starvationCapMs;
|
|
306
248
|
this.options = options;
|
|
307
249
|
}
|
|
308
|
-
recordAdmission(entry) {
|
|
309
|
-
const monotonicMs = performance.now();
|
|
310
|
-
const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
|
|
311
|
-
? null
|
|
312
|
-
: Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
|
|
313
|
-
const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
314
|
-
const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
|
|
315
|
-
const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
|
|
316
|
-
const waitingSinceAdmissionRequestMs = oldestPush
|
|
317
|
-
? Math.max(0, this.now() - oldestPush.enqueuedAt)
|
|
318
|
-
: null;
|
|
319
|
-
if (waitingPush || admittedIsPush) {
|
|
320
|
-
this.options.onAdmission?.({
|
|
321
|
-
entryId: entry.entryId,
|
|
322
|
-
workClass: entry.workClass,
|
|
323
|
-
lane: entry.lane,
|
|
324
|
-
transferDirection: entry.transferDirection,
|
|
325
|
-
transferSizeClass: entry.transferSizeClass ?? "unknown",
|
|
326
|
-
label: entry.label.slice(0, 160),
|
|
327
|
-
admittedAt: new Date().toISOString(),
|
|
328
|
-
schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
|
|
329
|
-
realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
|
|
330
|
-
oldestWaitingWatcherPush: oldestPush
|
|
331
|
-
? {
|
|
332
|
-
entryId: oldestPush.entryId,
|
|
333
|
-
label: oldestPush.label.slice(0, 160),
|
|
334
|
-
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
335
|
-
}
|
|
336
|
-
: null,
|
|
337
|
-
passedOver: waitingPush && waitingPush !== entry
|
|
338
|
-
? {
|
|
339
|
-
entryId: waitingPush.entryId,
|
|
340
|
-
label: waitingPush.label.slice(0, 160),
|
|
341
|
-
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
342
|
-
reason: entry.transferDirection === "pull"
|
|
343
|
-
? "pull-admitted-ahead-of-waiting-push"
|
|
344
|
-
: this._handoffActive
|
|
345
|
-
? "handoff-selector-admitted-other-work"
|
|
346
|
-
: "scheduler-admitted-other-work",
|
|
347
|
-
}
|
|
348
|
-
: null,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
|
|
352
|
-
}
|
|
353
250
|
get handoffActive() {
|
|
354
251
|
return this._handoffActive;
|
|
355
252
|
}
|
|
@@ -366,40 +263,12 @@ export class CooperativePassScheduler {
|
|
|
366
263
|
get hasActiveSlowRealtime() {
|
|
367
264
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
368
265
|
}
|
|
369
|
-
get isInCapYield() {
|
|
370
|
-
return this.capYieldDepth > 0;
|
|
371
|
-
}
|
|
372
266
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
373
267
|
get activeRealtimeWorkerCount() {
|
|
374
268
|
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
375
269
|
(this.activeFastRealtime === null ? 0 : 1) +
|
|
376
270
|
this.realtimeOccupants.size);
|
|
377
271
|
}
|
|
378
|
-
get diagnosticAdmissionSnapshot() {
|
|
379
|
-
const active = [
|
|
380
|
-
this.active,
|
|
381
|
-
this.activeFastRealtime,
|
|
382
|
-
...this.realtimeOccupants,
|
|
383
|
-
this.detachedChore,
|
|
384
|
-
].filter((entry) => entry !== null);
|
|
385
|
-
const holders = active.map((entry) => ({
|
|
386
|
-
entryId: entry.entryId,
|
|
387
|
-
kind: entry.transferDirection === "pull"
|
|
388
|
-
? "pull"
|
|
389
|
-
: entry.workClass === "chore"
|
|
390
|
-
? "chore"
|
|
391
|
-
: "push",
|
|
392
|
-
scope: entry.label.slice(0, 160),
|
|
393
|
-
ageMs: entry.startedAtMonotonicMs === undefined
|
|
394
|
-
? 0
|
|
395
|
-
: Math.max(0, performance.now() - entry.startedAtMonotonicMs),
|
|
396
|
-
}));
|
|
397
|
-
return {
|
|
398
|
-
queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
|
|
399
|
-
holders: holders.slice(0, 20),
|
|
400
|
-
lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
|
|
401
|
-
};
|
|
402
|
-
}
|
|
403
272
|
pendingLaneCount(lane) {
|
|
404
273
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
405
274
|
}
|
|
@@ -414,7 +283,6 @@ export class CooperativePassScheduler {
|
|
|
414
283
|
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
415
284
|
}
|
|
416
285
|
const directEntry = {
|
|
417
|
-
entryId: `guarded-${++this.nextEntryId}`,
|
|
418
286
|
task,
|
|
419
287
|
workClass,
|
|
420
288
|
label,
|
|
@@ -438,8 +306,7 @@ export class CooperativePassScheduler {
|
|
|
438
306
|
if (workClass === "realtime" &&
|
|
439
307
|
this._handoffActive &&
|
|
440
308
|
directEntry.allowConcurrentHandoff &&
|
|
441
|
-
this.canAdmitRealtimeOccupant(directEntry)
|
|
442
|
-
!this.pullMustYieldForWaitingPush(directEntry)) {
|
|
309
|
+
this.canAdmitRealtimeOccupant(directEntry)) {
|
|
443
310
|
return new Promise((resolve, reject) => {
|
|
444
311
|
const entry = {
|
|
445
312
|
...directEntry,
|
|
@@ -588,21 +455,16 @@ export class CooperativePassScheduler {
|
|
|
588
455
|
this.admitHandoffBacklog();
|
|
589
456
|
return;
|
|
590
457
|
}
|
|
591
|
-
if (this.active !== null
|
|
458
|
+
if (this.active !== null) {
|
|
592
459
|
const fastRealtime = this.nextFastRealtime();
|
|
593
|
-
if (
|
|
460
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
594
461
|
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
595
462
|
this.startFastRealtime(fastRealtime);
|
|
596
|
-
return;
|
|
597
|
-
}
|
|
598
|
-
const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
|
|
599
|
-
if (peerIndex >= 0) {
|
|
600
|
-
const peer = this.pending.splice(peerIndex, 1)[0];
|
|
601
|
-
if (peer)
|
|
602
|
-
this.startRealtimeOccupant(peer);
|
|
603
463
|
}
|
|
604
464
|
return;
|
|
605
465
|
}
|
|
466
|
+
if (this.activeFastRealtime !== null)
|
|
467
|
+
return;
|
|
606
468
|
const next = this.takeNext();
|
|
607
469
|
if (next)
|
|
608
470
|
this.start(next);
|
|
@@ -621,29 +483,11 @@ export class CooperativePassScheduler {
|
|
|
621
483
|
return this.pending.splice(realtime, 1)[0];
|
|
622
484
|
return this.pending.shift();
|
|
623
485
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
628
|
-
}
|
|
629
|
-
nextRealtimeIndex(eligible = () => true) {
|
|
630
|
-
const waitingPush = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.transferDirection === "push" && eligible(entry));
|
|
631
|
-
if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
|
|
632
|
-
this.consecutivePullAdmissions = 0;
|
|
633
|
-
return waitingPush;
|
|
634
|
-
}
|
|
635
|
-
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow" && eligible(entry));
|
|
636
|
-
const next = slowRealtime >= 0
|
|
486
|
+
nextRealtimeIndex() {
|
|
487
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
488
|
+
return slowRealtime >= 0
|
|
637
489
|
? slowRealtime
|
|
638
|
-
: this.pending.findIndex((entry) => entry.workClass === "realtime"
|
|
639
|
-
if (next >= 0) {
|
|
640
|
-
const selected = this.pending[next];
|
|
641
|
-
if (selected?.transferDirection === "pull")
|
|
642
|
-
this.consecutivePullAdmissions += 1;
|
|
643
|
-
else if (selected?.transferDirection === "push")
|
|
644
|
-
this.consecutivePullAdmissions = 0;
|
|
645
|
-
}
|
|
646
|
-
return next;
|
|
490
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
647
491
|
}
|
|
648
492
|
nextFastRealtime() {
|
|
649
493
|
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
@@ -661,20 +505,8 @@ export class CooperativePassScheduler {
|
|
|
661
505
|
this.hasActiveSlowRealtime &&
|
|
662
506
|
!this.hasStarvedChore());
|
|
663
507
|
}
|
|
664
|
-
/** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
|
|
665
|
-
canAdmitRealtimePeer(entry) {
|
|
666
|
-
if (entry.workClass !== "realtime" ||
|
|
667
|
-
!entry.allowConcurrentHandoff ||
|
|
668
|
-
entry.transferDirection === undefined ||
|
|
669
|
-
this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
670
|
-
return false;
|
|
671
|
-
return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
|
|
672
|
-
active.transferDirection !== undefined &&
|
|
673
|
-
active.transferDirection !== entry.transferDirection);
|
|
674
|
-
}
|
|
675
508
|
start(entry) {
|
|
676
509
|
this.clearAdmission(entry);
|
|
677
|
-
this.recordAdmission(entry);
|
|
678
510
|
entry.startedAt = new Date().toISOString();
|
|
679
511
|
// Holder duration uses the process monotonic clock; do not consume the
|
|
680
512
|
// injected scheduler clock, whose reads define cadence-slot deadlines.
|
|
@@ -713,8 +545,6 @@ export class CooperativePassScheduler {
|
|
|
713
545
|
}
|
|
714
546
|
startFastRealtime(entry) {
|
|
715
547
|
this.clearAdmission(entry);
|
|
716
|
-
this.recordAdmission(entry);
|
|
717
|
-
entry.startedAtMonotonicMs = performance.now();
|
|
718
548
|
this.activeFastRealtime = entry;
|
|
719
549
|
let current;
|
|
720
550
|
try {
|
|
@@ -970,8 +800,6 @@ export class CooperativePassScheduler {
|
|
|
970
800
|
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
971
801
|
startRealtimeOccupant(entry) {
|
|
972
802
|
this.clearAdmission(entry);
|
|
973
|
-
this.recordAdmission(entry);
|
|
974
|
-
entry.startedAtMonotonicMs = performance.now();
|
|
975
803
|
this.realtimeOccupants.add(entry);
|
|
976
804
|
this.handoffAdmitted = true;
|
|
977
805
|
if (this.capYieldDepth > 0)
|
|
@@ -986,7 +814,6 @@ export class CooperativePassScheduler {
|
|
|
986
814
|
this.handoffBacklog = null;
|
|
987
815
|
entry.reject(error);
|
|
988
816
|
this.notifyOccupantChange();
|
|
989
|
-
this.drain();
|
|
990
817
|
return;
|
|
991
818
|
}
|
|
992
819
|
void running.then((outcome) => {
|
|
@@ -996,7 +823,6 @@ export class CooperativePassScheduler {
|
|
|
996
823
|
entry.resolve(outcome);
|
|
997
824
|
this.promoteOccupancyArrivals();
|
|
998
825
|
this.notifyOccupantChange();
|
|
999
|
-
this.drain();
|
|
1000
826
|
}, (error) => {
|
|
1001
827
|
this.realtimeOccupants.delete(entry);
|
|
1002
828
|
if (this.handoffBacklog === entry)
|
|
@@ -1004,7 +830,6 @@ export class CooperativePassScheduler {
|
|
|
1004
830
|
entry.reject(error);
|
|
1005
831
|
this.promoteOccupancyArrivals();
|
|
1006
832
|
this.notifyOccupantChange();
|
|
1007
|
-
this.drain();
|
|
1008
833
|
});
|
|
1009
834
|
}
|
|
1010
835
|
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
@@ -1013,7 +838,8 @@ export class CooperativePassScheduler {
|
|
|
1013
838
|
(!this.networkTransferActive && !this.withinHandoffAdmission()))
|
|
1014
839
|
return;
|
|
1015
840
|
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
1016
|
-
const index = this.
|
|
841
|
+
const index = this.pending.findIndex((pending) => pending.workClass === "realtime" &&
|
|
842
|
+
pending.occupancyArrival === true &&
|
|
1017
843
|
pending.allowConcurrentHandoff &&
|
|
1018
844
|
this.canAdmitRealtimeOccupant(pending));
|
|
1019
845
|
if (index < 0)
|
|
@@ -1032,7 +858,7 @@ export class CooperativePassScheduler {
|
|
|
1032
858
|
return;
|
|
1033
859
|
if (!this.withinHandoffAdmission())
|
|
1034
860
|
return;
|
|
1035
|
-
const realtimeIndex = this.nextRealtimeIndex(
|
|
861
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
1036
862
|
if (realtimeIndex < 0)
|
|
1037
863
|
return;
|
|
1038
864
|
const pending = this.pending[realtimeIndex];
|
|
@@ -1466,151 +1292,6 @@ export function isChoreLocalPush(paths, largeUpload) {
|
|
|
1466
1292
|
}
|
|
1467
1293
|
return count > 0;
|
|
1468
1294
|
}
|
|
1469
|
-
/** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
|
|
1470
|
-
export function watcherWarmupPathsByRoute(relativePaths) {
|
|
1471
|
-
const counts = new Map();
|
|
1472
|
-
for (const relativePath of relativePaths) {
|
|
1473
|
-
const route = routeChangeToTarget(relativePath);
|
|
1474
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1475
|
-
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1476
|
-
}
|
|
1477
|
-
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1478
|
-
const result = {};
|
|
1479
|
-
for (const [key, count] of ranked.slice(0, 20))
|
|
1480
|
-
result[key] = count;
|
|
1481
|
-
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1482
|
-
if (other > 0)
|
|
1483
|
-
result.other = other;
|
|
1484
|
-
return result;
|
|
1485
|
-
}
|
|
1486
|
-
export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
|
|
1487
|
-
const candidatePrefixes = new Map();
|
|
1488
|
-
const candidateCounts = new Map();
|
|
1489
|
-
for (const relativePath of candidateRelativePaths) {
|
|
1490
|
-
const route = routeChangeToTarget(relativePath);
|
|
1491
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1492
|
-
candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
|
|
1493
|
-
const routePath = route?.kind === "company"
|
|
1494
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1495
|
-
: relativePath;
|
|
1496
|
-
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1497
|
-
const routeCounts = candidatePrefixes.get(key) ?? new Map();
|
|
1498
|
-
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1499
|
-
candidatePrefixes.set(key, routeCounts);
|
|
1500
|
-
}
|
|
1501
|
-
const eligibleCounts = new Map();
|
|
1502
|
-
for (const relativePath of eligibleRelativePaths) {
|
|
1503
|
-
const route = routeChangeToTarget(relativePath);
|
|
1504
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1505
|
-
const routePath = route?.kind === "company"
|
|
1506
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1507
|
-
: relativePath;
|
|
1508
|
-
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1509
|
-
const routeCounts = eligibleCounts.get(key) ?? new Map();
|
|
1510
|
-
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1511
|
-
eligibleCounts.set(key, routeCounts);
|
|
1512
|
-
}
|
|
1513
|
-
const result = {};
|
|
1514
|
-
// Eligible routes are all retained; fill any remaining diagnostic slots
|
|
1515
|
-
// with the largest candidate routes. This keeps alphabetical one-path
|
|
1516
|
-
// stubs from crowding out a large route such as personal.
|
|
1517
|
-
const eligibleRoutes = [...eligibleCounts.keys()];
|
|
1518
|
-
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1519
|
-
const rankedCandidates = [...candidateCounts.entries()]
|
|
1520
|
-
.filter(([route]) => !eligibleCounts.has(route))
|
|
1521
|
-
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1522
|
-
const selectedRoutes = [
|
|
1523
|
-
...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
|
|
1524
|
-
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1525
|
-
];
|
|
1526
|
-
for (const route of selectedRoutes) {
|
|
1527
|
-
const topPrefixes = (counts) => {
|
|
1528
|
-
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1529
|
-
const top = {};
|
|
1530
|
-
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1531
|
-
top[prefix] = count;
|
|
1532
|
-
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1533
|
-
if (other > 0)
|
|
1534
|
-
top.other = other;
|
|
1535
|
-
return top;
|
|
1536
|
-
};
|
|
1537
|
-
result[route] = {
|
|
1538
|
-
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1539
|
-
eligible: topPrefixes(eligibleCounts.get(route)),
|
|
1540
|
-
};
|
|
1541
|
-
}
|
|
1542
|
-
const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
|
|
1543
|
-
if (otherRoutes.length > 0) {
|
|
1544
|
-
const merged = new Map();
|
|
1545
|
-
for (const [, routeCounts] of otherRoutes) {
|
|
1546
|
-
for (const [prefix, count] of routeCounts)
|
|
1547
|
-
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1548
|
-
}
|
|
1549
|
-
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1550
|
-
}
|
|
1551
|
-
return result;
|
|
1552
|
-
}
|
|
1553
|
-
function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
|
|
1554
|
-
counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
|
|
1555
|
-
const firstSlash = routePath.indexOf("/");
|
|
1556
|
-
const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
|
|
1557
|
-
const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
|
|
1558
|
-
let byPrefix = prefixes.get(routeName);
|
|
1559
|
-
if (!byPrefix) {
|
|
1560
|
-
byPrefix = new Map();
|
|
1561
|
-
prefixes.set(routeName, byPrefix);
|
|
1562
|
-
}
|
|
1563
|
-
byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
|
|
1564
|
-
}
|
|
1565
|
-
function rankedWarmupCounts(counts) {
|
|
1566
|
-
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1567
|
-
const result = {};
|
|
1568
|
-
for (const [key, count] of ranked.slice(0, 20))
|
|
1569
|
-
result[key] = count;
|
|
1570
|
-
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1571
|
-
if (other > 0)
|
|
1572
|
-
result.other = other;
|
|
1573
|
-
return result;
|
|
1574
|
-
}
|
|
1575
|
-
function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
|
|
1576
|
-
const topPrefixes = (counts) => {
|
|
1577
|
-
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1578
|
-
const top = {};
|
|
1579
|
-
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1580
|
-
top[prefix] = count;
|
|
1581
|
-
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1582
|
-
if (other > 0)
|
|
1583
|
-
top.other = other;
|
|
1584
|
-
return top;
|
|
1585
|
-
};
|
|
1586
|
-
const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
|
|
1587
|
-
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1588
|
-
const rankedCandidates = [...candidateCounts.entries()]
|
|
1589
|
-
.filter(([route]) => !eligibleCounts.has(route))
|
|
1590
|
-
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1591
|
-
const selectedRoutes = [
|
|
1592
|
-
...eligibleRoutes,
|
|
1593
|
-
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1594
|
-
];
|
|
1595
|
-
const result = {};
|
|
1596
|
-
for (const route of selectedRoutes) {
|
|
1597
|
-
result[route] = {
|
|
1598
|
-
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1599
|
-
eligible: topPrefixes(eligiblePrefixes.get(route)),
|
|
1600
|
-
};
|
|
1601
|
-
}
|
|
1602
|
-
const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
|
|
1603
|
-
if (otherRoutes.length > 0) {
|
|
1604
|
-
const merged = new Map();
|
|
1605
|
-
for (const route of otherRoutes) {
|
|
1606
|
-
for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
|
|
1607
|
-
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1608
|
-
}
|
|
1609
|
-
}
|
|
1610
|
-
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1611
|
-
}
|
|
1612
|
-
return result;
|
|
1613
|
-
}
|
|
1614
1295
|
/** A receiver event needs a known small upsert size before it can go realtime. */
|
|
1615
1296
|
export function isFastReceiverBatch(eventCount) {
|
|
1616
1297
|
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
@@ -2082,10 +1763,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2082
1763
|
let recycleRequested = null;
|
|
2083
1764
|
const pendingAppliedUpserts = new Map();
|
|
2084
1765
|
const pendingAppliedDeletes = new Set();
|
|
2085
|
-
const uploadedAwaitingPublish = new Set();
|
|
2086
|
-
const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
|
|
2087
|
-
const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
|
|
2088
|
-
let uploadedAwaitingPublishDropLogged = false;
|
|
2089
1766
|
const rememberAppliedUpsert = (relativePath, contentHash) => {
|
|
2090
1767
|
pendingAppliedDeletes.delete(relativePath);
|
|
2091
1768
|
if (eventSyncLive && eventSync) {
|
|
@@ -2116,23 +1793,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2116
1793
|
pendingAppliedDeletes.delete(oldest);
|
|
2117
1794
|
}
|
|
2118
1795
|
};
|
|
2119
|
-
const flushPendingUploadEvents = () => {
|
|
2120
|
-
if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
|
|
2121
|
-
return;
|
|
2122
|
-
while (uploadedAwaitingPublish.size > 0) {
|
|
2123
|
-
const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
|
|
2124
|
-
const paths = new Map(uploadedPaths.map((relativePath) => [
|
|
2125
|
-
localPathForVaultKey(hqRoot, relativePath),
|
|
2126
|
-
relativePath,
|
|
2127
|
-
]));
|
|
2128
|
-
const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
|
|
2129
|
-
if (expanded.size > 0)
|
|
2130
|
-
eventSync.publishBatch({ paths: expanded });
|
|
2131
|
-
for (const relativePath of uploadedPaths) {
|
|
2132
|
-
uploadedAwaitingPublish.delete(relativePath);
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2135
|
-
};
|
|
2136
1796
|
const flushPendingAppliedUpserts = () => {
|
|
2137
1797
|
if (!eventSyncLive || !eventSync)
|
|
2138
1798
|
return;
|
|
@@ -2144,7 +1804,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2144
1804
|
eventSync.markContentDeleted?.(relativePath);
|
|
2145
1805
|
}
|
|
2146
1806
|
pendingAppliedDeletes.clear();
|
|
2147
|
-
flushPendingUploadEvents();
|
|
2148
1807
|
};
|
|
2149
1808
|
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
2150
1809
|
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
@@ -2194,17 +1853,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2194
1853
|
}, 0);
|
|
2195
1854
|
refresh.unref?.();
|
|
2196
1855
|
}
|
|
2197
|
-
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
2198
|
-
|
|
2199
|
-
onAdmission: (event) => {
|
|
2200
|
-
process.stderr.write(`${JSON.stringify({
|
|
2201
|
-
type: "watcher-push-scheduler-admission",
|
|
2202
|
-
diagnostic: true,
|
|
2203
|
-
...event,
|
|
2204
|
-
})}\n`);
|
|
2205
|
-
},
|
|
2206
|
-
});
|
|
2207
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
|
|
1856
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
1857
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
|
|
2208
1858
|
const passCheckpoint = pullSignal || receiverSignal
|
|
2209
1859
|
? Object.assign(async () => checkpoint?.(), {
|
|
2210
1860
|
shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
|
|
@@ -2234,43 +1884,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2234
1884
|
}
|
|
2235
1885
|
};
|
|
2236
1886
|
const invokePass = () => {
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
const complete = () => {
|
|
2242
|
-
watcherPushStage?.("pass return");
|
|
2243
|
-
activePassCount = Math.max(0, activePassCount - 1);
|
|
2244
|
-
emitSyncWorkState(true);
|
|
2245
|
-
};
|
|
2246
|
-
// Observe settlement without returning a chained promise to the pass
|
|
2247
|
-
// scheduler. The scheduler's synchronous test seam and watcher wake
|
|
2248
|
-
// ordering depend on the original pass promise's completion turn.
|
|
2249
|
-
void pass.then(complete, complete);
|
|
2250
|
-
return pass;
|
|
2251
|
-
};
|
|
2252
|
-
try {
|
|
2253
|
-
if (!runPassSupportsCooperativeCheckpoint) {
|
|
2254
|
-
if (!runPassSupportsQmdLifecycle)
|
|
2255
|
-
return trackPass(runPass(passArgvForRun));
|
|
2256
|
-
return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
|
|
2257
|
-
}
|
|
2258
|
-
// This callback is emitted by the production runner after journal
|
|
2259
|
-
// repair/seed and before fanout. It starts ingress early, but never
|
|
2260
|
-
// admits a second durable writer: queued realtime still waits for the
|
|
2261
|
-
// existing scheduler's cooperative hand-off checkpoint.
|
|
2262
|
-
const onJournalsReady = () => activateEventPushSurfaces();
|
|
2263
|
-
if (!runPassSupportsQmdLifecycle) {
|
|
2264
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
|
|
2265
|
-
}
|
|
2266
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
|
|
1887
|
+
if (!runPassSupportsCooperativeCheckpoint) {
|
|
1888
|
+
if (!runPassSupportsQmdLifecycle)
|
|
1889
|
+
return runPass(passArgvForRun);
|
|
1890
|
+
return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
|
|
2267
1891
|
}
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
1892
|
+
// This callback is emitted by the production runner after journal
|
|
1893
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
1894
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
1895
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
1896
|
+
const onJournalsReady = () => activateEventPushSurfaces();
|
|
1897
|
+
if (!runPassSupportsQmdLifecycle) {
|
|
1898
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
|
|
2273
1899
|
}
|
|
1900
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
|
|
2274
1901
|
};
|
|
2275
1902
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
2276
1903
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -2285,11 +1912,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2285
1912
|
return;
|
|
2286
1913
|
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
2287
1914
|
recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
|
|
2288
|
-
rememberUploadedPaths(outcome);
|
|
2289
|
-
if (typeof outcome !== "number") {
|
|
2290
|
-
acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
|
|
2291
|
-
}
|
|
2292
|
-
flushPendingUploadEvents();
|
|
2293
1915
|
};
|
|
2294
1916
|
// Pre-admit boundary observation.
|
|
2295
1917
|
heapCheckpoint?.();
|
|
@@ -2540,25 +2162,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2540
2162
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2541
2163
|
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
|
|
2542
2164
|
};
|
|
2543
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass
|
|
2544
|
-
if (watcherPushStage) {
|
|
2545
|
-
watcherPushStage("guarded scheduler admission requested");
|
|
2546
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2547
|
-
if (passScheduler.isInCapYield)
|
|
2548
|
-
watcherPushStage("CPU/heap cap wait");
|
|
2549
|
-
else if (admission.holders.some((holder) => holder.kind === "pull")) {
|
|
2550
|
-
watcherPushStage("pull-yield wait");
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2165
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
|
|
2553
2166
|
telemetry?.mark("operationGate");
|
|
2554
2167
|
const admissionSignal = signal && receiverSignal
|
|
2555
2168
|
? AbortSignal.any([signal, receiverSignal])
|
|
2556
2169
|
: signal ?? receiverSignal;
|
|
2557
2170
|
return runGuardedTask((context) => {
|
|
2558
|
-
watcherPushStage?.("guarded scheduler admission granted");
|
|
2559
2171
|
telemetry?.mark("appendLock");
|
|
2560
2172
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2561
|
-
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal
|
|
2173
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2562
2174
|
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
|
|
2563
2175
|
};
|
|
2564
2176
|
/*
|
|
@@ -2680,7 +2292,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2680
2292
|
return {
|
|
2681
2293
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2682
2294
|
accepted,
|
|
2683
|
-
yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
|
|
2684
2295
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2685
2296
|
.filter((path) => path.status === "refused")
|
|
2686
2297
|
.map((path) => path.path)),
|
|
@@ -2878,222 +2489,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2878
2489
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
2879
2490
|
let drainedUntrackedDeletes = 0;
|
|
2880
2491
|
const pendingWatcherPaths = new Map();
|
|
2881
|
-
const durableWatcherPaths = new Map();
|
|
2882
|
-
const durableWatcherPathLimit = 10_000;
|
|
2883
|
-
let durableWatcherOverflowed = false;
|
|
2884
|
-
let durableWatcherPersistTimer;
|
|
2885
|
-
let durableWatcherPersistQueue = Promise.resolve();
|
|
2886
|
-
let durableWatcherPersistGeneration = 0;
|
|
2887
|
-
let durableWatcherPersistedGeneration = 0;
|
|
2888
|
-
const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
|
|
2889
|
-
const persistDurableWatcherPaths = () => {
|
|
2890
|
-
const generation = durableWatcherPersistGeneration;
|
|
2891
|
-
if (generation === durableWatcherPersistedGeneration)
|
|
2892
|
-
return durableWatcherPersistQueue;
|
|
2893
|
-
const snapshot = durableWatcherOverflowed
|
|
2894
|
-
? { format: 1, hqRoot, overflowed: true }
|
|
2895
|
-
: { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
|
|
2896
|
-
durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
|
|
2897
|
-
try {
|
|
2898
|
-
if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
|
|
2899
|
-
await fs.promises.rm(watcherPushStatePath, { force: true });
|
|
2900
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2901
|
-
return;
|
|
2902
|
-
}
|
|
2903
|
-
await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
|
|
2904
|
-
const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
|
|
2905
|
-
await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
|
|
2906
|
-
await fs.promises.rename(tempPath, watcherPushStatePath);
|
|
2907
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2908
|
-
}
|
|
2909
|
-
catch (err) {
|
|
2910
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2911
|
-
}
|
|
2912
|
-
}).catch((err) => {
|
|
2913
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2914
|
-
});
|
|
2915
|
-
return durableWatcherPersistQueue;
|
|
2916
|
-
};
|
|
2917
|
-
const scheduleDurableWatcherPersist = () => {
|
|
2918
|
-
durableWatcherPersistGeneration += 1;
|
|
2919
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
2920
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
2921
|
-
durableWatcherPersistTimer = setTimeout(() => {
|
|
2922
|
-
durableWatcherPersistTimer = undefined;
|
|
2923
|
-
void persistDurableWatcherPaths();
|
|
2924
|
-
}, 1_000);
|
|
2925
|
-
durableWatcherPersistTimer.unref?.();
|
|
2926
|
-
};
|
|
2927
|
-
const addDurableWatcherPath = (absolutePath, relativePath) => {
|
|
2928
|
-
if (durableWatcherOverflowed)
|
|
2929
|
-
return;
|
|
2930
|
-
if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
|
|
2931
|
-
durableWatcherPaths.clear();
|
|
2932
|
-
durableWatcherOverflowed = true;
|
|
2933
|
-
}
|
|
2934
|
-
else if (!durableWatcherOverflowed) {
|
|
2935
|
-
durableWatcherPaths.set(absolutePath, relativePath);
|
|
2936
|
-
}
|
|
2937
|
-
scheduleDurableWatcherPersist();
|
|
2938
|
-
};
|
|
2939
|
-
const clearDurableWatcherOverflow = () => {
|
|
2940
|
-
if (!durableWatcherOverflowed)
|
|
2941
|
-
return;
|
|
2942
|
-
durableWatcherOverflowed = false;
|
|
2943
|
-
durableWatcherPaths.clear();
|
|
2944
|
-
for (const [absolutePath, relativePath] of pendingWatcherPaths) {
|
|
2945
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
2946
|
-
}
|
|
2947
|
-
scheduleDurableWatcherPersist();
|
|
2948
|
-
};
|
|
2949
|
-
const acknowledgeDurableWatcherUploads = (relativePaths) => {
|
|
2950
|
-
let changed = false;
|
|
2951
|
-
const uploaded = new Set(relativePaths);
|
|
2952
|
-
for (const [absolutePath, relativePath] of durableWatcherPaths) {
|
|
2953
|
-
if (!uploaded.has(relativePath))
|
|
2954
|
-
continue;
|
|
2955
|
-
durableWatcherPaths.delete(absolutePath);
|
|
2956
|
-
changed = true;
|
|
2957
|
-
}
|
|
2958
|
-
if (changed)
|
|
2959
|
-
scheduleDurableWatcherPersist();
|
|
2960
|
-
};
|
|
2961
|
-
const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
|
|
2962
|
-
const stillPending = new Set(pendingWatcherPaths.values());
|
|
2963
|
-
acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
|
|
2964
|
-
};
|
|
2965
|
-
let activePassCount = 0;
|
|
2966
|
-
let inFlightWatcherPushes = 0;
|
|
2967
|
-
const watcherPushStages = new Map();
|
|
2968
|
-
let nextWatcherPushGroupId = 0;
|
|
2969
|
-
const setWatcherPushStage = (groupId, stage) => {
|
|
2970
|
-
const record = watcherPushStages.get(groupId);
|
|
2971
|
-
if (!record || record.stage === stage)
|
|
2972
|
-
return;
|
|
2973
|
-
record.stage = stage;
|
|
2974
|
-
record.stageEnteredAt = Date.now();
|
|
2975
|
-
if (stage === "guarded scheduler admission requested") {
|
|
2976
|
-
record.admissionRequestedAtMonotonicMs = performance.now();
|
|
2977
|
-
record.admissionWaiting = true;
|
|
2978
|
-
}
|
|
2979
|
-
else if (stage === "guarded scheduler admission granted") {
|
|
2980
|
-
record.admissionWaiting = false;
|
|
2981
|
-
}
|
|
2982
|
-
process.stderr.write(`${JSON.stringify({
|
|
2983
|
-
type: "watcher-push-group-stage",
|
|
2984
|
-
diagnostic: true,
|
|
2985
|
-
groupId: record.groupId,
|
|
2986
|
-
route: record.route,
|
|
2987
|
-
paths: record.paths,
|
|
2988
|
-
stage: record.stage,
|
|
2989
|
-
stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
|
|
2990
|
-
})}\n`);
|
|
2991
|
-
};
|
|
2992
|
-
const startWatcherPushStageTracking = (route, paths) => {
|
|
2993
|
-
const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
|
|
2994
|
-
watcherPushStages.set(groupId, {
|
|
2995
|
-
groupId,
|
|
2996
|
-
route,
|
|
2997
|
-
paths,
|
|
2998
|
-
stage: "",
|
|
2999
|
-
stageEnteredAt: Date.now(),
|
|
3000
|
-
admissionRequestedAtMonotonicMs: null,
|
|
3001
|
-
admissionWaiting: false,
|
|
3002
|
-
});
|
|
3003
|
-
return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
|
|
3004
|
-
};
|
|
3005
|
-
const emitWatcherPushInflightStages = () => {
|
|
3006
|
-
if (inFlightWatcherPushes <= 0)
|
|
3007
|
-
return;
|
|
3008
|
-
const now = Date.now();
|
|
3009
|
-
const entries = [...watcherPushStages.values()];
|
|
3010
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
3011
|
-
const coordinatorAdmission = coordinator?.inspect();
|
|
3012
|
-
for (const record of entries) {
|
|
3013
|
-
if (!record.admissionWaiting)
|
|
3014
|
-
continue;
|
|
3015
|
-
const liveStage = resolveWatcherPushWaitStage({
|
|
3016
|
-
capYieldActive: passScheduler.isInCapYield,
|
|
3017
|
-
queueDepth: admission.queueDepth,
|
|
3018
|
-
holders: admission.holders,
|
|
3019
|
-
});
|
|
3020
|
-
if (liveStage !== record.stage)
|
|
3021
|
-
setWatcherPushStage(record.groupId, liveStage);
|
|
3022
|
-
}
|
|
3023
|
-
process.stderr.write(`${JSON.stringify({
|
|
3024
|
-
type: "watcher-push-inflight-stages",
|
|
3025
|
-
diagnostic: true,
|
|
3026
|
-
groups: entries.slice(0, 20).map((record) => {
|
|
3027
|
-
const monotonicNow = performance.now();
|
|
3028
|
-
const current = watcherPushWaitDiagnostics({
|
|
3029
|
-
stage: record.stage,
|
|
3030
|
-
admissionWaiting: record.admissionWaiting,
|
|
3031
|
-
admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
|
|
3032
|
-
lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
|
|
3033
|
-
nowMonotonicMs: monotonicNow,
|
|
3034
|
-
capYieldActive: passScheduler.isInCapYield,
|
|
3035
|
-
queueDepth: admission.queueDepth,
|
|
3036
|
-
holders: admission.holders,
|
|
3037
|
-
});
|
|
3038
|
-
return {
|
|
3039
|
-
route: record.route,
|
|
3040
|
-
paths: record.paths,
|
|
3041
|
-
...current,
|
|
3042
|
-
ageMs: Math.max(0, now - record.stageEnteredAt),
|
|
3043
|
-
};
|
|
3044
|
-
}),
|
|
3045
|
-
other: Math.max(0, entries.length - 20),
|
|
3046
|
-
realtimeAdmissionQueueDepth: admission.queueDepth,
|
|
3047
|
-
holders: admission.holders,
|
|
3048
|
-
coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
|
|
3049
|
-
coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
|
|
3050
|
-
coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
|
|
3051
|
-
fast: null,
|
|
3052
|
-
slow: null,
|
|
3053
|
-
overflowPush: null,
|
|
3054
|
-
},
|
|
3055
|
-
coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
|
|
3056
|
-
coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
|
|
3057
|
-
coordinatorAdmission === undefined
|
|
3058
|
-
? null
|
|
3059
|
-
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
|
|
3060
|
-
coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
|
|
3061
|
-
coordinatorAdmission === undefined
|
|
3062
|
-
? null
|
|
3063
|
-
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
|
|
3064
|
-
})}\n`);
|
|
3065
|
-
};
|
|
3066
|
-
const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
|
|
3067
|
-
watcherPushStagesTimer.unref?.();
|
|
3068
|
-
let pendingReceivers = 0;
|
|
3069
|
-
let lastSyncWorkStateEmitAt = 0;
|
|
3070
|
-
let syncWorkStateTimer;
|
|
3071
|
-
const emitSyncWorkState = (force = false) => {
|
|
3072
|
-
const now = Date.now();
|
|
3073
|
-
const elapsed = now - lastSyncWorkStateEmitAt;
|
|
3074
|
-
if (force || elapsed >= 250) {
|
|
3075
|
-
if (syncWorkStateTimer !== undefined)
|
|
3076
|
-
clearTimeout(syncWorkStateTimer);
|
|
3077
|
-
syncWorkStateTimer = undefined;
|
|
3078
|
-
lastSyncWorkStateEmitAt = now;
|
|
3079
|
-
const snapshot = {
|
|
3080
|
-
passActive: activePassCount > 0,
|
|
3081
|
-
queuedWatcherPushes: pendingWatcherPaths.size,
|
|
3082
|
-
inFlightWatcherPushes,
|
|
3083
|
-
pendingReceivers,
|
|
3084
|
-
};
|
|
3085
|
-
deps.onSyncWorkStateSnapshot?.(snapshot);
|
|
3086
|
-
stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
|
|
3087
|
-
return;
|
|
3088
|
-
}
|
|
3089
|
-
if (syncWorkStateTimer !== undefined)
|
|
3090
|
-
return;
|
|
3091
|
-
syncWorkStateTimer = setTimeout(() => {
|
|
3092
|
-
syncWorkStateTimer = undefined;
|
|
3093
|
-
emitSyncWorkState(true);
|
|
3094
|
-
}, Math.max(1, 250 - elapsed));
|
|
3095
|
-
syncWorkStateTimer.unref?.();
|
|
3096
|
-
};
|
|
3097
2492
|
const droppedUnresolvedWatcherSlugs = new Set();
|
|
3098
2493
|
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
3099
2494
|
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
@@ -3161,11 +2556,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3161
2556
|
}
|
|
3162
2557
|
}
|
|
3163
2558
|
if (batch) {
|
|
3164
|
-
if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
|
|
3165
|
-
durableWatcherPaths.clear();
|
|
3166
|
-
durableWatcherOverflowed = true;
|
|
3167
|
-
scheduleDurableWatcherPersist();
|
|
3168
|
-
}
|
|
3169
2559
|
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
3170
2560
|
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
3171
2561
|
return previousSequence === undefined ||
|
|
@@ -3174,7 +2564,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3174
2564
|
});
|
|
3175
2565
|
if (entries.length === 0)
|
|
3176
2566
|
return false;
|
|
3177
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
3178
2567
|
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
3179
2568
|
pendingWatcherOriginalBatch =
|
|
3180
2569
|
pendingWatcherPaths.size === 0 &&
|
|
@@ -3186,13 +2575,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3186
2575
|
: null;
|
|
3187
2576
|
for (const [absolutePath, relativePath] of entries) {
|
|
3188
2577
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3189
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3190
2578
|
const change = batch.changes?.get(absolutePath);
|
|
3191
2579
|
if (change)
|
|
3192
2580
|
pendingWatcherChanges.set(absolutePath, change);
|
|
3193
2581
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3194
2582
|
}
|
|
3195
|
-
emitSyncWorkState(queueWasEmpty);
|
|
3196
2583
|
if (batch.overflowed) {
|
|
3197
2584
|
pendingWatcherOverflowed = true;
|
|
3198
2585
|
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
@@ -3216,20 +2603,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3216
2603
|
// counterpart must be re-derived through the codec — a raw join would
|
|
3217
2604
|
// point at the impossible colon path on Windows.
|
|
3218
2605
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
3219
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
3220
2606
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
3221
|
-
addDurableWatcherPath(absolutePath, changedRelPath);
|
|
3222
2607
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3223
2608
|
// A bare path signal carries no delete intent; drop any change record
|
|
3224
2609
|
// a prior batch left for this path so the stale intent cannot leak
|
|
3225
2610
|
// into the coalesced batch (#227).
|
|
3226
2611
|
pendingWatcherChanges.delete(absolutePath);
|
|
3227
|
-
emitSyncWorkState(queueWasEmpty);
|
|
3228
2612
|
return true;
|
|
3229
2613
|
}
|
|
3230
2614
|
pendingWatcherOriginalBatch = null;
|
|
3231
2615
|
pendingWatcherBareChange = true;
|
|
3232
|
-
emitSyncWorkState();
|
|
3233
2616
|
return true;
|
|
3234
2617
|
};
|
|
3235
2618
|
const takePendingWatcherChange = () => {
|
|
@@ -3274,7 +2657,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3274
2657
|
pendingWatcherDroppedRouteHints.clear();
|
|
3275
2658
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
3276
2659
|
pendingWatcherBatchGeneration = null;
|
|
3277
|
-
emitSyncWorkState();
|
|
3278
2660
|
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
3279
2661
|
};
|
|
3280
2662
|
const restorePendingWatcherPaths = (paths, generation, changes) => {
|
|
@@ -3288,54 +2670,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3288
2670
|
if (pendingWatcherChangeSequences.has(absolutePath))
|
|
3289
2671
|
continue;
|
|
3290
2672
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3291
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3292
2673
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3293
2674
|
const change = changes?.get(absolutePath);
|
|
3294
2675
|
if (change)
|
|
3295
2676
|
pendingWatcherChanges.set(absolutePath, change);
|
|
3296
2677
|
}
|
|
3297
|
-
emitSyncWorkState();
|
|
3298
2678
|
};
|
|
3299
|
-
if (eventPush) {
|
|
3300
|
-
try {
|
|
3301
|
-
const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
|
|
3302
|
-
if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
|
|
3303
|
-
(!Array.isArray(stored.paths) && stored.overflowed !== true)) {
|
|
3304
|
-
throw new Error("stored watcher push state has an unsupported shape or root");
|
|
3305
|
-
}
|
|
3306
|
-
if (stored.overflowed === true) {
|
|
3307
|
-
durableWatcherOverflowed = true;
|
|
3308
|
-
pendingWatcherOverflowed = true;
|
|
3309
|
-
pendingWatcherWarmupCatchup = true;
|
|
3310
|
-
pendingWatcherDroppedRoutesUnknown = true;
|
|
3311
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3312
|
-
emitSyncWorkState(true);
|
|
3313
|
-
process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
|
|
3314
|
-
}
|
|
3315
|
-
for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
|
|
3316
|
-
if (typeof relativePath !== "string" || relativePath.length === 0)
|
|
3317
|
-
continue;
|
|
3318
|
-
const route = routeChangeToTarget(relativePath);
|
|
3319
|
-
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3320
|
-
continue;
|
|
3321
|
-
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
3322
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3323
|
-
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3324
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3325
|
-
}
|
|
3326
|
-
if (pendingWatcherPaths.size > 0) {
|
|
3327
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3328
|
-
pendingWatcherWarmupCatchup = true;
|
|
3329
|
-
emitSyncWorkState(true);
|
|
3330
|
-
process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
|
|
3331
|
-
}
|
|
3332
|
-
}
|
|
3333
|
-
catch (err) {
|
|
3334
|
-
if (!(err instanceof Error && err.code === "ENOENT")) {
|
|
3335
|
-
process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
|
|
3336
|
-
}
|
|
3337
|
-
}
|
|
3338
|
-
}
|
|
3339
2679
|
const routePushFailureBackoff = new RoutePushFailureBackoff();
|
|
3340
2680
|
const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
|
|
3341
2681
|
const routePushRetryTimerHandles = new Map();
|
|
@@ -3916,12 +3256,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3916
3256
|
const relativePath = route.kind === "personal"
|
|
3917
3257
|
? key
|
|
3918
3258
|
: ["companies", route.slug, key].join("/");
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
if (
|
|
3259
|
+
const localPath = path.resolve(hqRoot, ...relativePath.split("/"));
|
|
3260
|
+
const fromRoot = path.relative(hqRoot, localPath);
|
|
3261
|
+
const parentPrefix = ["..", path.sep].join("");
|
|
3262
|
+
if (fromRoot === ".." ||
|
|
3263
|
+
fromRoot.startsWith(parentPrefix) ||
|
|
3264
|
+
path.isAbsolute(fromRoot)) {
|
|
3923
3265
|
continue;
|
|
3924
|
-
|
|
3266
|
+
}
|
|
3925
3267
|
if (shouldSync(localPath)) {
|
|
3926
3268
|
paths.push(relativePath);
|
|
3927
3269
|
continue;
|
|
@@ -4184,6 +3526,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4184
3526
|
// a file and still be refused as stale (a download changed the directory
|
|
4185
3527
|
// mid-pass); the retry then finds the file already in the cloud and uploads
|
|
4186
3528
|
// nothing, so without this the file is never announced to peers.
|
|
3529
|
+
const uploadedAwaitingPublish = new Set();
|
|
3530
|
+
const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
|
|
4187
3531
|
const rememberUploadedPaths = (result) => {
|
|
4188
3532
|
if (result === undefined || typeof result === "number")
|
|
4189
3533
|
return;
|
|
@@ -4196,14 +3540,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4196
3540
|
const oldest = uploadedAwaitingPublish.values().next().value;
|
|
4197
3541
|
if (oldest !== undefined)
|
|
4198
3542
|
uploadedAwaitingPublish.delete(oldest);
|
|
4199
|
-
if (!uploadedAwaitingPublishDropLogged) {
|
|
4200
|
-
uploadedAwaitingPublishDropLogged = true;
|
|
4201
|
-
process.stderr.write(`${JSON.stringify({
|
|
4202
|
-
type: "event-sync-upload-buffer-overflow",
|
|
4203
|
-
cap: UPLOADED_AWAITING_PUBLISH_CAP,
|
|
4204
|
-
action: "drop-oldest",
|
|
4205
|
-
})}\n`);
|
|
4206
|
-
}
|
|
4207
3543
|
}
|
|
4208
3544
|
}
|
|
4209
3545
|
};
|
|
@@ -4318,48 +3654,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4318
3654
|
? localUploadSizeClass(uploadPaths)
|
|
4319
3655
|
: "small";
|
|
4320
3656
|
const largeUpload = sizeClass !== "small";
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
watcherPushStage("worker entry");
|
|
4335
|
-
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
4336
|
-
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
4337
|
-
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
4338
|
-
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
4339
|
-
? () => prepareWatcherChanges(admittedChanges)
|
|
4340
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
|
|
4341
|
-
// Watcher-originated pushes may share the guarded
|
|
4342
|
-
// realtime slot with an in-flight pull. Coordinator
|
|
4343
|
-
// fencing already serializes overlapping path epochs;
|
|
4344
|
-
// this must not depend on the optional full-pass flag.
|
|
4345
|
-
true, undefined, routeId, watcherPushStage);
|
|
4346
|
-
}, sizeClass)
|
|
4347
|
-
: (() => {
|
|
4348
|
-
watcherPushStage("worker entry");
|
|
4349
|
-
return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
4350
|
-
? () => prepareWatcherChanges(group.changes)
|
|
4351
|
-
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
|
|
4352
|
-
result,
|
|
4353
|
-
accepted: true,
|
|
4354
|
-
coordinatorRefusedPaths: [],
|
|
4355
|
-
}));
|
|
4356
|
-
})());
|
|
4357
|
-
}
|
|
4358
|
-
finally {
|
|
4359
|
-
watcherPushStages.delete(groupId);
|
|
4360
|
-
inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
|
|
4361
|
-
emitSyncWorkState(true);
|
|
4362
|
-
}
|
|
3657
|
+
const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
|
|
3658
|
+
const execution = parallelCoordinatorEnabled && !chorePush
|
|
3659
|
+
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
|
|
3660
|
+
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
3661
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
3662
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
3663
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
3664
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
3665
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
|
|
3666
|
+
}, sizeClass)
|
|
3667
|
+
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
3668
|
+
? () => prepareWatcherChanges(group.changes)
|
|
3669
|
+
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
4363
3670
|
const result = execution.result;
|
|
4364
3671
|
rememberUploadedPaths(result);
|
|
4365
3672
|
const pushExit = passExitCode(result);
|
|
@@ -4378,7 +3685,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4378
3685
|
}
|
|
4379
3686
|
const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
|
|
4380
3687
|
relativePath.startsWith(`${refusedPath}/`));
|
|
4381
|
-
acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
|
|
4382
3688
|
const publishPaths = new Map();
|
|
4383
3689
|
for (const [absolutePath, relativePath] of group.paths) {
|
|
4384
3690
|
if (isRefusedPath(relativePath))
|
|
@@ -4418,11 +3724,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4418
3724
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
4419
3725
|
}
|
|
4420
3726
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
4421
|
-
acknowledgeDurableWatcherUploads(group.relPaths);
|
|
4422
3727
|
routePushFailureBackoff.succeeded(routeId);
|
|
4423
3728
|
clearRoutePushRetry(routeId);
|
|
4424
3729
|
if (group.route.kind === "company") {
|
|
4425
|
-
deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
|
|
4426
3730
|
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
4427
3731
|
}
|
|
4428
3732
|
}
|
|
@@ -4484,7 +3788,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4484
3788
|
const execution = parallelCoordinatorEnabled
|
|
4485
3789
|
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
4486
3790
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4487
|
-
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced
|
|
3791
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
4488
3792
|
})
|
|
4489
3793
|
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
4490
3794
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
@@ -4536,7 +3840,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4536
3840
|
addRoute(hint);
|
|
4537
3841
|
const skipCompanies = resolveSkipCompanies();
|
|
4538
3842
|
if (parallelCoordinatorEnabled) {
|
|
4539
|
-
let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
|
|
4540
3843
|
for (const route of routes.values()) {
|
|
4541
3844
|
if (route.kind === "company" &&
|
|
4542
3845
|
skipCompanies.has(route.slug.toLowerCase())) {
|
|
@@ -4565,11 +3868,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4565
3868
|
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
4566
3869
|
interruptWait();
|
|
4567
3870
|
}
|
|
4568
|
-
else {
|
|
4569
|
-
routesRemaining -= 1;
|
|
4570
|
-
if (routesRemaining === 0)
|
|
4571
|
-
clearDurableWatcherOverflow();
|
|
4572
|
-
}
|
|
4573
3871
|
});
|
|
4574
3872
|
}
|
|
4575
3873
|
return 0;
|
|
@@ -4685,9 +3983,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4685
3983
|
// churn overflowed the 200-path batch and skipped publishBatch.
|
|
4686
3984
|
personalMode: true,
|
|
4687
3985
|
includeCompanyPaths: companiesMode || companyMode,
|
|
4688
|
-
// Inventory filtered candidates so the route's push filter remains
|
|
4689
|
-
// authoritative and can attribute exclusions in warm-up diagnostics.
|
|
4690
|
-
includeFilteredWarmupCandidates: true,
|
|
4691
3986
|
onDegraded: opts.onDegraded,
|
|
4692
3987
|
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
4693
3988
|
queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
|
|
@@ -4809,7 +4104,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4809
4104
|
}
|
|
4810
4105
|
}
|
|
4811
4106
|
watcher.start();
|
|
4812
|
-
let catchupPathStatWarningReported = false;
|
|
4813
4107
|
const reportWatcherReady = () => {
|
|
4814
4108
|
if (stopped || watcher === null)
|
|
4815
4109
|
return;
|
|
@@ -4850,202 +4144,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4850
4144
|
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
|
|
4851
4145
|
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
4852
4146
|
return;
|
|
4853
|
-
const
|
|
4854
|
-
const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
|
|
4855
|
-
const eligiblePaths = new Map();
|
|
4856
|
-
const eligibleChanges = new Map();
|
|
4857
|
-
const excludedByReason = {};
|
|
4858
|
-
const excludedByReasonByRoute = {};
|
|
4859
|
-
const personalFilterExclusions = new Map();
|
|
4860
|
-
const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
|
|
4861
|
-
const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
|
|
4862
|
-
const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
|
|
4863
|
-
const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
|
|
4864
|
-
const clearedResolvedCompanies = new Set();
|
|
4865
|
-
const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
|
|
4866
|
-
const candidateRouteCounts = new Map();
|
|
4867
|
-
const candidateRoutePrefixes = new Map();
|
|
4868
|
-
const eligibleRouteCounts = new Map();
|
|
4869
|
-
const eligibleRoutePrefixes = new Map();
|
|
4870
|
-
const backupClass = (relativePath) => {
|
|
4871
|
-
if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
|
|
4872
|
-
return "attioCleanup";
|
|
4873
|
-
if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
|
|
4874
|
-
return "readModelRollout";
|
|
4875
|
-
return undefined;
|
|
4876
|
-
};
|
|
4877
|
-
const exclude = (reason, routeName) => {
|
|
4878
|
-
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
|
|
4879
|
-
const routeReasons = excludedByReasonByRoute[routeName] ??= {};
|
|
4880
|
-
routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
|
|
4881
|
-
};
|
|
4882
|
-
let filteredCandidateCount = 0;
|
|
4883
|
-
for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
|
|
4884
|
-
filteredCandidateCount += count;
|
|
4885
|
-
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
|
|
4886
|
-
const routeReasons = excludedByReasonByRoute.filtered ??= {};
|
|
4887
|
-
routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
|
|
4888
|
-
}
|
|
4889
|
-
if (filteredCandidateCount > 0)
|
|
4890
|
-
candidateRouteCounts.set("filtered", filteredCandidateCount);
|
|
4891
|
-
let examinedWarmupPaths = 0;
|
|
4892
|
-
for (const [absolutePath, relativePath] of catchup.batch.paths) {
|
|
4893
|
-
examinedWarmupPaths += 1;
|
|
4894
|
-
const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
|
|
4895
|
-
if (warmupYield)
|
|
4896
|
-
await warmupYield;
|
|
4897
|
-
if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
|
|
4898
|
-
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
|
|
4899
|
-
exclude("unsafe-path", "unrouted");
|
|
4900
|
-
continue;
|
|
4901
|
-
}
|
|
4902
|
-
const route = routeChangeToTarget(relativePath);
|
|
4903
|
-
const routeName = route ? routeKey(route) : "unrouted";
|
|
4904
|
-
const summaryRoutePath = route?.kind === "company"
|
|
4905
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4906
|
-
: relativePath;
|
|
4907
|
-
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
|
|
4908
|
-
const ontologyBackupClass = backupClass(relativePath);
|
|
4909
|
-
if (ontologyBackupClass) {
|
|
4910
|
-
exclude("local-rule:ontology-backup", routeName);
|
|
4911
|
-
ontologyBackupPaths[ontologyBackupClass] += 1;
|
|
4912
|
-
continue;
|
|
4913
|
-
}
|
|
4914
|
-
if (route?.kind === "company") {
|
|
4915
|
-
const resolvedNow = cloudCompanySlugs.has(route.slug) ||
|
|
4916
|
-
skipStore?.hasCurrentGrantCompany(route.slug) ||
|
|
4917
|
-
skipStore?.hasCurrentGrantVersion(route.slug) ||
|
|
4918
|
-
(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
|
|
4919
|
-
if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
|
|
4920
|
-
skipStore?.clearUnresolvedCompany(route.slug);
|
|
4921
|
-
clearedResolvedCompanies.add(route.slug);
|
|
4922
|
-
}
|
|
4923
|
-
if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
|
|
4924
|
-
exclude("unresolved-company", routeName);
|
|
4925
|
-
continue;
|
|
4926
|
-
}
|
|
4927
|
-
}
|
|
4928
|
-
if (route?.kind === "company" &&
|
|
4929
|
-
!cloudCompanySlugs.has(route.slug) &&
|
|
4930
|
-
!skipStore?.hasCurrentGrantCompany(route.slug) &&
|
|
4931
|
-
!skipStore?.hasCurrentGrantVersion(route.slug) &&
|
|
4932
|
-
!(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
|
|
4933
|
-
exclude("local-only-company", routeName);
|
|
4934
|
-
continue;
|
|
4935
|
-
}
|
|
4936
|
-
const company = route?.kind === "company" ? route.slug : "personal";
|
|
4937
|
-
const routePath = route?.kind === "company"
|
|
4938
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4939
|
-
: relativePath;
|
|
4940
|
-
if (isEphemeralPath(routePath)) {
|
|
4941
|
-
exclude("push-filter:conflict-mirror", routeName);
|
|
4942
|
-
continue;
|
|
4943
|
-
}
|
|
4944
|
-
// The production TreeWatcher collector has already applied this
|
|
4945
|
-
// shared filter stack and reports rejected files as reason counts.
|
|
4946
|
-
// Injected/custom collectors without that proof still evaluate the
|
|
4947
|
-
// route filter here, with personal defaults first for `.env` attribution.
|
|
4948
|
-
const personalDefaultExclusion = catchup.pushFiltersApplied === true
|
|
4949
|
-
? undefined
|
|
4950
|
-
: route?.kind === "personal"
|
|
4951
|
-
? matchPersonalVaultExclusion(relativePath, false)
|
|
4952
|
-
: undefined;
|
|
4953
|
-
const isPersonalRoute = route?.kind !== "company";
|
|
4954
|
-
const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
|
|
4955
|
-
? personalPushFilter(absolutePath, false, relativePath)
|
|
4956
|
-
: ignoreFilter(absolutePath, false, relativePath)));
|
|
4957
|
-
if (!acceptedByPushFilter) {
|
|
4958
|
-
const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
|
|
4959
|
-
? personalFilterExclusions.get(relativePath)
|
|
4960
|
-
: undefined);
|
|
4961
|
-
exclude(exclusionId
|
|
4962
|
-
? `push-filter:personal:${exclusionId}`
|
|
4963
|
-
: "push-filter:ignore", routeName);
|
|
4964
|
-
continue;
|
|
4965
|
-
}
|
|
4966
|
-
if (isPersonalRoute &&
|
|
4967
|
-
!shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
|
|
4968
|
-
isPersonalVaultScopeExcluded(relativePath, {
|
|
4969
|
-
continuityPaths,
|
|
4970
|
-
hasAgencyCarveout,
|
|
4971
|
-
hasSessionLogsCarveout,
|
|
4972
|
-
})) {
|
|
4973
|
-
exclude("push-filter:personal:excluded-top-level", routeName);
|
|
4974
|
-
continue;
|
|
4975
|
-
}
|
|
4976
|
-
if (route && isLocallyIneligibleForPush(routePath, {
|
|
4977
|
-
boardClientOwned: true,
|
|
4978
|
-
ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
|
|
4979
|
-
})) {
|
|
4980
|
-
const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
|
|
4981
|
-
exclude(`local-rule:${pathClass}`, routeName);
|
|
4982
|
-
continue;
|
|
4983
|
-
}
|
|
4984
|
-
const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
|
|
4985
|
-
const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
|
|
4986
|
-
try {
|
|
4987
|
-
const stat = !catchup.symlinkPaths || hasSkipRecord
|
|
4988
|
-
? fs.lstatSync(absolutePath)
|
|
4989
|
-
: undefined;
|
|
4990
|
-
if (route && (knownSymlink || stat?.isSymbolicLink())) {
|
|
4991
|
-
const scopeRoot = route.kind === "company"
|
|
4992
|
-
? path.join(hqRoot, "companies", route.slug)
|
|
4993
|
-
: relativePath.startsWith("personal/")
|
|
4994
|
-
? path.join(hqRoot, "personal")
|
|
4995
|
-
: hqRoot;
|
|
4996
|
-
const inspection = inspectLocalSymlinkScope({
|
|
4997
|
-
absoluteLinkPath: absolutePath,
|
|
4998
|
-
linkPath: relativePath,
|
|
4999
|
-
scopeRoot,
|
|
5000
|
-
scope: route.kind === "company"
|
|
5001
|
-
? { kind: "company", companySlug: route.slug }
|
|
5002
|
-
: { kind: "personal" },
|
|
5003
|
-
});
|
|
5004
|
-
if (inspection.refused) {
|
|
5005
|
-
exclude("symlink", routeName);
|
|
5006
|
-
continue;
|
|
5007
|
-
}
|
|
5008
|
-
}
|
|
5009
|
-
if (hasSkipRecord && stat &&
|
|
5010
|
-
skipStore?.matches(company, routePath, stat)) {
|
|
5011
|
-
exclude("record-match", routeName);
|
|
5012
|
-
continue;
|
|
5013
|
-
}
|
|
5014
|
-
}
|
|
5015
|
-
catch (error) {
|
|
5016
|
-
// Fail open: unreadable state or a filesystem race must not suppress a candidate.
|
|
5017
|
-
if (!catchupPathStatWarningReported) {
|
|
5018
|
-
catchupPathStatWarningReported = true;
|
|
5019
|
-
process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
|
|
5020
|
-
}
|
|
5021
|
-
}
|
|
5022
|
-
eligiblePaths.set(absolutePath, relativePath);
|
|
5023
|
-
recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
|
|
5024
|
-
const change = catchup.batch.changes?.get(absolutePath);
|
|
5025
|
-
if (change)
|
|
5026
|
-
eligibleChanges.set(absolutePath, change);
|
|
5027
|
-
}
|
|
5028
|
-
const queued = addPendingWatcherChange(undefined, {
|
|
5029
|
-
paths: eligiblePaths,
|
|
5030
|
-
changes: eligibleChanges,
|
|
5031
|
-
}, {
|
|
4147
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
5032
4148
|
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
5033
4149
|
});
|
|
5034
4150
|
pendingWatcherWarmupCatchup ||= queued;
|
|
5035
|
-
const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
|
|
5036
4151
|
process.stderr.write(JSON.stringify({
|
|
5037
4152
|
type: "watcher-warmup-catchup",
|
|
5038
4153
|
reason: "watch-set-ready",
|
|
5039
|
-
pathsFound:
|
|
5040
|
-
candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
|
|
5041
|
-
pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
|
|
5042
|
-
pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
|
|
5043
|
-
excludedByReason: {
|
|
5044
|
-
...excludedByReason,
|
|
5045
|
-
...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
|
|
5046
|
-
},
|
|
5047
|
-
excludedByReasonByRoute,
|
|
5048
|
-
ontologyBackupPaths,
|
|
4154
|
+
pathsFound: catchup.batch.paths.size,
|
|
5049
4155
|
scannedPaths: catchup.scannedPaths,
|
|
5050
4156
|
elapsedMs: Date.now() - startedAt,
|
|
5051
4157
|
queued,
|
|
@@ -5112,16 +4218,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5112
4218
|
const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
5113
4219
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
5114
4220
|
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
5115
|
-
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal
|
|
4221
|
+
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
5116
4222
|
});
|
|
5117
4223
|
const pathCount = events.length;
|
|
5118
4224
|
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
5119
4225
|
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
5120
4226
|
if (execution.admissionTimeout)
|
|
5121
4227
|
throw execution.admissionTimeout;
|
|
5122
|
-
if (execution.yieldedToPush) {
|
|
5123
|
-
throw new RealtimePushYieldError();
|
|
5124
|
-
}
|
|
5125
4228
|
const exitCode = passExitCode(execution.result);
|
|
5126
4229
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
5127
4230
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -5143,82 +4246,74 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5143
4246
|
const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
|
|
5144
4247
|
if (stopped)
|
|
5145
4248
|
return;
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
if (lane) {
|
|
5163
|
-
receiverLaneByBatch.delete(events);
|
|
5164
|
-
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5165
|
-
return;
|
|
5166
|
-
}
|
|
5167
|
-
// Direct embedders may invoke the engine without first going through
|
|
5168
|
-
// SqsPushReceiver. Preserve that seam while applying the same
|
|
5169
|
-
// delivery-wide partition the receiver would expose.
|
|
5170
|
-
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5171
|
-
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5172
|
-
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4249
|
+
// The injectable receiver seam and the production SQS receiver both
|
|
4250
|
+
// arrive here only after a successful receive. Keep the cadence tight
|
|
4251
|
+
// until the receiver later reports an empty outer long-poll.
|
|
4252
|
+
onReceiveActivity(true);
|
|
4253
|
+
// A received wake/high-water signal means there is bounded work now.
|
|
4254
|
+
// It wakes the scheduler but is still funneled through runGuarded below,
|
|
4255
|
+
// so it cannot overlap the pass that may currently own the operation
|
|
4256
|
+
// lock.
|
|
4257
|
+
// Apply the caps once across the whole delivery, then retain the residue
|
|
4258
|
+
// in the normal lane. The receiver already coalesces to one latest event
|
|
4259
|
+
// per path, so the partition cannot split revisions of one path.
|
|
4260
|
+
if (parallelCoordinatorEnabled) {
|
|
4261
|
+
const lane = receiverLaneByBatch.get(events);
|
|
4262
|
+
if (lane) {
|
|
4263
|
+
receiverLaneByBatch.delete(events);
|
|
4264
|
+
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5173
4265
|
return;
|
|
5174
4266
|
}
|
|
4267
|
+
// Direct embedders may invoke the engine without first going through
|
|
4268
|
+
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4269
|
+
// delivery-wide partition the receiver would expose.
|
|
5175
4270
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
}
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
}
|
|
5205
|
-
if (exitCode !== 0) {
|
|
4271
|
+
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4272
|
+
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4273
|
+
return;
|
|
4274
|
+
}
|
|
4275
|
+
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4276
|
+
const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
|
|
4277
|
+
const eventsByRoute = new Map();
|
|
4278
|
+
for (const event of laneEvents) {
|
|
4279
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
4280
|
+
// The receiver is an independent ingress from the local watcher.
|
|
4281
|
+
// Apply the same route-selection boundary before it can enqueue a
|
|
4282
|
+
// fast pull: a --skip-personal runner must acknowledge, not pull,
|
|
4283
|
+
// a personal-namespace delivery.
|
|
4284
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
4285
|
+
continue;
|
|
4286
|
+
const key = routeKey(route);
|
|
4287
|
+
const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
|
|
4288
|
+
group.events.push(event);
|
|
4289
|
+
if (event.kind === "upsert")
|
|
4290
|
+
group.upsertPaths.push(event.relativePath);
|
|
4291
|
+
eventsByRoute.set(key, group);
|
|
4292
|
+
}
|
|
4293
|
+
for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
|
|
4294
|
+
const paths = routeEvents.map((event) => event.relativePath);
|
|
4295
|
+
const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
|
|
4296
|
+
const exitCode = passExitCode(result);
|
|
4297
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
4298
|
+
if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
|
|
5206
4299
|
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
5207
4300
|
}
|
|
4301
|
+
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
5208
4302
|
await settleEvents?.(routeEvents);
|
|
4303
|
+
continue;
|
|
5209
4304
|
}
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
4305
|
+
if (exitCode !== 0) {
|
|
4306
|
+
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4307
|
+
}
|
|
4308
|
+
await settleEvents?.(routeEvents);
|
|
4309
|
+
}
|
|
4310
|
+
};
|
|
4311
|
+
// Dispatch realtime work before awaiting its bulk residue. This lets an
|
|
4312
|
+
// established slow pass continue while the small scoped pull uses its
|
|
4313
|
+
// ref-counted lock carve-out; the residue remains serialized and cannot
|
|
4314
|
+
// starve the slow lane.
|
|
4315
|
+
await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
|
|
4316
|
+
await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
|
|
5222
4317
|
};
|
|
5223
4318
|
receiverSyncBatchFn.partition = (events, maxPaths) => {
|
|
5224
4319
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
@@ -5375,14 +4470,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5375
4470
|
if (stopped)
|
|
5376
4471
|
return shutdownPromise ?? Promise.resolve();
|
|
5377
4472
|
stopped = true;
|
|
5378
|
-
clearInterval(watcherPushStagesTimer);
|
|
5379
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
5380
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
5381
|
-
durableWatcherPersistTimer = undefined;
|
|
5382
|
-
if (syncWorkStateTimer !== undefined)
|
|
5383
|
-
clearTimeout(syncWorkStateTimer);
|
|
5384
|
-
syncWorkStateTimer = undefined;
|
|
5385
|
-
emitSyncWorkState(true);
|
|
5386
4473
|
for (const route of routePushRetryTimerHandles.keys()) {
|
|
5387
4474
|
clearRoutePushRetry(route);
|
|
5388
4475
|
}
|
|
@@ -5412,7 +4499,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5412
4499
|
coordinatorDisposal,
|
|
5413
4500
|
boundedEventSyncStartup,
|
|
5414
4501
|
eventSyncDisposal,
|
|
5415
|
-
persistDurableWatcherPaths(),
|
|
5416
4502
|
]).then(() => undefined);
|
|
5417
4503
|
try {
|
|
5418
4504
|
void receiver?.dispose();
|
|
@@ -5830,14 +4916,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5830
4916
|
lastCompletedFullReconcileWallAt = Date.now();
|
|
5831
4917
|
fullPassJournalHealthy = execution.accepted &&
|
|
5832
4918
|
isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
|
|
5833
|
-
if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
|
|
5834
|
-
clearDurableWatcherOverflow();
|
|
5835
|
-
}
|
|
5836
|
-
else if (fullPassJournalHealthy && pending.batch !== null) {
|
|
5837
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
5838
|
-
...pending.batch.paths.values(),
|
|
5839
|
-
]);
|
|
5840
|
-
}
|
|
5841
4919
|
if (fullPassJournalHealthy) {
|
|
5842
4920
|
fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
|
|
5843
4921
|
}
|
|
@@ -5895,11 +4973,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5895
4973
|
const catchupBatch = workingBatch;
|
|
5896
4974
|
const catchupGeneration = pending.generation;
|
|
5897
4975
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
5898
|
-
if (exit === 0)
|
|
5899
|
-
clearDurableWatcherOverflow();
|
|
5900
|
-
return;
|
|
5901
|
-
}
|
|
5902
|
-
if (exit === SCOPED_DRAIN_DEFERRED)
|
|
4976
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
5903
4977
|
return;
|
|
5904
4978
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
5905
4979
|
interruptWait();
|
|
@@ -6007,19 +5081,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
6007
5081
|
? exitCode === 0
|
|
6008
5082
|
: hasCompletedRunnerPass(result));
|
|
6009
5083
|
if (completedPostRepairPass) {
|
|
6010
|
-
if (!parallelCoordinatorEnabled &&
|
|
6011
|
-
pending.batch?.overflowed === true &&
|
|
6012
|
-
((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
|
|
6013
|
-
clearDurableWatcherOverflow();
|
|
6014
|
-
}
|
|
6015
|
-
else if (!parallelCoordinatorEnabled &&
|
|
6016
|
-
ranFullReconcile &&
|
|
6017
|
-
fullPassJournalHealthy &&
|
|
6018
|
-
pending.batch !== null) {
|
|
6019
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
6020
|
-
...pending.batch.paths.values(),
|
|
6021
|
-
]);
|
|
6022
|
-
}
|
|
6023
5084
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
6024
5085
|
// which follows journal repair. Its numeric seam has the same
|
|
6025
5086
|
// contract: zero means the injected pass completed. This preserves
|