@indigoai-us/hq-cloud 6.18.52 → 6.18.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-audit.d.ts +26 -0
- package/dist/bin/sync-runner-audit.d.ts.map +1 -0
- package/dist/bin/sync-runner-audit.js +51 -0
- package/dist/bin/sync-runner-audit.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +18 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +186 -9
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +107 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +1382 -183
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +63 -11
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +350 -95
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/watcher-catchup-skip-store.d.ts +60 -0
- package/dist/bin/watcher-catchup-skip-store.d.ts.map +1 -0
- package/dist/bin/watcher-catchup-skip-store.js +282 -0
- package/dist/bin/watcher-catchup-skip-store.js.map +1 -0
- package/dist/cli/conflict.d.ts +6 -0
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +3 -1
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +95 -5
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +25 -1
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +69 -4
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +27 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +151 -30
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +18 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +90 -25
- package/dist/cli/sync.js.map +1 -1
- package/dist/ignore.d.ts +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +2 -2
- package/dist/ignore.js.map +1 -1
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +6 -0
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts +8 -0
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +208 -17
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal.d.ts +18 -2
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +248 -17
- package/dist/journal.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +9 -2
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +34 -2
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +2 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +2 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/personal-vault-exclusions.d.ts +2 -2
- package/dist/personal-vault-exclusions.d.ts.map +1 -1
- package/dist/personal-vault-exclusions.js +5 -3
- package/dist/personal-vault-exclusions.js.map +1 -1
- package/dist/personal-vault.d.ts +7 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/qmd-runner-admission.d.ts +22 -0
- package/dist/qmd-runner-admission.d.ts.map +1 -0
- package/dist/qmd-runner-admission.js +107 -0
- package/dist/qmd-runner-admission.js.map +1 -0
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +2 -0
- package/dist/s3.js.map +1 -1
- package/dist/symlink-target-policy.d.ts +11 -0
- package/dist/symlink-target-policy.d.ts.map +1 -1
- package/dist/symlink-target-policy.js +110 -0
- package/dist/symlink-target-policy.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +3 -1
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +3 -0
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +3 -0
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +29 -6
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/mutation-session.d.ts.map +1 -1
- package/dist/sync/mutation-session.js +4 -0
- package/dist/sync/mutation-session.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +29 -0
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +229 -16
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +5 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +32 -10
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/state-store.d.ts +7 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +113 -6
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watch-owner-lease.d.ts +4 -0
- package/dist/watch-owner-lease.d.ts.map +1 -1
- package/dist/watch-owner-lease.js +40 -6
- package/dist/watch-owner-lease.js.map +1 -1
- package/dist/watcher.d.ts +16 -3
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +114 -21
- package/dist/watcher.js.map +1 -1
- package/package.json +4 -3
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
4
5
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
5
6
|
import * as v8 from "node:v8";
|
|
6
7
|
import { performance } from "node:perf_hooks";
|
|
@@ -15,13 +16,18 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
|
15
16
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
16
17
|
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
17
18
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
19
|
+
import { isEphemeralPath } from "../cli/share.js";
|
|
20
|
+
import { isSafeRelativePath } from "../sync/push-event.js";
|
|
21
|
+
import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
|
|
22
|
+
import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
|
|
18
23
|
import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
|
|
19
24
|
import { createIgnoreFilter } from "../ignore.js";
|
|
25
|
+
import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
20
26
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
21
27
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
22
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
|
|
23
|
-
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
24
|
-
import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
28
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, readQuarantinedJournalPaths, hashSymlinkTarget, applyJournalDeltaAsync, } from "../journal.js";
|
|
29
|
+
import { isPersonalVaultDeletionExcluded, readCloudBackedSlugsFromManifest, computeContinuityPointerPaths, computeAgencySyncPaths, computeSessionLogsSyncPaths, isPersonalVaultScopeExcluded, } from "../personal-vault.js";
|
|
30
|
+
import { NoopPushReceiver, ReceiverDispatchError, RealtimePushYieldError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
25
31
|
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
26
32
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
27
33
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
@@ -44,6 +50,36 @@ const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
|
44
50
|
const warnedUnresolvedReceiverCompanies = new Set();
|
|
45
51
|
/** Maximum time shutdown waits for event-sync startup before abandoning it. */
|
|
46
52
|
export const EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS = 5_000;
|
|
53
|
+
/** Group watcher changes by target route while retaining source insertion order. */
|
|
54
|
+
export function groupBatchByRoute(batch) {
|
|
55
|
+
const grouped = new Map();
|
|
56
|
+
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
57
|
+
const change = batch.changes?.get(absolutePath);
|
|
58
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
59
|
+
if (!route)
|
|
60
|
+
continue;
|
|
61
|
+
const category = isSessionLogsPushOnlyPath(relativePath) ? "session-log-backup" : "interactive";
|
|
62
|
+
const key = `${routeKey(route)}:${category}`;
|
|
63
|
+
const existing = grouped.get(key) ?? {
|
|
64
|
+
route,
|
|
65
|
+
paths: new Map(),
|
|
66
|
+
changes: new Map(),
|
|
67
|
+
relPaths: [],
|
|
68
|
+
deleteScopeRoots: [],
|
|
69
|
+
};
|
|
70
|
+
existing.paths.set(absolutePath, relativePath);
|
|
71
|
+
if (change)
|
|
72
|
+
existing.changes.set(absolutePath, change);
|
|
73
|
+
grouped.set(key, existing);
|
|
74
|
+
}
|
|
75
|
+
for (const group of grouped.values()) {
|
|
76
|
+
group.relPaths = [...new Set(group.paths.values())];
|
|
77
|
+
group.deleteScopeRoots = [
|
|
78
|
+
...new Set([...group.changes.values()].flatMap((item) => (item.deleteSnapshots ?? []).flatMap((snapshot) => snapshot.deleteScopeRoot === undefined ? [] : [snapshot.deleteScopeRoot]))),
|
|
79
|
+
];
|
|
80
|
+
}
|
|
81
|
+
return grouped;
|
|
82
|
+
}
|
|
47
83
|
/** A deferred full chore must get the next admission after this long. */
|
|
48
84
|
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
49
85
|
/**
|
|
@@ -61,13 +97,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
61
97
|
*/
|
|
62
98
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
63
99
|
/**
|
|
64
|
-
*
|
|
100
|
+
* Maximum guarded realtime executes, including the reserved watcher push peer.
|
|
65
101
|
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
66
|
-
* workers. One long
|
|
67
|
-
*
|
|
68
|
-
*
|
|
102
|
+
* workers. One long pull and one long push may each retain a normal worker;
|
|
103
|
+
* one additional watcher push can still make bounded progress. In-flight
|
|
104
|
+
* work is never aborted for this admission.
|
|
69
105
|
*/
|
|
70
|
-
export const REALTIME_WORKER_OCCUPANCY_CAP =
|
|
106
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 3;
|
|
71
107
|
/** Objects at least one multipart part are sent through the chore hand-off. */
|
|
72
108
|
export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
|
|
73
109
|
let realtimeAdmissionDeadlineMsForTest;
|
|
@@ -91,6 +127,29 @@ function realtimeDrainBurstMs() {
|
|
|
91
127
|
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
92
128
|
*/
|
|
93
129
|
export const REALTIME_ADMISSION_RETRY_MS = 5_000;
|
|
130
|
+
/** Bound consecutive pull admissions while a watcher push is queued. */
|
|
131
|
+
export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
|
|
132
|
+
/** Bound synchronous warm-up candidate work between event-loop turns. */
|
|
133
|
+
export const WATCHER_WARMUP_YIELD_EVERY = 128;
|
|
134
|
+
export function shouldYieldWatcherWarmup(examinedPaths) {
|
|
135
|
+
return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
|
|
136
|
+
}
|
|
137
|
+
export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
|
|
138
|
+
return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
|
|
139
|
+
}
|
|
140
|
+
export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
|
|
141
|
+
// TreeWatcher marks this only when its production filter has already applied
|
|
142
|
+
// the same personal scope predicate to every collected candidate. Unrouted
|
|
143
|
+
// paths retain the runner check because the collector may include company paths.
|
|
144
|
+
return pushFiltersApplied && routeKind === "personal";
|
|
145
|
+
}
|
|
146
|
+
export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
|
|
147
|
+
const ignoreFilter = createIgnore(hqRoot);
|
|
148
|
+
const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
|
|
149
|
+
onPersonalExcluded(relativePath, match.id);
|
|
150
|
+
});
|
|
151
|
+
return { ignoreFilter, personalPushFilter };
|
|
152
|
+
}
|
|
94
153
|
/**
|
|
95
154
|
* In-process result of a scoped drain whose only shortfall was admission
|
|
96
155
|
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
@@ -191,6 +250,66 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
|
|
|
191
250
|
return explicitlyAllowed ||
|
|
192
251
|
(workClass === "realtime" && isFullPassDurableProgressEnabled());
|
|
193
252
|
}
|
|
253
|
+
export async function runGuardedPassPreparation(prepare, onStage, onFailure) {
|
|
254
|
+
onStage?.("pass preparation");
|
|
255
|
+
if (!prepare)
|
|
256
|
+
return;
|
|
257
|
+
try {
|
|
258
|
+
await prepare();
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
const name = error instanceof Error ? error.name : "UnknownError";
|
|
262
|
+
const errorClass = /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name)
|
|
263
|
+
? name
|
|
264
|
+
: "UnknownError";
|
|
265
|
+
onStage?.("pass preparation failed");
|
|
266
|
+
onFailure?.({ type: "watcher-push-preparation-failed", errorClass });
|
|
267
|
+
onStage?.("pass return");
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
function pathFreeErrorClass(error) {
|
|
272
|
+
if (!(error instanceof Error))
|
|
273
|
+
return undefined;
|
|
274
|
+
return /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name)
|
|
275
|
+
? error.name
|
|
276
|
+
: "UnknownError";
|
|
277
|
+
}
|
|
278
|
+
export function buildScopedPushFailureDiagnostic(fields, error, caughtError, caughtErrorClass) {
|
|
279
|
+
const errorClass = caughtErrorClass ?? pathFreeErrorClass(caughtError);
|
|
280
|
+
return {
|
|
281
|
+
type: "scoped-push-failure",
|
|
282
|
+
...fields,
|
|
283
|
+
...(errorClass ? { errorClass } : {}),
|
|
284
|
+
error,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
export function resolveWatcherPushWaitStage(input) {
|
|
288
|
+
if (input.capYieldActive)
|
|
289
|
+
return "CPU/heap cap wait";
|
|
290
|
+
if (input.holders.some((holder) => holder.kind === "pull"))
|
|
291
|
+
return "pull-yield wait";
|
|
292
|
+
if (input.queueDepth > 0)
|
|
293
|
+
return "queued behind realtime work";
|
|
294
|
+
if (input.holders.some((holder) => holder.kind === "chore"))
|
|
295
|
+
return "chore holder wait";
|
|
296
|
+
if (input.holders.length > 0)
|
|
297
|
+
return "scheduler holder wait";
|
|
298
|
+
return "scheduler admission pending";
|
|
299
|
+
}
|
|
300
|
+
export function watcherPushWaitDiagnostics(input) {
|
|
301
|
+
return {
|
|
302
|
+
stage: input.admissionWaiting
|
|
303
|
+
? resolveWatcherPushWaitStage(input)
|
|
304
|
+
: input.stage,
|
|
305
|
+
waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
|
|
306
|
+
? null
|
|
307
|
+
: Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
|
|
308
|
+
schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
|
|
309
|
+
? null
|
|
310
|
+
: Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
|
|
311
|
+
};
|
|
312
|
+
}
|
|
194
313
|
/**
|
|
195
314
|
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
196
315
|
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
@@ -205,6 +324,8 @@ export class CooperativePassScheduler {
|
|
|
205
324
|
starvationCapMs;
|
|
206
325
|
options;
|
|
207
326
|
pending = [];
|
|
327
|
+
lastAnyAdmissionAtMonotonicMs = null;
|
|
328
|
+
nextEntryId = 0;
|
|
208
329
|
active = null;
|
|
209
330
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
210
331
|
realtimeOccupants = new Set();
|
|
@@ -241,12 +362,65 @@ export class CooperativePassScheduler {
|
|
|
241
362
|
/** Successful callers still applying results after the network hand-off. */
|
|
242
363
|
networkTransferCompletionCount = 0;
|
|
243
364
|
networkTransferApplyRelease = null;
|
|
365
|
+
consecutivePullAdmissions = 0;
|
|
366
|
+
consecutiveFastAdmissions = 0;
|
|
244
367
|
resolveNetworkTransferApplyRelease = null;
|
|
245
368
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
246
369
|
this.now = now;
|
|
247
370
|
this.starvationCapMs = starvationCapMs;
|
|
248
371
|
this.options = options;
|
|
249
372
|
}
|
|
373
|
+
recordAdmission(entry) {
|
|
374
|
+
const monotonicMs = performance.now();
|
|
375
|
+
const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
|
|
376
|
+
? null
|
|
377
|
+
: Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
|
|
378
|
+
const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
379
|
+
const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
|
|
380
|
+
if (entry.workClass === "realtime") {
|
|
381
|
+
if (entry.lane === "fast")
|
|
382
|
+
this.consecutiveFastAdmissions += 1;
|
|
383
|
+
else
|
|
384
|
+
this.consecutiveFastAdmissions = 0;
|
|
385
|
+
}
|
|
386
|
+
const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
|
|
387
|
+
const waitingSinceAdmissionRequestMs = oldestPush
|
|
388
|
+
? Math.max(0, this.now() - oldestPush.enqueuedAt)
|
|
389
|
+
: null;
|
|
390
|
+
if (waitingPush || admittedIsPush) {
|
|
391
|
+
this.options.onAdmission?.({
|
|
392
|
+
entryId: entry.entryId,
|
|
393
|
+
workClass: entry.workClass,
|
|
394
|
+
lane: entry.lane,
|
|
395
|
+
transferDirection: entry.transferDirection,
|
|
396
|
+
transferSizeClass: entry.transferSizeClass ?? "unknown",
|
|
397
|
+
label: entry.label.slice(0, 160),
|
|
398
|
+
admittedAt: new Date().toISOString(),
|
|
399
|
+
schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
|
|
400
|
+
realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
|
|
401
|
+
oldestWaitingWatcherPush: oldestPush
|
|
402
|
+
? {
|
|
403
|
+
entryId: oldestPush.entryId,
|
|
404
|
+
label: oldestPush.label.slice(0, 160),
|
|
405
|
+
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
406
|
+
}
|
|
407
|
+
: null,
|
|
408
|
+
passedOver: waitingPush && waitingPush !== entry
|
|
409
|
+
? {
|
|
410
|
+
entryId: waitingPush.entryId,
|
|
411
|
+
label: waitingPush.label.slice(0, 160),
|
|
412
|
+
waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
|
|
413
|
+
reason: entry.transferDirection === "pull"
|
|
414
|
+
? "pull-admitted-ahead-of-waiting-push"
|
|
415
|
+
: this._handoffActive
|
|
416
|
+
? "handoff-selector-admitted-other-work"
|
|
417
|
+
: "scheduler-admitted-other-work",
|
|
418
|
+
}
|
|
419
|
+
: null,
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
|
|
423
|
+
}
|
|
250
424
|
get handoffActive() {
|
|
251
425
|
return this._handoffActive;
|
|
252
426
|
}
|
|
@@ -263,12 +437,40 @@ export class CooperativePassScheduler {
|
|
|
263
437
|
get hasActiveSlowRealtime() {
|
|
264
438
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
265
439
|
}
|
|
440
|
+
get isInCapYield() {
|
|
441
|
+
return this.capYieldDepth > 0;
|
|
442
|
+
}
|
|
266
443
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
267
444
|
get activeRealtimeWorkerCount() {
|
|
268
445
|
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
269
446
|
(this.activeFastRealtime === null ? 0 : 1) +
|
|
270
447
|
this.realtimeOccupants.size);
|
|
271
448
|
}
|
|
449
|
+
get diagnosticAdmissionSnapshot() {
|
|
450
|
+
const active = [
|
|
451
|
+
this.active,
|
|
452
|
+
this.activeFastRealtime,
|
|
453
|
+
...this.realtimeOccupants,
|
|
454
|
+
this.detachedChore,
|
|
455
|
+
].filter((entry) => entry !== null);
|
|
456
|
+
const holders = active.map((entry) => ({
|
|
457
|
+
entryId: entry.entryId,
|
|
458
|
+
kind: entry.transferDirection === "pull"
|
|
459
|
+
? "pull"
|
|
460
|
+
: entry.workClass === "chore"
|
|
461
|
+
? "chore"
|
|
462
|
+
: "push",
|
|
463
|
+
scope: entry.label.slice(0, 160),
|
|
464
|
+
ageMs: entry.startedAtMonotonicMs === undefined
|
|
465
|
+
? 0
|
|
466
|
+
: Math.max(0, performance.now() - entry.startedAtMonotonicMs),
|
|
467
|
+
}));
|
|
468
|
+
return {
|
|
469
|
+
queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
|
|
470
|
+
holders: holders.slice(0, 20),
|
|
471
|
+
lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
|
|
472
|
+
};
|
|
473
|
+
}
|
|
272
474
|
pendingLaneCount(lane) {
|
|
273
475
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
274
476
|
}
|
|
@@ -283,6 +485,7 @@ export class CooperativePassScheduler {
|
|
|
283
485
|
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
284
486
|
}
|
|
285
487
|
const directEntry = {
|
|
488
|
+
entryId: `guarded-${++this.nextEntryId}`,
|
|
286
489
|
task,
|
|
287
490
|
workClass,
|
|
288
491
|
label,
|
|
@@ -306,7 +509,11 @@ export class CooperativePassScheduler {
|
|
|
306
509
|
if (workClass === "realtime" &&
|
|
307
510
|
this._handoffActive &&
|
|
308
511
|
directEntry.allowConcurrentHandoff &&
|
|
309
|
-
this.canAdmitRealtimeOccupant(directEntry)
|
|
512
|
+
this.canAdmitRealtimeOccupant(directEntry) &&
|
|
513
|
+
!(directEntry.lane === "fast" && this.consecutiveFastAdmissions > 0 &&
|
|
514
|
+
this.pending.some((pending) => pending.workClass === "realtime" &&
|
|
515
|
+
pending.lane === "slow" && pending.transferDirection === directEntry.transferDirection)) &&
|
|
516
|
+
!this.pullMustYieldForWaitingPush(directEntry)) {
|
|
310
517
|
return new Promise((resolve, reject) => {
|
|
311
518
|
const entry = {
|
|
312
519
|
...directEntry,
|
|
@@ -455,16 +662,21 @@ export class CooperativePassScheduler {
|
|
|
455
662
|
this.admitHandoffBacklog();
|
|
456
663
|
return;
|
|
457
664
|
}
|
|
458
|
-
if (this.active !== null) {
|
|
665
|
+
if (this.active !== null || this.activeFastRealtime !== null || this.realtimeOccupants.size > 0) {
|
|
459
666
|
const fastRealtime = this.nextFastRealtime();
|
|
460
|
-
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
667
|
+
if (this.active !== null && fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
461
668
|
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
462
669
|
this.startFastRealtime(fastRealtime);
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
|
|
673
|
+
if (peerIndex >= 0) {
|
|
674
|
+
const peer = this.pending.splice(peerIndex, 1)[0];
|
|
675
|
+
if (peer)
|
|
676
|
+
this.startRealtimeOccupant(peer);
|
|
463
677
|
}
|
|
464
678
|
return;
|
|
465
679
|
}
|
|
466
|
-
if (this.activeFastRealtime !== null)
|
|
467
|
-
return;
|
|
468
680
|
const next = this.takeNext();
|
|
469
681
|
if (next)
|
|
470
682
|
this.start(next);
|
|
@@ -483,30 +695,116 @@ export class CooperativePassScheduler {
|
|
|
483
695
|
return this.pending.splice(realtime, 1)[0];
|
|
484
696
|
return this.pending.shift();
|
|
485
697
|
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
698
|
+
pullMustYieldForWaitingPush(entry) {
|
|
699
|
+
return entry.transferDirection === "pull" &&
|
|
700
|
+
this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH &&
|
|
701
|
+
this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
|
|
702
|
+
}
|
|
703
|
+
nextRealtimeIndex(eligible = () => true) {
|
|
704
|
+
let firstRealtime = -1;
|
|
705
|
+
let waitingPush = -1;
|
|
706
|
+
let slowPull = -1;
|
|
707
|
+
let fastPush = -1;
|
|
708
|
+
let slowPush = -1;
|
|
709
|
+
let firstFastLane = -1;
|
|
710
|
+
let firstSlowLane = -1;
|
|
711
|
+
for (let index = 0; index < this.pending.length; index += 1) {
|
|
712
|
+
const entry = this.pending[index];
|
|
713
|
+
if (!entry || entry.workClass !== "realtime" || !eligible(entry))
|
|
714
|
+
continue;
|
|
715
|
+
if (firstRealtime < 0)
|
|
716
|
+
firstRealtime = index;
|
|
717
|
+
if (entry.transferDirection === undefined) {
|
|
718
|
+
if (entry.lane === "fast" && firstFastLane < 0)
|
|
719
|
+
firstFastLane = index;
|
|
720
|
+
if (entry.lane === "slow" && firstSlowLane < 0)
|
|
721
|
+
firstSlowLane = index;
|
|
722
|
+
}
|
|
723
|
+
if (entry.transferDirection === "push" && waitingPush < 0)
|
|
724
|
+
waitingPush = index;
|
|
725
|
+
if (entry.transferDirection === "pull" && entry.lane === "slow" && slowPull < 0) {
|
|
726
|
+
slowPull = index;
|
|
727
|
+
}
|
|
728
|
+
if (entry.transferDirection === "push" && entry.lane === "fast" && fastPush < 0) {
|
|
729
|
+
fastPush = index;
|
|
730
|
+
}
|
|
731
|
+
if (entry.transferDirection === "push" && entry.lane === "slow" && slowPush < 0) {
|
|
732
|
+
slowPush = index;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
|
|
736
|
+
this.consecutivePullAdmissions = 0;
|
|
737
|
+
this.consecutiveFastAdmissions = 0;
|
|
738
|
+
return waitingPush;
|
|
739
|
+
}
|
|
740
|
+
// A waiting slow pull is the receiver's progress boundary and retains its
|
|
741
|
+
// bounded turn ahead of pushes. For pushes, let the oldest fast batch take
|
|
742
|
+
// one turn before a queued bulk batch, then give the bulk batch a turn
|
|
743
|
+
// before admitting another fast batch. This prevents bulk arrivals from
|
|
744
|
+
// starving an older interactive write and prevents a stream of fast writes
|
|
745
|
+
// from starving a queued bulk upload.
|
|
746
|
+
const next = slowPull >= 0
|
|
747
|
+
? slowPull
|
|
748
|
+
: fastPush >= 0 && slowPush >= 0
|
|
749
|
+
? this.consecutiveFastAdmissions === 0 ? fastPush : slowPush
|
|
750
|
+
: firstFastLane >= 0 && firstSlowLane >= 0
|
|
751
|
+
? this.consecutiveFastAdmissions === 0 ? firstFastLane : firstSlowLane
|
|
752
|
+
: firstRealtime;
|
|
753
|
+
if (next >= 0) {
|
|
754
|
+
const selected = this.pending[next];
|
|
755
|
+
if (selected?.transferDirection === "pull")
|
|
756
|
+
this.consecutivePullAdmissions += 1;
|
|
757
|
+
else if (selected?.transferDirection === "push")
|
|
758
|
+
this.consecutivePullAdmissions = 0;
|
|
759
|
+
if (selected?.transferDirection === "push" && selected.lane === "fast" && slowPush >= 0) {
|
|
760
|
+
this.consecutiveFastAdmissions += 1;
|
|
761
|
+
}
|
|
762
|
+
else if (selected?.transferDirection === "push") {
|
|
763
|
+
this.consecutiveFastAdmissions = 0;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
return next;
|
|
491
767
|
}
|
|
492
768
|
nextFastRealtime() {
|
|
493
|
-
|
|
769
|
+
const fast = this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
770
|
+
if (!fast)
|
|
771
|
+
return undefined;
|
|
772
|
+
const slowOfSameDirectionWaiting = this.pending.some((entry) => entry.workClass === "realtime" && entry.lane === "slow" &&
|
|
773
|
+
entry.transferDirection === fast.transferDirection);
|
|
774
|
+
if (slowOfSameDirectionWaiting && this.consecutiveFastAdmissions > 0)
|
|
775
|
+
return undefined;
|
|
776
|
+
return fast;
|
|
494
777
|
}
|
|
495
778
|
hasStarvedChore() {
|
|
496
779
|
const now = this.now();
|
|
497
780
|
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
498
781
|
}
|
|
499
782
|
canStartFastRealtime(entry) {
|
|
783
|
+
const slowOfSameDirectionWaiting = this.pending.some((pending) => pending.workClass === "realtime" && pending.lane === "slow" &&
|
|
784
|
+
pending.transferDirection === entry.transferDirection);
|
|
500
785
|
return (!this._handoffActive &&
|
|
501
786
|
this.activeFastRealtime === null &&
|
|
502
787
|
entry.workClass === "realtime" &&
|
|
503
788
|
entry.lane === "fast" &&
|
|
789
|
+
!(slowOfSameDirectionWaiting && this.consecutiveFastAdmissions > 0) &&
|
|
504
790
|
entry.allowFastRealtimeOvertake &&
|
|
505
791
|
this.hasActiveSlowRealtime &&
|
|
506
792
|
!this.hasStarvedChore());
|
|
507
793
|
}
|
|
794
|
+
/** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
|
|
795
|
+
canAdmitRealtimePeer(entry) {
|
|
796
|
+
if (entry.workClass !== "realtime" ||
|
|
797
|
+
!entry.allowConcurrentHandoff ||
|
|
798
|
+
entry.transferDirection === undefined ||
|
|
799
|
+
this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
800
|
+
return false;
|
|
801
|
+
return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
|
|
802
|
+
active.transferDirection !== undefined &&
|
|
803
|
+
active.transferDirection !== entry.transferDirection);
|
|
804
|
+
}
|
|
508
805
|
start(entry) {
|
|
509
806
|
this.clearAdmission(entry);
|
|
807
|
+
this.recordAdmission(entry);
|
|
510
808
|
entry.startedAt = new Date().toISOString();
|
|
511
809
|
// Holder duration uses the process monotonic clock; do not consume the
|
|
512
810
|
// injected scheduler clock, whose reads define cadence-slot deadlines.
|
|
@@ -545,6 +843,8 @@ export class CooperativePassScheduler {
|
|
|
545
843
|
}
|
|
546
844
|
startFastRealtime(entry) {
|
|
547
845
|
this.clearAdmission(entry);
|
|
846
|
+
this.recordAdmission(entry);
|
|
847
|
+
entry.startedAtMonotonicMs = performance.now();
|
|
548
848
|
this.activeFastRealtime = entry;
|
|
549
849
|
let current;
|
|
550
850
|
try {
|
|
@@ -800,6 +1100,8 @@ export class CooperativePassScheduler {
|
|
|
800
1100
|
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
801
1101
|
startRealtimeOccupant(entry) {
|
|
802
1102
|
this.clearAdmission(entry);
|
|
1103
|
+
this.recordAdmission(entry);
|
|
1104
|
+
entry.startedAtMonotonicMs = performance.now();
|
|
803
1105
|
this.realtimeOccupants.add(entry);
|
|
804
1106
|
this.handoffAdmitted = true;
|
|
805
1107
|
if (this.capYieldDepth > 0)
|
|
@@ -814,6 +1116,7 @@ export class CooperativePassScheduler {
|
|
|
814
1116
|
this.handoffBacklog = null;
|
|
815
1117
|
entry.reject(error);
|
|
816
1118
|
this.notifyOccupantChange();
|
|
1119
|
+
this.drain();
|
|
817
1120
|
return;
|
|
818
1121
|
}
|
|
819
1122
|
void running.then((outcome) => {
|
|
@@ -823,6 +1126,7 @@ export class CooperativePassScheduler {
|
|
|
823
1126
|
entry.resolve(outcome);
|
|
824
1127
|
this.promoteOccupancyArrivals();
|
|
825
1128
|
this.notifyOccupantChange();
|
|
1129
|
+
this.drain();
|
|
826
1130
|
}, (error) => {
|
|
827
1131
|
this.realtimeOccupants.delete(entry);
|
|
828
1132
|
if (this.handoffBacklog === entry)
|
|
@@ -830,6 +1134,7 @@ export class CooperativePassScheduler {
|
|
|
830
1134
|
entry.reject(error);
|
|
831
1135
|
this.promoteOccupancyArrivals();
|
|
832
1136
|
this.notifyOccupantChange();
|
|
1137
|
+
this.drain();
|
|
833
1138
|
});
|
|
834
1139
|
}
|
|
835
1140
|
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
@@ -838,8 +1143,7 @@ export class CooperativePassScheduler {
|
|
|
838
1143
|
(!this.networkTransferActive && !this.withinHandoffAdmission()))
|
|
839
1144
|
return;
|
|
840
1145
|
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
841
|
-
const index = this.
|
|
842
|
-
pending.occupancyArrival === true &&
|
|
1146
|
+
const index = this.nextRealtimeIndex((pending) => pending.occupancyArrival === true &&
|
|
843
1147
|
pending.allowConcurrentHandoff &&
|
|
844
1148
|
this.canAdmitRealtimeOccupant(pending));
|
|
845
1149
|
if (index < 0)
|
|
@@ -858,7 +1162,7 @@ export class CooperativePassScheduler {
|
|
|
858
1162
|
return;
|
|
859
1163
|
if (!this.withinHandoffAdmission())
|
|
860
1164
|
return;
|
|
861
|
-
const realtimeIndex = this.nextRealtimeIndex();
|
|
1165
|
+
const realtimeIndex = this.nextRealtimeIndex((pending) => !pending.occupancyArrival && this.canAdmitRealtimeOccupant(pending));
|
|
862
1166
|
if (realtimeIndex < 0)
|
|
863
1167
|
return;
|
|
864
1168
|
const pending = this.pending[realtimeIndex];
|
|
@@ -1292,6 +1596,151 @@ export function isChoreLocalPush(paths, largeUpload) {
|
|
|
1292
1596
|
}
|
|
1293
1597
|
return count > 0;
|
|
1294
1598
|
}
|
|
1599
|
+
/** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
|
|
1600
|
+
export function watcherWarmupPathsByRoute(relativePaths) {
|
|
1601
|
+
const counts = new Map();
|
|
1602
|
+
for (const relativePath of relativePaths) {
|
|
1603
|
+
const route = routeChangeToTarget(relativePath);
|
|
1604
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1605
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1606
|
+
}
|
|
1607
|
+
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1608
|
+
const result = {};
|
|
1609
|
+
for (const [key, count] of ranked.slice(0, 20))
|
|
1610
|
+
result[key] = count;
|
|
1611
|
+
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1612
|
+
if (other > 0)
|
|
1613
|
+
result.other = other;
|
|
1614
|
+
return result;
|
|
1615
|
+
}
|
|
1616
|
+
export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
|
|
1617
|
+
const candidatePrefixes = new Map();
|
|
1618
|
+
const candidateCounts = new Map();
|
|
1619
|
+
for (const relativePath of candidateRelativePaths) {
|
|
1620
|
+
const route = routeChangeToTarget(relativePath);
|
|
1621
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1622
|
+
candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
|
|
1623
|
+
const routePath = route?.kind === "company"
|
|
1624
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1625
|
+
: relativePath;
|
|
1626
|
+
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1627
|
+
const routeCounts = candidatePrefixes.get(key) ?? new Map();
|
|
1628
|
+
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1629
|
+
candidatePrefixes.set(key, routeCounts);
|
|
1630
|
+
}
|
|
1631
|
+
const eligibleCounts = new Map();
|
|
1632
|
+
for (const relativePath of eligibleRelativePaths) {
|
|
1633
|
+
const route = routeChangeToTarget(relativePath);
|
|
1634
|
+
const key = route ? routeKey(route) : "unrouted";
|
|
1635
|
+
const routePath = route?.kind === "company"
|
|
1636
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1637
|
+
: relativePath;
|
|
1638
|
+
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1639
|
+
const routeCounts = eligibleCounts.get(key) ?? new Map();
|
|
1640
|
+
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1641
|
+
eligibleCounts.set(key, routeCounts);
|
|
1642
|
+
}
|
|
1643
|
+
const result = {};
|
|
1644
|
+
// Eligible routes are all retained; fill any remaining diagnostic slots
|
|
1645
|
+
// with the largest candidate routes. This keeps alphabetical one-path
|
|
1646
|
+
// stubs from crowding out a large route such as personal.
|
|
1647
|
+
const eligibleRoutes = [...eligibleCounts.keys()];
|
|
1648
|
+
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1649
|
+
const rankedCandidates = [...candidateCounts.entries()]
|
|
1650
|
+
.filter(([route]) => !eligibleCounts.has(route))
|
|
1651
|
+
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1652
|
+
const selectedRoutes = [
|
|
1653
|
+
...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
|
|
1654
|
+
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1655
|
+
];
|
|
1656
|
+
for (const route of selectedRoutes) {
|
|
1657
|
+
const topPrefixes = (counts) => {
|
|
1658
|
+
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1659
|
+
const top = {};
|
|
1660
|
+
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1661
|
+
top[prefix] = count;
|
|
1662
|
+
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1663
|
+
if (other > 0)
|
|
1664
|
+
top.other = other;
|
|
1665
|
+
return top;
|
|
1666
|
+
};
|
|
1667
|
+
result[route] = {
|
|
1668
|
+
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1669
|
+
eligible: topPrefixes(eligibleCounts.get(route)),
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
|
|
1673
|
+
if (otherRoutes.length > 0) {
|
|
1674
|
+
const merged = new Map();
|
|
1675
|
+
for (const [, routeCounts] of otherRoutes) {
|
|
1676
|
+
for (const [prefix, count] of routeCounts)
|
|
1677
|
+
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1678
|
+
}
|
|
1679
|
+
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1680
|
+
}
|
|
1681
|
+
return result;
|
|
1682
|
+
}
|
|
1683
|
+
function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
|
|
1684
|
+
counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
|
|
1685
|
+
const firstSlash = routePath.indexOf("/");
|
|
1686
|
+
const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
|
|
1687
|
+
const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
|
|
1688
|
+
let byPrefix = prefixes.get(routeName);
|
|
1689
|
+
if (!byPrefix) {
|
|
1690
|
+
byPrefix = new Map();
|
|
1691
|
+
prefixes.set(routeName, byPrefix);
|
|
1692
|
+
}
|
|
1693
|
+
byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
|
|
1694
|
+
}
|
|
1695
|
+
function rankedWarmupCounts(counts) {
|
|
1696
|
+
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1697
|
+
const result = {};
|
|
1698
|
+
for (const [key, count] of ranked.slice(0, 20))
|
|
1699
|
+
result[key] = count;
|
|
1700
|
+
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1701
|
+
if (other > 0)
|
|
1702
|
+
result.other = other;
|
|
1703
|
+
return result;
|
|
1704
|
+
}
|
|
1705
|
+
function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
|
|
1706
|
+
const topPrefixes = (counts) => {
|
|
1707
|
+
const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1708
|
+
const top = {};
|
|
1709
|
+
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1710
|
+
top[prefix] = count;
|
|
1711
|
+
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1712
|
+
if (other > 0)
|
|
1713
|
+
top.other = other;
|
|
1714
|
+
return top;
|
|
1715
|
+
};
|
|
1716
|
+
const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
|
|
1717
|
+
const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
|
|
1718
|
+
const rankedCandidates = [...candidateCounts.entries()]
|
|
1719
|
+
.filter(([route]) => !eligibleCounts.has(route))
|
|
1720
|
+
.sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1721
|
+
const selectedRoutes = [
|
|
1722
|
+
...eligibleRoutes,
|
|
1723
|
+
...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
|
|
1724
|
+
];
|
|
1725
|
+
const result = {};
|
|
1726
|
+
for (const route of selectedRoutes) {
|
|
1727
|
+
result[route] = {
|
|
1728
|
+
candidate: topPrefixes(candidatePrefixes.get(route)),
|
|
1729
|
+
eligible: topPrefixes(eligiblePrefixes.get(route)),
|
|
1730
|
+
};
|
|
1731
|
+
}
|
|
1732
|
+
const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
|
|
1733
|
+
if (otherRoutes.length > 0) {
|
|
1734
|
+
const merged = new Map();
|
|
1735
|
+
for (const route of otherRoutes) {
|
|
1736
|
+
for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
|
|
1737
|
+
merged.set(prefix, (merged.get(prefix) ?? 0) + count);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
|
|
1741
|
+
}
|
|
1742
|
+
return result;
|
|
1743
|
+
}
|
|
1295
1744
|
/** A receiver event needs a known small upsert size before it can go realtime. */
|
|
1296
1745
|
export function isFastReceiverBatch(eventCount) {
|
|
1297
1746
|
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
@@ -1762,9 +2211,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1762
2211
|
// RUNNER_HEAP_RECYCLE_EXIT through the normal shutdown()/finally teardown.
|
|
1763
2212
|
let recycleRequested = null;
|
|
1764
2213
|
const pendingAppliedUpserts = new Map();
|
|
2214
|
+
const appliedUpsertHashesForWatcher = new Map();
|
|
2215
|
+
const APPLIED_UPSERT_WATCHER_HASH_CAP = 10_000;
|
|
1765
2216
|
const pendingAppliedDeletes = new Set();
|
|
2217
|
+
const uploadedAwaitingPublish = new Set();
|
|
2218
|
+
const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
|
|
2219
|
+
const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
|
|
2220
|
+
let uploadedAwaitingPublishDropLogged = false;
|
|
1766
2221
|
const rememberAppliedUpsert = (relativePath, contentHash) => {
|
|
1767
2222
|
pendingAppliedDeletes.delete(relativePath);
|
|
2223
|
+
appliedUpsertHashesForWatcher.delete(relativePath);
|
|
2224
|
+
appliedUpsertHashesForWatcher.set(relativePath, contentHash);
|
|
2225
|
+
while (appliedUpsertHashesForWatcher.size > APPLIED_UPSERT_WATCHER_HASH_CAP) {
|
|
2226
|
+
const oldest = appliedUpsertHashesForWatcher.keys().next().value;
|
|
2227
|
+
if (oldest === undefined)
|
|
2228
|
+
break;
|
|
2229
|
+
appliedUpsertHashesForWatcher.delete(oldest);
|
|
2230
|
+
}
|
|
1768
2231
|
if (eventSyncLive && eventSync) {
|
|
1769
2232
|
eventSync.markContentPublished?.(relativePath, contentHash);
|
|
1770
2233
|
return;
|
|
@@ -1779,6 +2242,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1779
2242
|
}
|
|
1780
2243
|
};
|
|
1781
2244
|
const rememberAppliedDelete = (relativePath) => {
|
|
2245
|
+
appliedUpsertHashesForWatcher.delete(relativePath);
|
|
1782
2246
|
pendingAppliedUpserts.delete(relativePath);
|
|
1783
2247
|
if (eventSyncLive && eventSync) {
|
|
1784
2248
|
eventSync.markContentDeleted?.(relativePath);
|
|
@@ -1793,6 +2257,65 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1793
2257
|
pendingAppliedDeletes.delete(oldest);
|
|
1794
2258
|
}
|
|
1795
2259
|
};
|
|
2260
|
+
const isUnchangedAppliedUpsert = async (relativePath) => {
|
|
2261
|
+
const expectedHash = appliedUpsertHashesForWatcher.get(relativePath);
|
|
2262
|
+
if (!expectedHash)
|
|
2263
|
+
return false;
|
|
2264
|
+
// Receiver-applied hashes suppress only the first watcher observation.
|
|
2265
|
+
// A later local edit may legitimately revert to this content.
|
|
2266
|
+
appliedUpsertHashesForWatcher.delete(relativePath);
|
|
2267
|
+
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
2268
|
+
try {
|
|
2269
|
+
const stat = await fs.promises.lstat(absolutePath);
|
|
2270
|
+
let actualHash;
|
|
2271
|
+
if (stat.isSymbolicLink()) {
|
|
2272
|
+
actualHash = `sha256:${hashSymlinkTarget(await fs.promises.readlink(absolutePath))}`;
|
|
2273
|
+
}
|
|
2274
|
+
else if (stat.isFile()) {
|
|
2275
|
+
const digest = createHash("sha256");
|
|
2276
|
+
await new Promise((resolve, reject) => {
|
|
2277
|
+
const input = fs.createReadStream(absolutePath);
|
|
2278
|
+
input.on("data", (chunk) => digest.update(chunk));
|
|
2279
|
+
input.once("error", reject);
|
|
2280
|
+
input.once("end", resolve);
|
|
2281
|
+
});
|
|
2282
|
+
actualHash = `sha256:${digest.digest("hex")}`;
|
|
2283
|
+
}
|
|
2284
|
+
else {
|
|
2285
|
+
return false;
|
|
2286
|
+
}
|
|
2287
|
+
return actualHash === expectedHash;
|
|
2288
|
+
}
|
|
2289
|
+
catch (error) {
|
|
2290
|
+
const code = error && typeof error === "object" && "code" in error
|
|
2291
|
+
? error.code
|
|
2292
|
+
: undefined;
|
|
2293
|
+
if (code !== "ENOENT") {
|
|
2294
|
+
process.stderr.write(`${JSON.stringify({
|
|
2295
|
+
type: "watcher-applied-upsert-hash-check-failed",
|
|
2296
|
+
errorClass: pathFreeErrorClass(error) ?? "UnknownError",
|
|
2297
|
+
})}\n`);
|
|
2298
|
+
}
|
|
2299
|
+
return false;
|
|
2300
|
+
}
|
|
2301
|
+
};
|
|
2302
|
+
const flushPendingUploadEvents = () => {
|
|
2303
|
+
if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
|
|
2304
|
+
return;
|
|
2305
|
+
while (uploadedAwaitingPublish.size > 0) {
|
|
2306
|
+
const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
|
|
2307
|
+
const paths = new Map(uploadedPaths.map((relativePath) => [
|
|
2308
|
+
localPathForVaultKey(hqRoot, relativePath),
|
|
2309
|
+
relativePath,
|
|
2310
|
+
]));
|
|
2311
|
+
const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
|
|
2312
|
+
if (expanded.size > 0)
|
|
2313
|
+
eventSync.publishBatch({ paths: expanded });
|
|
2314
|
+
for (const relativePath of uploadedPaths) {
|
|
2315
|
+
uploadedAwaitingPublish.delete(relativePath);
|
|
2316
|
+
}
|
|
2317
|
+
}
|
|
2318
|
+
};
|
|
1796
2319
|
const flushPendingAppliedUpserts = () => {
|
|
1797
2320
|
if (!eventSyncLive || !eventSync)
|
|
1798
2321
|
return;
|
|
@@ -1804,6 +2327,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1804
2327
|
eventSync.markContentDeleted?.(relativePath);
|
|
1805
2328
|
}
|
|
1806
2329
|
pendingAppliedDeletes.clear();
|
|
2330
|
+
flushPendingUploadEvents();
|
|
1807
2331
|
};
|
|
1808
2332
|
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
1809
2333
|
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
@@ -1853,8 +2377,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1853
2377
|
}, 0);
|
|
1854
2378
|
refresh.unref?.();
|
|
1855
2379
|
}
|
|
1856
|
-
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
1857
|
-
|
|
2380
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
|
|
2381
|
+
caps: fullPassCaps,
|
|
2382
|
+
onAdmission: (event) => {
|
|
2383
|
+
process.stderr.write(`${JSON.stringify({
|
|
2384
|
+
type: "watcher-push-scheduler-admission",
|
|
2385
|
+
diagnostic: true,
|
|
2386
|
+
...event,
|
|
2387
|
+
})}\n`);
|
|
2388
|
+
},
|
|
2389
|
+
});
|
|
2390
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
|
|
1858
2391
|
const passCheckpoint = pullSignal || receiverSignal
|
|
1859
2392
|
? Object.assign(async () => checkpoint?.(), {
|
|
1860
2393
|
shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
|
|
@@ -1884,21 +2417,50 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1884
2417
|
}
|
|
1885
2418
|
};
|
|
1886
2419
|
const invokePass = () => {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
2420
|
+
watcherPushStage?.("pass invocation");
|
|
2421
|
+
activePassCount += 1;
|
|
2422
|
+
emitSyncWorkState(true);
|
|
2423
|
+
const trackPass = (pass) => {
|
|
2424
|
+
const complete = () => {
|
|
2425
|
+
watcherPushStage?.("pass return");
|
|
2426
|
+
activePassCount = Math.max(0, activePassCount - 1);
|
|
2427
|
+
emitSyncWorkState(true);
|
|
2428
|
+
};
|
|
2429
|
+
// Observe settlement without returning a chained promise to the pass
|
|
2430
|
+
// scheduler. The scheduler's synchronous test seam and watcher wake
|
|
2431
|
+
// ordering depend on the original pass promise's completion turn.
|
|
2432
|
+
void pass.then(complete, complete);
|
|
2433
|
+
return pass;
|
|
2434
|
+
};
|
|
2435
|
+
try {
|
|
2436
|
+
if (!runPassSupportsCooperativeCheckpoint) {
|
|
2437
|
+
if (!runPassSupportsQmdLifecycle)
|
|
2438
|
+
return trackPass(runPass(passArgvForRun));
|
|
2439
|
+
return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
|
|
2440
|
+
}
|
|
2441
|
+
// This callback is emitted by the production runner after journal
|
|
2442
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
2443
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
2444
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
2445
|
+
const onJournalsReady = () => activateEventPushSurfaces();
|
|
2446
|
+
if (!runPassSupportsQmdLifecycle) {
|
|
2447
|
+
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
|
|
2448
|
+
}
|
|
2449
|
+
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
|
|
2450
|
+
}
|
|
2451
|
+
catch (error) {
|
|
2452
|
+
watcherPushStage?.("pass return");
|
|
2453
|
+
activePassCount = Math.max(0, activePassCount - 1);
|
|
2454
|
+
emitSyncWorkState(true);
|
|
2455
|
+
throw error;
|
|
2456
|
+
}
|
|
1901
2457
|
};
|
|
2458
|
+
const preparePass = () => runGuardedPassPreparation(prepare, watcherPushStage, watcherPushStage
|
|
2459
|
+
? (event) => process.stderr.write(`${JSON.stringify({
|
|
2460
|
+
...event,
|
|
2461
|
+
diagnostic: true,
|
|
2462
|
+
})}\n`)
|
|
2463
|
+
: undefined);
|
|
1902
2464
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1903
2465
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
1904
2466
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -1912,6 +2474,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1912
2474
|
return;
|
|
1913
2475
|
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
1914
2476
|
recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
|
|
2477
|
+
rememberUploadedPaths(outcome);
|
|
2478
|
+
if (typeof outcome !== "number") {
|
|
2479
|
+
acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
|
|
2480
|
+
}
|
|
2481
|
+
flushPendingUploadEvents();
|
|
1915
2482
|
};
|
|
1916
2483
|
// Pre-admit boundary observation.
|
|
1917
2484
|
heapCheckpoint?.();
|
|
@@ -1931,9 +2498,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1931
2498
|
try {
|
|
1932
2499
|
telemetry?.mark("network");
|
|
1933
2500
|
const outcome = telemetry ? await telemetry.measureNetwork(async () => {
|
|
1934
|
-
await
|
|
2501
|
+
await preparePass();
|
|
1935
2502
|
return invokePass();
|
|
1936
|
-
}) : await (async () => { await
|
|
2503
|
+
}) : await (async () => { await preparePass(); return invokePass(); })();
|
|
1937
2504
|
recordAppliedUpserts(outcome);
|
|
1938
2505
|
return outcome;
|
|
1939
2506
|
}
|
|
@@ -1948,9 +2515,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1948
2515
|
try {
|
|
1949
2516
|
telemetry?.mark("network");
|
|
1950
2517
|
const outcome = telemetry ? await telemetry.measureNetwork(async () => {
|
|
1951
|
-
await
|
|
2518
|
+
await preparePass();
|
|
1952
2519
|
return invokePass();
|
|
1953
|
-
}) : await (async () => { await
|
|
2520
|
+
}) : await (async () => { await preparePass(); return invokePass(); })();
|
|
1954
2521
|
recordAppliedUpserts(outcome);
|
|
1955
2522
|
return outcome;
|
|
1956
2523
|
}
|
|
@@ -1972,9 +2539,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1972
2539
|
acquireOperationLockUser();
|
|
1973
2540
|
}
|
|
1974
2541
|
const outcome = telemetry ? await telemetry.measureNetwork(async () => {
|
|
1975
|
-
await
|
|
2542
|
+
await preparePass();
|
|
1976
2543
|
return invokePass();
|
|
1977
|
-
}) : await (async () => { await
|
|
2544
|
+
}) : await (async () => { await preparePass(); return invokePass(); })();
|
|
1978
2545
|
recordAppliedUpserts(outcome);
|
|
1979
2546
|
return outcome;
|
|
1980
2547
|
}
|
|
@@ -2010,9 +2577,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2010
2577
|
}
|
|
2011
2578
|
try {
|
|
2012
2579
|
const outcome = telemetry ? await telemetry.measureNetwork(async () => {
|
|
2013
|
-
await
|
|
2580
|
+
await preparePass();
|
|
2014
2581
|
return invokePass();
|
|
2015
|
-
}) : await (async () => { await
|
|
2582
|
+
}) : await (async () => { await preparePass(); return invokePass(); })();
|
|
2016
2583
|
recordAppliedUpserts(outcome);
|
|
2017
2584
|
return outcome;
|
|
2018
2585
|
}
|
|
@@ -2162,15 +2729,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2162
2729
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2163
2730
|
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
|
|
2164
2731
|
};
|
|
2165
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
|
|
2732
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass, watcherPushStage) => {
|
|
2733
|
+
if (watcherPushStage) {
|
|
2734
|
+
watcherPushStage("guarded scheduler admission requested");
|
|
2735
|
+
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2736
|
+
if (passScheduler.isInCapYield)
|
|
2737
|
+
watcherPushStage("CPU/heap cap wait");
|
|
2738
|
+
else if (admission.holders.some((holder) => holder.kind === "pull")) {
|
|
2739
|
+
watcherPushStage("pull-yield wait");
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2166
2742
|
telemetry?.mark("operationGate");
|
|
2167
2743
|
const admissionSignal = signal && receiverSignal
|
|
2168
2744
|
? AbortSignal.any([signal, receiverSignal])
|
|
2169
2745
|
: signal ?? receiverSignal;
|
|
2170
2746
|
return runGuardedTask((context) => {
|
|
2747
|
+
watcherPushStage?.("guarded scheduler admission granted");
|
|
2171
2748
|
telemetry?.mark("appendLock");
|
|
2172
2749
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2173
|
-
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2750
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal, watcherPushStage);
|
|
2174
2751
|
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
|
|
2175
2752
|
};
|
|
2176
2753
|
/*
|
|
@@ -2206,11 +2783,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2206
2783
|
}
|
|
2207
2784
|
const pathOutcome = coordinatorPathOutcome(passExitCode(execution.result));
|
|
2208
2785
|
return {
|
|
2209
|
-
paths: slice.paths.map((item) => (
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2786
|
+
paths: slice.paths.map((item) => execution.refusedPaths?.has(item.canonicalPath)
|
|
2787
|
+
? {
|
|
2788
|
+
path: item.canonicalPath,
|
|
2789
|
+
epoch: item.epoch,
|
|
2790
|
+
status: "refused",
|
|
2791
|
+
reason: "remote upsert already present locally",
|
|
2792
|
+
}
|
|
2793
|
+
: {
|
|
2794
|
+
path: item.canonicalPath,
|
|
2795
|
+
epoch: item.epoch,
|
|
2796
|
+
...pathOutcome,
|
|
2797
|
+
}),
|
|
2214
2798
|
};
|
|
2215
2799
|
};
|
|
2216
2800
|
const createCoordinator = () => new DefaultParallelSyncCoordinator({
|
|
@@ -2256,6 +2840,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2256
2840
|
result = await task(slice, telemetry, true);
|
|
2257
2841
|
}
|
|
2258
2842
|
catch (error) {
|
|
2843
|
+
execution.errorClass = pathFreeErrorClass(error);
|
|
2259
2844
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
2260
2845
|
execution.admissionTimeout = error;
|
|
2261
2846
|
}
|
|
@@ -2292,9 +2877,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2292
2877
|
return {
|
|
2293
2878
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2294
2879
|
accepted,
|
|
2880
|
+
yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
|
|
2295
2881
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2296
2882
|
.filter((path) => path.status === "refused")
|
|
2297
2883
|
.map((path) => path.path)),
|
|
2884
|
+
...(execution.errorClass ? { errorClass: execution.errorClass } : {}),
|
|
2298
2885
|
...(admissionTimeout ? { admissionTimeout } : {}),
|
|
2299
2886
|
};
|
|
2300
2887
|
}
|
|
@@ -2489,6 +3076,222 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2489
3076
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
2490
3077
|
let drainedUntrackedDeletes = 0;
|
|
2491
3078
|
const pendingWatcherPaths = new Map();
|
|
3079
|
+
const durableWatcherPaths = new Map();
|
|
3080
|
+
const durableWatcherPathLimit = 10_000;
|
|
3081
|
+
let durableWatcherOverflowed = false;
|
|
3082
|
+
let durableWatcherPersistTimer;
|
|
3083
|
+
let durableWatcherPersistQueue = Promise.resolve();
|
|
3084
|
+
let durableWatcherPersistGeneration = 0;
|
|
3085
|
+
let durableWatcherPersistedGeneration = 0;
|
|
3086
|
+
const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
|
|
3087
|
+
const persistDurableWatcherPaths = () => {
|
|
3088
|
+
const generation = durableWatcherPersistGeneration;
|
|
3089
|
+
if (generation === durableWatcherPersistedGeneration)
|
|
3090
|
+
return durableWatcherPersistQueue;
|
|
3091
|
+
const snapshot = durableWatcherOverflowed
|
|
3092
|
+
? { format: 1, hqRoot, overflowed: true }
|
|
3093
|
+
: { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
|
|
3094
|
+
durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
|
|
3095
|
+
try {
|
|
3096
|
+
if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
|
|
3097
|
+
await fs.promises.rm(watcherPushStatePath, { force: true });
|
|
3098
|
+
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
3099
|
+
return;
|
|
3100
|
+
}
|
|
3101
|
+
await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
|
|
3102
|
+
const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
|
|
3103
|
+
await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
|
|
3104
|
+
await fs.promises.rename(tempPath, watcherPushStatePath);
|
|
3105
|
+
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
3106
|
+
}
|
|
3107
|
+
catch (err) {
|
|
3108
|
+
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
3109
|
+
}
|
|
3110
|
+
}).catch((err) => {
|
|
3111
|
+
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
3112
|
+
});
|
|
3113
|
+
return durableWatcherPersistQueue;
|
|
3114
|
+
};
|
|
3115
|
+
const scheduleDurableWatcherPersist = () => {
|
|
3116
|
+
durableWatcherPersistGeneration += 1;
|
|
3117
|
+
if (durableWatcherPersistTimer !== undefined)
|
|
3118
|
+
clearTimeout(durableWatcherPersistTimer);
|
|
3119
|
+
durableWatcherPersistTimer = setTimeout(() => {
|
|
3120
|
+
durableWatcherPersistTimer = undefined;
|
|
3121
|
+
void persistDurableWatcherPaths();
|
|
3122
|
+
}, 1_000);
|
|
3123
|
+
durableWatcherPersistTimer.unref?.();
|
|
3124
|
+
};
|
|
3125
|
+
const addDurableWatcherPath = (absolutePath, relativePath) => {
|
|
3126
|
+
if (durableWatcherOverflowed)
|
|
3127
|
+
return;
|
|
3128
|
+
if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
|
|
3129
|
+
durableWatcherPaths.clear();
|
|
3130
|
+
durableWatcherOverflowed = true;
|
|
3131
|
+
}
|
|
3132
|
+
else if (!durableWatcherOverflowed) {
|
|
3133
|
+
durableWatcherPaths.set(absolutePath, relativePath);
|
|
3134
|
+
}
|
|
3135
|
+
scheduleDurableWatcherPersist();
|
|
3136
|
+
};
|
|
3137
|
+
const clearDurableWatcherOverflow = () => {
|
|
3138
|
+
if (!durableWatcherOverflowed)
|
|
3139
|
+
return;
|
|
3140
|
+
durableWatcherOverflowed = false;
|
|
3141
|
+
durableWatcherPaths.clear();
|
|
3142
|
+
for (const [absolutePath, relativePath] of pendingWatcherPaths) {
|
|
3143
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
3144
|
+
}
|
|
3145
|
+
scheduleDurableWatcherPersist();
|
|
3146
|
+
};
|
|
3147
|
+
const acknowledgeDurableWatcherUploads = (relativePaths) => {
|
|
3148
|
+
let changed = false;
|
|
3149
|
+
const uploaded = new Set(relativePaths);
|
|
3150
|
+
for (const [absolutePath, relativePath] of durableWatcherPaths) {
|
|
3151
|
+
if (!uploaded.has(relativePath))
|
|
3152
|
+
continue;
|
|
3153
|
+
durableWatcherPaths.delete(absolutePath);
|
|
3154
|
+
changed = true;
|
|
3155
|
+
}
|
|
3156
|
+
if (changed)
|
|
3157
|
+
scheduleDurableWatcherPersist();
|
|
3158
|
+
};
|
|
3159
|
+
const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
|
|
3160
|
+
const stillPending = new Set(pendingWatcherPaths.values());
|
|
3161
|
+
acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
|
|
3162
|
+
};
|
|
3163
|
+
let activePassCount = 0;
|
|
3164
|
+
let inFlightWatcherPushes = 0;
|
|
3165
|
+
const watcherPushStages = new Map();
|
|
3166
|
+
let nextWatcherPushGroupId = 0;
|
|
3167
|
+
const setWatcherPushStage = (groupId, stage) => {
|
|
3168
|
+
const record = watcherPushStages.get(groupId);
|
|
3169
|
+
if (!record || record.stage === stage)
|
|
3170
|
+
return;
|
|
3171
|
+
record.stage = stage;
|
|
3172
|
+
record.stageEnteredAt = Date.now();
|
|
3173
|
+
if (stage === "guarded scheduler admission requested") {
|
|
3174
|
+
record.admissionRequestedAtMonotonicMs = performance.now();
|
|
3175
|
+
record.admissionWaiting = true;
|
|
3176
|
+
}
|
|
3177
|
+
else if (stage === "guarded scheduler admission granted") {
|
|
3178
|
+
record.admissionWaiting = false;
|
|
3179
|
+
}
|
|
3180
|
+
process.stderr.write(`${JSON.stringify({
|
|
3181
|
+
type: "watcher-push-group-stage",
|
|
3182
|
+
diagnostic: true,
|
|
3183
|
+
groupId: record.groupId,
|
|
3184
|
+
route: record.route,
|
|
3185
|
+
paths: record.paths,
|
|
3186
|
+
stage: record.stage,
|
|
3187
|
+
stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
|
|
3188
|
+
})}\n`);
|
|
3189
|
+
};
|
|
3190
|
+
const startWatcherPushStageTracking = (route, paths) => {
|
|
3191
|
+
const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
|
|
3192
|
+
watcherPushStages.set(groupId, {
|
|
3193
|
+
groupId,
|
|
3194
|
+
route,
|
|
3195
|
+
paths,
|
|
3196
|
+
stage: "",
|
|
3197
|
+
stageEnteredAt: Date.now(),
|
|
3198
|
+
admissionRequestedAtMonotonicMs: null,
|
|
3199
|
+
admissionWaiting: false,
|
|
3200
|
+
});
|
|
3201
|
+
return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
|
|
3202
|
+
};
|
|
3203
|
+
const emitWatcherPushInflightStages = () => {
|
|
3204
|
+
if (inFlightWatcherPushes <= 0)
|
|
3205
|
+
return;
|
|
3206
|
+
const now = Date.now();
|
|
3207
|
+
const entries = [...watcherPushStages.values()];
|
|
3208
|
+
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
3209
|
+
const coordinatorAdmission = coordinator?.inspect();
|
|
3210
|
+
for (const record of entries) {
|
|
3211
|
+
if (!record.admissionWaiting)
|
|
3212
|
+
continue;
|
|
3213
|
+
const liveStage = resolveWatcherPushWaitStage({
|
|
3214
|
+
capYieldActive: passScheduler.isInCapYield,
|
|
3215
|
+
queueDepth: admission.queueDepth,
|
|
3216
|
+
holders: admission.holders,
|
|
3217
|
+
});
|
|
3218
|
+
if (liveStage !== record.stage)
|
|
3219
|
+
setWatcherPushStage(record.groupId, liveStage);
|
|
3220
|
+
}
|
|
3221
|
+
process.stderr.write(`${JSON.stringify({
|
|
3222
|
+
type: "watcher-push-inflight-stages",
|
|
3223
|
+
diagnostic: true,
|
|
3224
|
+
groups: entries.slice(0, 20).map((record) => {
|
|
3225
|
+
const monotonicNow = performance.now();
|
|
3226
|
+
const current = watcherPushWaitDiagnostics({
|
|
3227
|
+
stage: record.stage,
|
|
3228
|
+
admissionWaiting: record.admissionWaiting,
|
|
3229
|
+
admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
|
|
3230
|
+
lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
|
|
3231
|
+
nowMonotonicMs: monotonicNow,
|
|
3232
|
+
capYieldActive: passScheduler.isInCapYield,
|
|
3233
|
+
queueDepth: admission.queueDepth,
|
|
3234
|
+
holders: admission.holders,
|
|
3235
|
+
});
|
|
3236
|
+
return {
|
|
3237
|
+
route: record.route,
|
|
3238
|
+
paths: record.paths,
|
|
3239
|
+
...current,
|
|
3240
|
+
ageMs: Math.max(0, now - record.stageEnteredAt),
|
|
3241
|
+
};
|
|
3242
|
+
}),
|
|
3243
|
+
other: Math.max(0, entries.length - 20),
|
|
3244
|
+
realtimeAdmissionQueueDepth: admission.queueDepth,
|
|
3245
|
+
holders: admission.holders,
|
|
3246
|
+
coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
|
|
3247
|
+
coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
|
|
3248
|
+
coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
|
|
3249
|
+
fast: null,
|
|
3250
|
+
slow: null,
|
|
3251
|
+
overflowPush: null,
|
|
3252
|
+
},
|
|
3253
|
+
coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
|
|
3254
|
+
coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
|
|
3255
|
+
coordinatorAdmission === undefined
|
|
3256
|
+
? null
|
|
3257
|
+
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
|
|
3258
|
+
coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
|
|
3259
|
+
coordinatorAdmission === undefined
|
|
3260
|
+
? null
|
|
3261
|
+
: Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
|
|
3262
|
+
})}\n`);
|
|
3263
|
+
};
|
|
3264
|
+
const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
|
|
3265
|
+
watcherPushStagesTimer.unref?.();
|
|
3266
|
+
let pendingReceivers = 0;
|
|
3267
|
+
let lastSyncWorkStateEmitAt = 0;
|
|
3268
|
+
let syncWorkStateTimer;
|
|
3269
|
+
const emitSyncWorkState = (force = false) => {
|
|
3270
|
+
const now = Date.now();
|
|
3271
|
+
const elapsed = now - lastSyncWorkStateEmitAt;
|
|
3272
|
+
if (force || elapsed >= 250) {
|
|
3273
|
+
if (syncWorkStateTimer !== undefined)
|
|
3274
|
+
clearTimeout(syncWorkStateTimer);
|
|
3275
|
+
syncWorkStateTimer = undefined;
|
|
3276
|
+
lastSyncWorkStateEmitAt = now;
|
|
3277
|
+
const snapshot = {
|
|
3278
|
+
passActive: activePassCount > 0,
|
|
3279
|
+
queuedWatcherPushes: pendingWatcherPaths.size,
|
|
3280
|
+
inFlightWatcherPushes,
|
|
3281
|
+
pendingReceivers,
|
|
3282
|
+
};
|
|
3283
|
+
deps.onSyncWorkStateSnapshot?.(snapshot);
|
|
3284
|
+
stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
|
|
3285
|
+
return;
|
|
3286
|
+
}
|
|
3287
|
+
if (syncWorkStateTimer !== undefined)
|
|
3288
|
+
return;
|
|
3289
|
+
syncWorkStateTimer = setTimeout(() => {
|
|
3290
|
+
syncWorkStateTimer = undefined;
|
|
3291
|
+
emitSyncWorkState(true);
|
|
3292
|
+
}, Math.max(1, 250 - elapsed));
|
|
3293
|
+
syncWorkStateTimer.unref?.();
|
|
3294
|
+
};
|
|
2492
3295
|
const droppedUnresolvedWatcherSlugs = new Set();
|
|
2493
3296
|
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
2494
3297
|
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
@@ -2531,10 +3334,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2531
3334
|
const changes = new Map();
|
|
2532
3335
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
2533
3336
|
const change = batch.changes?.get(absolutePath);
|
|
2534
|
-
|
|
3337
|
+
let routedRelativePath = relativePath;
|
|
3338
|
+
const absoluteRootRelativePath = path.relative(hqRoot, absolutePath)
|
|
3339
|
+
.split(path.sep)
|
|
3340
|
+
.join("/");
|
|
3341
|
+
if (isSafeRelativePath(absoluteRootRelativePath)) {
|
|
3342
|
+
const absoluteRoute = routeChangeToTarget(absoluteRootRelativePath, change?.kind);
|
|
3343
|
+
const reportedRoute = routeChangeToTarget(relativePath, change?.kind);
|
|
3344
|
+
if (absoluteRoute?.kind === "company" &&
|
|
3345
|
+
(reportedRoute?.kind !== "company" ||
|
|
3346
|
+
reportedRoute.slug.toLowerCase() !== absoluteRoute.slug.toLowerCase())) {
|
|
3347
|
+
routedRelativePath = absoluteRootRelativePath;
|
|
3348
|
+
}
|
|
3349
|
+
}
|
|
3350
|
+
const route = routeChangeToTarget(routedRelativePath, change?.kind);
|
|
2535
3351
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
2536
3352
|
continue;
|
|
2537
|
-
paths.set(absolutePath,
|
|
3353
|
+
paths.set(absolutePath, routedRelativePath);
|
|
2538
3354
|
if (change)
|
|
2539
3355
|
changes.set(absolutePath, change);
|
|
2540
3356
|
}
|
|
@@ -2556,6 +3372,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2556
3372
|
}
|
|
2557
3373
|
}
|
|
2558
3374
|
if (batch) {
|
|
3375
|
+
if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
|
|
3376
|
+
durableWatcherPaths.clear();
|
|
3377
|
+
durableWatcherOverflowed = true;
|
|
3378
|
+
scheduleDurableWatcherPersist();
|
|
3379
|
+
}
|
|
2559
3380
|
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
2560
3381
|
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
2561
3382
|
return previousSequence === undefined ||
|
|
@@ -2564,6 +3385,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2564
3385
|
});
|
|
2565
3386
|
if (entries.length === 0)
|
|
2566
3387
|
return false;
|
|
3388
|
+
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2567
3389
|
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
2568
3390
|
pendingWatcherOriginalBatch =
|
|
2569
3391
|
pendingWatcherPaths.size === 0 &&
|
|
@@ -2575,11 +3397,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2575
3397
|
: null;
|
|
2576
3398
|
for (const [absolutePath, relativePath] of entries) {
|
|
2577
3399
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3400
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
2578
3401
|
const change = batch.changes?.get(absolutePath);
|
|
2579
3402
|
if (change)
|
|
2580
3403
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2581
3404
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2582
3405
|
}
|
|
3406
|
+
emitSyncWorkState(queueWasEmpty);
|
|
2583
3407
|
if (batch.overflowed) {
|
|
2584
3408
|
pendingWatcherOverflowed = true;
|
|
2585
3409
|
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
@@ -2603,16 +3427,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2603
3427
|
// counterpart must be re-derived through the codec — a raw join would
|
|
2604
3428
|
// point at the impossible colon path on Windows.
|
|
2605
3429
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
3430
|
+
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2606
3431
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
3432
|
+
addDurableWatcherPath(absolutePath, changedRelPath);
|
|
2607
3433
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2608
3434
|
// A bare path signal carries no delete intent; drop any change record
|
|
2609
3435
|
// a prior batch left for this path so the stale intent cannot leak
|
|
2610
3436
|
// into the coalesced batch (#227).
|
|
2611
3437
|
pendingWatcherChanges.delete(absolutePath);
|
|
3438
|
+
emitSyncWorkState(queueWasEmpty);
|
|
2612
3439
|
return true;
|
|
2613
3440
|
}
|
|
2614
3441
|
pendingWatcherOriginalBatch = null;
|
|
2615
3442
|
pendingWatcherBareChange = true;
|
|
3443
|
+
emitSyncWorkState();
|
|
2616
3444
|
return true;
|
|
2617
3445
|
};
|
|
2618
3446
|
const takePendingWatcherChange = () => {
|
|
@@ -2657,6 +3485,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2657
3485
|
pendingWatcherDroppedRouteHints.clear();
|
|
2658
3486
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
2659
3487
|
pendingWatcherBatchGeneration = null;
|
|
3488
|
+
emitSyncWorkState();
|
|
2660
3489
|
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
2661
3490
|
};
|
|
2662
3491
|
const restorePendingWatcherPaths = (paths, generation, changes) => {
|
|
@@ -2670,12 +3499,54 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2670
3499
|
if (pendingWatcherChangeSequences.has(absolutePath))
|
|
2671
3500
|
continue;
|
|
2672
3501
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3502
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
2673
3503
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2674
3504
|
const change = changes?.get(absolutePath);
|
|
2675
3505
|
if (change)
|
|
2676
3506
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2677
3507
|
}
|
|
3508
|
+
emitSyncWorkState();
|
|
2678
3509
|
};
|
|
3510
|
+
if (eventPush) {
|
|
3511
|
+
try {
|
|
3512
|
+
const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
|
|
3513
|
+
if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
|
|
3514
|
+
(!Array.isArray(stored.paths) && stored.overflowed !== true)) {
|
|
3515
|
+
throw new Error("stored watcher push state has an unsupported shape or root");
|
|
3516
|
+
}
|
|
3517
|
+
if (stored.overflowed === true) {
|
|
3518
|
+
durableWatcherOverflowed = true;
|
|
3519
|
+
pendingWatcherOverflowed = true;
|
|
3520
|
+
pendingWatcherWarmupCatchup = true;
|
|
3521
|
+
pendingWatcherDroppedRoutesUnknown = true;
|
|
3522
|
+
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3523
|
+
emitSyncWorkState(true);
|
|
3524
|
+
process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
|
|
3525
|
+
}
|
|
3526
|
+
for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
|
|
3527
|
+
if (typeof relativePath !== "string" || relativePath.length === 0)
|
|
3528
|
+
continue;
|
|
3529
|
+
const route = routeChangeToTarget(relativePath);
|
|
3530
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3531
|
+
continue;
|
|
3532
|
+
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
3533
|
+
addDurableWatcherPath(absolutePath, relativePath);
|
|
3534
|
+
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3535
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3536
|
+
}
|
|
3537
|
+
if (pendingWatcherPaths.size > 0) {
|
|
3538
|
+
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3539
|
+
pendingWatcherWarmupCatchup = true;
|
|
3540
|
+
emitSyncWorkState(true);
|
|
3541
|
+
process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
|
|
3542
|
+
}
|
|
3543
|
+
}
|
|
3544
|
+
catch (err) {
|
|
3545
|
+
if (!(err instanceof Error && err.code === "ENOENT")) {
|
|
3546
|
+
process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
|
|
3547
|
+
}
|
|
3548
|
+
}
|
|
3549
|
+
}
|
|
2679
3550
|
const routePushFailureBackoff = new RoutePushFailureBackoff();
|
|
2680
3551
|
const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
|
|
2681
3552
|
const routePushRetryTimerHandles = new Map();
|
|
@@ -3256,14 +4127,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3256
4127
|
const relativePath = route.kind === "personal"
|
|
3257
4128
|
? key
|
|
3258
4129
|
: ["companies", route.slug, key].join("/");
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
if (
|
|
3263
|
-
fromRoot.startsWith(parentPrefix) ||
|
|
3264
|
-
path.isAbsolute(fromRoot)) {
|
|
4130
|
+
// Journal keys are POSIX relative paths. Reject unsafe segments with
|
|
4131
|
+
// a string-only check before building the absolute path; path.resolve
|
|
4132
|
+
// plus path.relative allocated and normalized every journal row.
|
|
4133
|
+
if (!isSafeRelativePath(relativePath))
|
|
3265
4134
|
continue;
|
|
3266
|
-
|
|
4135
|
+
const localPath = path.join(hqRoot, relativePath);
|
|
3267
4136
|
if (shouldSync(localPath)) {
|
|
3268
4137
|
paths.push(relativePath);
|
|
3269
4138
|
continue;
|
|
@@ -3428,36 +4297,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3428
4297
|
}
|
|
3429
4298
|
}
|
|
3430
4299
|
};
|
|
3431
|
-
const groupBatchByRoute = (batch) => {
|
|
3432
|
-
const grouped = new Map();
|
|
3433
|
-
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
3434
|
-
const change = batch.changes?.get(absolutePath);
|
|
3435
|
-
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
3436
|
-
if (!route)
|
|
3437
|
-
continue;
|
|
3438
|
-
const category = isSessionLogsPushOnlyPath(relativePath) ? "session-log-backup" : "interactive";
|
|
3439
|
-
const key = `${routeKey(route)}:${category}`;
|
|
3440
|
-
const existing = grouped.get(key) ??
|
|
3441
|
-
{
|
|
3442
|
-
route,
|
|
3443
|
-
paths: new Map(),
|
|
3444
|
-
changes: new Map(),
|
|
3445
|
-
relPaths: [],
|
|
3446
|
-
deleteScopeRoots: [],
|
|
3447
|
-
};
|
|
3448
|
-
existing.paths.set(absolutePath, relativePath);
|
|
3449
|
-
if (change)
|
|
3450
|
-
existing.changes.set(absolutePath, change);
|
|
3451
|
-
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
3452
|
-
existing.deleteScopeRoots = [
|
|
3453
|
-
...new Set([...existing.changes.values()].flatMap((item) => (item.deleteSnapshots ?? []).flatMap((snapshot) => snapshot.deleteScopeRoot === undefined
|
|
3454
|
-
? []
|
|
3455
|
-
: [snapshot.deleteScopeRoot]))),
|
|
3456
|
-
];
|
|
3457
|
-
grouped.set(key, existing);
|
|
3458
|
-
}
|
|
3459
|
-
return grouped;
|
|
3460
|
-
};
|
|
3461
4300
|
/** Resolve deferred delete snapshots immediately before their scoped drain. */
|
|
3462
4301
|
const resolveDeleteSnapshotCaptures = async (batch) => {
|
|
3463
4302
|
for (const [absolutePath, change] of batch.changes ?? []) {
|
|
@@ -3526,8 +4365,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3526
4365
|
// a file and still be refused as stale (a download changed the directory
|
|
3527
4366
|
// mid-pass); the retry then finds the file already in the cloud and uploads
|
|
3528
4367
|
// nothing, so without this the file is never announced to peers.
|
|
3529
|
-
const uploadedAwaitingPublish = new Set();
|
|
3530
|
-
const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
|
|
3531
4368
|
const rememberUploadedPaths = (result) => {
|
|
3532
4369
|
if (result === undefined || typeof result === "number")
|
|
3533
4370
|
return;
|
|
@@ -3540,6 +4377,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3540
4377
|
const oldest = uploadedAwaitingPublish.values().next().value;
|
|
3541
4378
|
if (oldest !== undefined)
|
|
3542
4379
|
uploadedAwaitingPublish.delete(oldest);
|
|
4380
|
+
if (!uploadedAwaitingPublishDropLogged) {
|
|
4381
|
+
uploadedAwaitingPublishDropLogged = true;
|
|
4382
|
+
process.stderr.write(`${JSON.stringify({
|
|
4383
|
+
type: "event-sync-upload-buffer-overflow",
|
|
4384
|
+
cap: UPLOADED_AWAITING_PUBLISH_CAP,
|
|
4385
|
+
action: "drop-oldest",
|
|
4386
|
+
})}\n`);
|
|
4387
|
+
}
|
|
3543
4388
|
}
|
|
3544
4389
|
}
|
|
3545
4390
|
};
|
|
@@ -3556,8 +4401,37 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3556
4401
|
};
|
|
3557
4402
|
let emitScopedPushFailureTelemetry = null;
|
|
3558
4403
|
const runScopedDrain = async (batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
3559
|
-
if (batch)
|
|
3560
|
-
|
|
4404
|
+
if (batch) {
|
|
4405
|
+
// Read each routed quarantine shard once for this watcher batch. Filter
|
|
4406
|
+
// before resolving delete snapshots or submitting work to the coordinator
|
|
4407
|
+
// so a quarantined key cannot acquire a new upload/delete intent.
|
|
4408
|
+
const pathsByJournal = new Map();
|
|
4409
|
+
let unresolvedCount = 0;
|
|
4410
|
+
for (const [absolutePath, relativePath] of [...batch.paths]) {
|
|
4411
|
+
const kind = batch.changes?.get(absolutePath)?.kind;
|
|
4412
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
4413
|
+
if (!coordinates)
|
|
4414
|
+
continue;
|
|
4415
|
+
let quarantined = pathsByJournal.get(coordinates.slug);
|
|
4416
|
+
if (!quarantined) {
|
|
4417
|
+
quarantined = readQuarantinedJournalPaths(coordinates.slug);
|
|
4418
|
+
pathsByJournal.set(coordinates.slug, quarantined);
|
|
4419
|
+
}
|
|
4420
|
+
if (!quarantined.has(coordinates.key))
|
|
4421
|
+
continue;
|
|
4422
|
+
batch.paths.delete(absolutePath);
|
|
4423
|
+
batch.changes?.delete(absolutePath);
|
|
4424
|
+
unresolvedCount += 1;
|
|
4425
|
+
}
|
|
4426
|
+
if (unresolvedCount > 0) {
|
|
4427
|
+
(deps.stderr ?? process.stderr).write(`${JSON.stringify({
|
|
4428
|
+
type: "journal-quarantine-unresolved",
|
|
4429
|
+
count: unresolvedCount,
|
|
4430
|
+
})}\n`);
|
|
4431
|
+
}
|
|
4432
|
+
if (batch.paths.size > 0)
|
|
4433
|
+
await resolveDeleteSnapshotCaptures(batch);
|
|
4434
|
+
}
|
|
3561
4435
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
3562
4436
|
const skipCompanies = grouped ? resolveSkipCompanies() : null;
|
|
3563
4437
|
const isPausedRoute = (route) => route.kind === "company" &&
|
|
@@ -3582,7 +4456,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3582
4456
|
return routeFailure?.message ?? failures[0]?.message ??
|
|
3583
4457
|
`sync pass exited ${result.exitCode}`;
|
|
3584
4458
|
};
|
|
3585
|
-
const recordPushFailure = (route, exitCode, message) => {
|
|
4459
|
+
const recordPushFailure = (route, exitCode, message, caughtError, caughtErrorClass) => {
|
|
3586
4460
|
const routeId = routeKey(route);
|
|
3587
4461
|
const companyRoot = route.kind === "company"
|
|
3588
4462
|
? path.join(hqRoot, "companies", route.slug)
|
|
@@ -3599,11 +4473,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3599
4473
|
nextRetryDelayMs: state.delayMs,
|
|
3600
4474
|
errorSummary: pushFailureTelemetryLabel(errorText),
|
|
3601
4475
|
};
|
|
3602
|
-
const diagnostic =
|
|
3603
|
-
type: "scoped-push-failure",
|
|
3604
|
-
...event,
|
|
3605
|
-
error: errorText,
|
|
3606
|
-
};
|
|
4476
|
+
const diagnostic = buildScopedPushFailureDiagnostic(event, errorText, caughtError, caughtErrorClass);
|
|
3607
4477
|
(deps.stderr ?? process.stderr).write(`${JSON.stringify(diagnostic)}\n`);
|
|
3608
4478
|
if (deps.emitScopedPushFailureTelemetry) {
|
|
3609
4479
|
deps.emitScopedPushFailureTelemetry(event);
|
|
@@ -3654,19 +4524,86 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3654
4524
|
? localUploadSizeClass(uploadPaths)
|
|
3655
4525
|
: "small";
|
|
3656
4526
|
const largeUpload = sizeClass !== "small";
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
4527
|
+
// With coordinator fencing enabled, even large watcher batches can
|
|
4528
|
+
// use bounded realtime admission. Keeping them as chores would make
|
|
4529
|
+
// them wait behind a long-lived receiver pull's slot indefinitely.
|
|
4530
|
+
const chorePush = isChoreLocalPush(group.relPaths, largeUpload && !parallelCoordinatorEnabled);
|
|
4531
|
+
const { groupId, setStage: watcherPushStage } = startWatcherPushStageTracking(routeId, group.relPaths.length);
|
|
4532
|
+
watcherPushStage("coordinator submit");
|
|
4533
|
+
inFlightWatcherPushes += group.relPaths.length;
|
|
4534
|
+
emitSyncWorkState(true);
|
|
4535
|
+
emitWatcherPushInflightStages();
|
|
4536
|
+
let execution;
|
|
4537
|
+
try {
|
|
4538
|
+
execution = await (parallelCoordinatorEnabled && !chorePush
|
|
4539
|
+
? runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
|
|
4540
|
+
watcherPushStage("worker entry");
|
|
4541
|
+
const candidatePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4542
|
+
const changeByRelativePath = new Map([...group.paths.entries()].map(([absolutePath, relativePath]) => [
|
|
4543
|
+
relativePath,
|
|
4544
|
+
group.changes.get(absolutePath),
|
|
4545
|
+
]));
|
|
4546
|
+
const refusedPaths = new Set();
|
|
4547
|
+
const admittedPaths = [];
|
|
4548
|
+
for (const relativePath of candidatePaths) {
|
|
4549
|
+
const kind = changeByRelativePath.get(relativePath)?.kind;
|
|
4550
|
+
const isUpsert = kind === undefined || kind === "add" || kind === "change";
|
|
4551
|
+
if (isUpsert && await isUnchangedAppliedUpsert(relativePath)) {
|
|
4552
|
+
refusedPaths.add(relativePath);
|
|
4553
|
+
}
|
|
4554
|
+
else {
|
|
4555
|
+
// Admission means this is a local edit (or a delete); it
|
|
4556
|
+
// supersedes any stale receiver echo marker.
|
|
4557
|
+
appliedUpsertHashesForWatcher.delete(relativePath);
|
|
4558
|
+
admittedPaths.push(relativePath);
|
|
4559
|
+
}
|
|
4560
|
+
}
|
|
4561
|
+
if (refusedPaths.size > 0) {
|
|
4562
|
+
const execution = coordinatorExecutions.get(slice.work);
|
|
4563
|
+
if (execution)
|
|
4564
|
+
execution.refusedPaths = refusedPaths;
|
|
4565
|
+
process.stderr.write(`${JSON.stringify({
|
|
4566
|
+
type: "watcher-applied-upsert-echo-suppressed",
|
|
4567
|
+
count: refusedPaths.size,
|
|
4568
|
+
})}\n`);
|
|
4569
|
+
}
|
|
4570
|
+
if (admittedPaths.length === 0) {
|
|
4571
|
+
if (refusedPaths.size > 0 && eventSyncLive && eventSync) {
|
|
4572
|
+
// The emitter's empty-batch outcome is the settlement
|
|
4573
|
+
// signal for this completed echo check. No path is
|
|
4574
|
+
// published, but receivers and diagnostics can still
|
|
4575
|
+
// observe that the watcher drained the batch.
|
|
4576
|
+
eventSync.publishBatch({ paths: new Map() });
|
|
4577
|
+
}
|
|
4578
|
+
return 0;
|
|
4579
|
+
}
|
|
4580
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
4581
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
4582
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
4583
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
4584
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
|
|
4585
|
+
// Watcher-originated pushes may share the guarded
|
|
4586
|
+
// realtime slot with an in-flight pull. Coordinator
|
|
4587
|
+
// fencing already serializes overlapping path epochs;
|
|
4588
|
+
// this must not depend on the optional full-pass flag.
|
|
4589
|
+
true, undefined, routeId, watcherPushStage);
|
|
4590
|
+
}, sizeClass)
|
|
4591
|
+
: (() => {
|
|
4592
|
+
watcherPushStage("worker entry");
|
|
4593
|
+
return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
4594
|
+
? () => prepareWatcherChanges(group.changes)
|
|
4595
|
+
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
|
|
4596
|
+
result,
|
|
4597
|
+
accepted: true,
|
|
4598
|
+
coordinatorRefusedPaths: [],
|
|
4599
|
+
}));
|
|
4600
|
+
})());
|
|
4601
|
+
}
|
|
4602
|
+
finally {
|
|
4603
|
+
watcherPushStages.delete(groupId);
|
|
4604
|
+
inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
|
|
4605
|
+
emitSyncWorkState(true);
|
|
4606
|
+
}
|
|
3670
4607
|
const result = execution.result;
|
|
3671
4608
|
rememberUploadedPaths(result);
|
|
3672
4609
|
const pushExit = passExitCode(result);
|
|
@@ -3677,6 +4614,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3677
4614
|
const refusedPaths = new Set(pushPathResults
|
|
3678
4615
|
.filter((pathResult) => pathResult.status === "refused")
|
|
3679
4616
|
.map((pathResult) => pathResult.relativePath));
|
|
4617
|
+
for (const pathResult of pushPathResults) {
|
|
4618
|
+
if (pathResult.status === "accepted" && pathResult.operation !== "delete") {
|
|
4619
|
+
appliedUpsertHashesForWatcher.delete(pathResult.relativePath);
|
|
4620
|
+
}
|
|
4621
|
+
}
|
|
3680
4622
|
const successfullyDeletedPaths = new Set(pushPathResults
|
|
3681
4623
|
.filter((pathResult) => pathResult.status === "accepted" && pathResult.operation === "delete")
|
|
3682
4624
|
.map((pathResult) => pathResult.relativePath));
|
|
@@ -3685,6 +4627,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3685
4627
|
}
|
|
3686
4628
|
const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
|
|
3687
4629
|
relativePath.startsWith(`${refusedPath}/`));
|
|
4630
|
+
acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
|
|
3688
4631
|
const publishPaths = new Map();
|
|
3689
4632
|
for (const [absolutePath, relativePath] of group.paths) {
|
|
3690
4633
|
if (isRefusedPath(relativePath))
|
|
@@ -3724,16 +4667,21 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3724
4667
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
3725
4668
|
}
|
|
3726
4669
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
4670
|
+
acknowledgeDurableWatcherUploads(group.relPaths);
|
|
3727
4671
|
routePushFailureBackoff.succeeded(routeId);
|
|
3728
4672
|
clearRoutePushRetry(routeId);
|
|
3729
4673
|
if (group.route.kind === "company") {
|
|
4674
|
+
deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
|
|
3730
4675
|
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
3731
4676
|
}
|
|
3732
4677
|
}
|
|
3733
4678
|
else {
|
|
4679
|
+
// A failure before pass invocation (including journal preparation)
|
|
4680
|
+
// has no durable upload outcome. Keep every path pending so the
|
|
4681
|
+
// next scoped drain retries it after the failed coordinator slice.
|
|
3734
4682
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
|
|
3735
4683
|
if (pushExit !== 0) {
|
|
3736
|
-
recordPushFailure(group.route, pushExit, pushFailureText(result, group.route));
|
|
4684
|
+
recordPushFailure(group.route, pushExit, pushFailureText(result, group.route), undefined, execution.errorClass);
|
|
3737
4685
|
}
|
|
3738
4686
|
else if (_wakeOnPushFailure) {
|
|
3739
4687
|
interruptWait();
|
|
@@ -3745,7 +4693,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3745
4693
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
3746
4694
|
continue;
|
|
3747
4695
|
}
|
|
3748
|
-
recordPushFailure(group.route, 1, pushFailureText(1, group.route, err));
|
|
4696
|
+
recordPushFailure(group.route, 1, pushFailureText(1, group.route, err), err);
|
|
3749
4697
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
|
|
3750
4698
|
}
|
|
3751
4699
|
}
|
|
@@ -3788,7 +4736,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3788
4736
|
const execution = parallelCoordinatorEnabled
|
|
3789
4737
|
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
3790
4738
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
3791
|
-
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
4739
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route));
|
|
3792
4740
|
})
|
|
3793
4741
|
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
3794
4742
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
@@ -3840,6 +4788,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3840
4788
|
addRoute(hint);
|
|
3841
4789
|
const skipCompanies = resolveSkipCompanies();
|
|
3842
4790
|
if (parallelCoordinatorEnabled) {
|
|
4791
|
+
let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
|
|
4792
|
+
// An overflow whose known touched routes are all explicitly disabled
|
|
4793
|
+
// has no chores to enqueue, so no completion callback can clear it.
|
|
4794
|
+
// Clear only when there was at least one known selected route; an empty
|
|
4795
|
+
// route set still means the overflow was not safely attributable here.
|
|
4796
|
+
if (routes.size > 0 && routesRemaining === 0) {
|
|
4797
|
+
clearDurableWatcherOverflow();
|
|
4798
|
+
return 0;
|
|
4799
|
+
}
|
|
3843
4800
|
for (const route of routes.values()) {
|
|
3844
4801
|
if (route.kind === "company" &&
|
|
3845
4802
|
skipCompanies.has(route.slug.toLowerCase())) {
|
|
@@ -3868,6 +4825,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3868
4825
|
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
3869
4826
|
interruptWait();
|
|
3870
4827
|
}
|
|
4828
|
+
else {
|
|
4829
|
+
routesRemaining -= 1;
|
|
4830
|
+
if (routesRemaining === 0)
|
|
4831
|
+
clearDurableWatcherOverflow();
|
|
4832
|
+
}
|
|
3871
4833
|
});
|
|
3872
4834
|
}
|
|
3873
4835
|
return 0;
|
|
@@ -3983,6 +4945,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3983
4945
|
// churn overflowed the 200-path batch and skipped publishBatch.
|
|
3984
4946
|
personalMode: true,
|
|
3985
4947
|
includeCompanyPaths: companiesMode || companyMode,
|
|
4948
|
+
// Inventory filtered candidates so the route's push filter remains
|
|
4949
|
+
// authoritative and can attribute exclusions in warm-up diagnostics.
|
|
4950
|
+
includeFilteredWarmupCandidates: true,
|
|
3986
4951
|
onDegraded: opts.onDegraded,
|
|
3987
4952
|
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
3988
4953
|
queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
|
|
@@ -4104,6 +5069,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4104
5069
|
}
|
|
4105
5070
|
}
|
|
4106
5071
|
watcher.start();
|
|
5072
|
+
let catchupPathStatWarningReported = false;
|
|
4107
5073
|
const reportWatcherReady = () => {
|
|
4108
5074
|
if (stopped || watcher === null)
|
|
4109
5075
|
return;
|
|
@@ -4144,14 +5110,202 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4144
5110
|
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
|
|
4145
5111
|
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
4146
5112
|
return;
|
|
4147
|
-
const
|
|
5113
|
+
const skipStore = deps.watcherCatchupSkipStore;
|
|
5114
|
+
const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
|
|
5115
|
+
const eligiblePaths = new Map();
|
|
5116
|
+
const eligibleChanges = new Map();
|
|
5117
|
+
const excludedByReason = {};
|
|
5118
|
+
const excludedByReasonByRoute = {};
|
|
5119
|
+
const personalFilterExclusions = new Map();
|
|
5120
|
+
const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
|
|
5121
|
+
const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
|
|
5122
|
+
const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
|
|
5123
|
+
const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
|
|
5124
|
+
const clearedResolvedCompanies = new Set();
|
|
5125
|
+
const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
|
|
5126
|
+
const candidateRouteCounts = new Map();
|
|
5127
|
+
const candidateRoutePrefixes = new Map();
|
|
5128
|
+
const eligibleRouteCounts = new Map();
|
|
5129
|
+
const eligibleRoutePrefixes = new Map();
|
|
5130
|
+
const backupClass = (relativePath) => {
|
|
5131
|
+
if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
|
|
5132
|
+
return "attioCleanup";
|
|
5133
|
+
if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
|
|
5134
|
+
return "readModelRollout";
|
|
5135
|
+
return undefined;
|
|
5136
|
+
};
|
|
5137
|
+
const exclude = (reason, routeName) => {
|
|
5138
|
+
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
|
|
5139
|
+
const routeReasons = excludedByReasonByRoute[routeName] ??= {};
|
|
5140
|
+
routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
|
|
5141
|
+
};
|
|
5142
|
+
let filteredCandidateCount = 0;
|
|
5143
|
+
for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
|
|
5144
|
+
filteredCandidateCount += count;
|
|
5145
|
+
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
|
|
5146
|
+
const routeReasons = excludedByReasonByRoute.filtered ??= {};
|
|
5147
|
+
routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
|
|
5148
|
+
}
|
|
5149
|
+
if (filteredCandidateCount > 0)
|
|
5150
|
+
candidateRouteCounts.set("filtered", filteredCandidateCount);
|
|
5151
|
+
let examinedWarmupPaths = 0;
|
|
5152
|
+
for (const [absolutePath, relativePath] of catchup.batch.paths) {
|
|
5153
|
+
examinedWarmupPaths += 1;
|
|
5154
|
+
const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
|
|
5155
|
+
if (warmupYield)
|
|
5156
|
+
await warmupYield;
|
|
5157
|
+
if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
|
|
5158
|
+
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
|
|
5159
|
+
exclude("unsafe-path", "unrouted");
|
|
5160
|
+
continue;
|
|
5161
|
+
}
|
|
5162
|
+
const route = routeChangeToTarget(relativePath);
|
|
5163
|
+
const routeName = route ? routeKey(route) : "unrouted";
|
|
5164
|
+
const summaryRoutePath = route?.kind === "company"
|
|
5165
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
5166
|
+
: relativePath;
|
|
5167
|
+
recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
|
|
5168
|
+
const ontologyBackupClass = backupClass(relativePath);
|
|
5169
|
+
if (ontologyBackupClass) {
|
|
5170
|
+
exclude("local-rule:ontology-backup", routeName);
|
|
5171
|
+
ontologyBackupPaths[ontologyBackupClass] += 1;
|
|
5172
|
+
continue;
|
|
5173
|
+
}
|
|
5174
|
+
if (route?.kind === "company") {
|
|
5175
|
+
const resolvedNow = cloudCompanySlugs.has(route.slug) ||
|
|
5176
|
+
skipStore?.hasCurrentGrantCompany(route.slug) ||
|
|
5177
|
+
skipStore?.hasCurrentGrantVersion(route.slug) ||
|
|
5178
|
+
(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
|
|
5179
|
+
if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
|
|
5180
|
+
skipStore?.clearUnresolvedCompany(route.slug);
|
|
5181
|
+
clearedResolvedCompanies.add(route.slug);
|
|
5182
|
+
}
|
|
5183
|
+
if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
|
|
5184
|
+
exclude("unresolved-company", routeName);
|
|
5185
|
+
continue;
|
|
5186
|
+
}
|
|
5187
|
+
}
|
|
5188
|
+
if (route?.kind === "company" &&
|
|
5189
|
+
!cloudCompanySlugs.has(route.slug) &&
|
|
5190
|
+
!skipStore?.hasCurrentGrantCompany(route.slug) &&
|
|
5191
|
+
!skipStore?.hasCurrentGrantVersion(route.slug) &&
|
|
5192
|
+
!(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
|
|
5193
|
+
exclude("local-only-company", routeName);
|
|
5194
|
+
continue;
|
|
5195
|
+
}
|
|
5196
|
+
const company = route?.kind === "company" ? route.slug : "personal";
|
|
5197
|
+
const routePath = route?.kind === "company"
|
|
5198
|
+
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
5199
|
+
: relativePath;
|
|
5200
|
+
if (isEphemeralPath(routePath)) {
|
|
5201
|
+
exclude("push-filter:conflict-mirror", routeName);
|
|
5202
|
+
continue;
|
|
5203
|
+
}
|
|
5204
|
+
// The production TreeWatcher collector has already applied this
|
|
5205
|
+
// shared filter stack and reports rejected files as reason counts.
|
|
5206
|
+
// Injected/custom collectors without that proof still evaluate the
|
|
5207
|
+
// route filter here, with personal defaults first for `.env` attribution.
|
|
5208
|
+
const personalDefaultExclusion = catchup.pushFiltersApplied === true
|
|
5209
|
+
? undefined
|
|
5210
|
+
: route?.kind === "personal"
|
|
5211
|
+
? matchPersonalVaultExclusion(relativePath, false)
|
|
5212
|
+
: undefined;
|
|
5213
|
+
const isPersonalRoute = route?.kind !== "company";
|
|
5214
|
+
const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
|
|
5215
|
+
? personalPushFilter(absolutePath, false, relativePath)
|
|
5216
|
+
: ignoreFilter(absolutePath, false, relativePath)));
|
|
5217
|
+
if (!acceptedByPushFilter) {
|
|
5218
|
+
const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
|
|
5219
|
+
? personalFilterExclusions.get(relativePath)
|
|
5220
|
+
: undefined);
|
|
5221
|
+
exclude(exclusionId
|
|
5222
|
+
? `push-filter:personal:${exclusionId}`
|
|
5223
|
+
: "push-filter:ignore", routeName);
|
|
5224
|
+
continue;
|
|
5225
|
+
}
|
|
5226
|
+
if (isPersonalRoute &&
|
|
5227
|
+
!shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
|
|
5228
|
+
isPersonalVaultScopeExcluded(relativePath, {
|
|
5229
|
+
continuityPaths,
|
|
5230
|
+
hasAgencyCarveout,
|
|
5231
|
+
hasSessionLogsCarveout,
|
|
5232
|
+
})) {
|
|
5233
|
+
exclude("push-filter:personal:excluded-top-level", routeName);
|
|
5234
|
+
continue;
|
|
5235
|
+
}
|
|
5236
|
+
if (route && isLocallyIneligibleForPush(routePath, {
|
|
5237
|
+
boardClientOwned: true,
|
|
5238
|
+
ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
|
|
5239
|
+
})) {
|
|
5240
|
+
const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
|
|
5241
|
+
exclude(`local-rule:${pathClass}`, routeName);
|
|
5242
|
+
continue;
|
|
5243
|
+
}
|
|
5244
|
+
const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
|
|
5245
|
+
const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
|
|
5246
|
+
try {
|
|
5247
|
+
const stat = !catchup.symlinkPaths || hasSkipRecord
|
|
5248
|
+
? fs.lstatSync(absolutePath)
|
|
5249
|
+
: undefined;
|
|
5250
|
+
if (route && (knownSymlink || stat?.isSymbolicLink())) {
|
|
5251
|
+
const scopeRoot = route.kind === "company"
|
|
5252
|
+
? path.join(hqRoot, "companies", route.slug)
|
|
5253
|
+
: relativePath.startsWith("personal/")
|
|
5254
|
+
? path.join(hqRoot, "personal")
|
|
5255
|
+
: hqRoot;
|
|
5256
|
+
const inspection = inspectLocalSymlinkScope({
|
|
5257
|
+
absoluteLinkPath: absolutePath,
|
|
5258
|
+
linkPath: relativePath,
|
|
5259
|
+
scopeRoot,
|
|
5260
|
+
scope: route.kind === "company"
|
|
5261
|
+
? { kind: "company", companySlug: route.slug }
|
|
5262
|
+
: { kind: "personal" },
|
|
5263
|
+
});
|
|
5264
|
+
if (inspection.refused) {
|
|
5265
|
+
exclude("symlink", routeName);
|
|
5266
|
+
continue;
|
|
5267
|
+
}
|
|
5268
|
+
}
|
|
5269
|
+
if (hasSkipRecord && stat &&
|
|
5270
|
+
skipStore?.matches(company, routePath, stat)) {
|
|
5271
|
+
exclude("record-match", routeName);
|
|
5272
|
+
continue;
|
|
5273
|
+
}
|
|
5274
|
+
}
|
|
5275
|
+
catch (error) {
|
|
5276
|
+
// Fail open: unreadable state or a filesystem race must not suppress a candidate.
|
|
5277
|
+
if (!catchupPathStatWarningReported) {
|
|
5278
|
+
catchupPathStatWarningReported = true;
|
|
5279
|
+
process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
|
|
5280
|
+
}
|
|
5281
|
+
}
|
|
5282
|
+
eligiblePaths.set(absolutePath, relativePath);
|
|
5283
|
+
recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
|
|
5284
|
+
const change = catchup.batch.changes?.get(absolutePath);
|
|
5285
|
+
if (change)
|
|
5286
|
+
eligibleChanges.set(absolutePath, change);
|
|
5287
|
+
}
|
|
5288
|
+
const queued = addPendingWatcherChange(undefined, {
|
|
5289
|
+
paths: eligiblePaths,
|
|
5290
|
+
changes: eligibleChanges,
|
|
5291
|
+
}, {
|
|
4148
5292
|
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
4149
5293
|
});
|
|
4150
5294
|
pendingWatcherWarmupCatchup ||= queued;
|
|
5295
|
+
const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
|
|
4151
5296
|
process.stderr.write(JSON.stringify({
|
|
4152
5297
|
type: "watcher-warmup-catchup",
|
|
4153
5298
|
reason: "watch-set-ready",
|
|
4154
|
-
pathsFound:
|
|
5299
|
+
pathsFound: eligiblePaths.size,
|
|
5300
|
+
candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
|
|
5301
|
+
pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
|
|
5302
|
+
pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
|
|
5303
|
+
excludedByReason: {
|
|
5304
|
+
...excludedByReason,
|
|
5305
|
+
...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
|
|
5306
|
+
},
|
|
5307
|
+
excludedByReasonByRoute,
|
|
5308
|
+
ontologyBackupPaths,
|
|
4155
5309
|
scannedPaths: catchup.scannedPaths,
|
|
4156
5310
|
elapsedMs: Date.now() - startedAt,
|
|
4157
5311
|
queued,
|
|
@@ -4218,13 +5372,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4218
5372
|
const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
4219
5373
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4220
5374
|
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
4221
|
-
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
5375
|
+
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal, routeKey(route)));
|
|
4222
5376
|
});
|
|
4223
5377
|
const pathCount = events.length;
|
|
4224
5378
|
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
4225
5379
|
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
4226
5380
|
if (execution.admissionTimeout)
|
|
4227
5381
|
throw execution.admissionTimeout;
|
|
5382
|
+
if (execution.yieldedToPush) {
|
|
5383
|
+
throw new RealtimePushYieldError();
|
|
5384
|
+
}
|
|
4228
5385
|
const exitCode = passExitCode(execution.result);
|
|
4229
5386
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
4230
5387
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -4246,74 +5403,82 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4246
5403
|
const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
|
|
4247
5404
|
if (stopped)
|
|
4248
5405
|
return;
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
5406
|
+
try {
|
|
5407
|
+
// The injectable receiver seam and the production SQS receiver both
|
|
5408
|
+
// arrive here only after a successful receive. Keep the cadence tight
|
|
5409
|
+
// until the receiver later reports an empty outer long-poll.
|
|
5410
|
+
onReceiveActivity(true);
|
|
5411
|
+
pendingReceivers += events.length;
|
|
5412
|
+
emitSyncWorkState();
|
|
5413
|
+
// A received wake/high-water signal means there is bounded work now.
|
|
5414
|
+
// It wakes the scheduler but is still funneled through runGuarded below,
|
|
5415
|
+
// so it cannot overlap the pass that may currently own the operation
|
|
5416
|
+
// lock.
|
|
5417
|
+
// Apply the caps once across the whole delivery, then retain the residue
|
|
5418
|
+
// in the normal lane. The receiver already coalesces to one latest event
|
|
5419
|
+
// per path, so the partition cannot split revisions of one path.
|
|
5420
|
+
if (parallelCoordinatorEnabled) {
|
|
5421
|
+
const lane = receiverLaneByBatch.get(events);
|
|
5422
|
+
if (lane) {
|
|
5423
|
+
receiverLaneByBatch.delete(events);
|
|
5424
|
+
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5425
|
+
return;
|
|
5426
|
+
}
|
|
5427
|
+
// Direct embedders may invoke the engine without first going through
|
|
5428
|
+
// SqsPushReceiver. Preserve that seam while applying the same
|
|
5429
|
+
// delivery-wide partition the receiver would expose.
|
|
5430
|
+
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
5431
|
+
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
5432
|
+
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4265
5433
|
return;
|
|
4266
5434
|
}
|
|
4267
|
-
// Direct embedders may invoke the engine without first going through
|
|
4268
|
-
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4269
|
-
// delivery-wide partition the receiver would expose.
|
|
4270
5435
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
5436
|
+
const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
|
|
5437
|
+
const eventsByRoute = new Map();
|
|
5438
|
+
for (const event of laneEvents) {
|
|
5439
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
5440
|
+
// The receiver is an independent ingress from the local watcher.
|
|
5441
|
+
// Apply the same route-selection boundary before it can enqueue a
|
|
5442
|
+
// fast pull: a --skip-personal runner must acknowledge, not pull,
|
|
5443
|
+
// a personal-namespace delivery.
|
|
5444
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
5445
|
+
continue;
|
|
5446
|
+
const key = routeKey(route);
|
|
5447
|
+
const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
|
|
5448
|
+
group.events.push(event);
|
|
5449
|
+
if (event.kind === "upsert")
|
|
5450
|
+
group.upsertPaths.push(event.relativePath);
|
|
5451
|
+
eventsByRoute.set(key, group);
|
|
5452
|
+
}
|
|
5453
|
+
for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
|
|
5454
|
+
const paths = routeEvents.map((event) => event.relativePath);
|
|
5455
|
+
const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
|
|
5456
|
+
const exitCode = passExitCode(result);
|
|
5457
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
5458
|
+
if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
|
|
5459
|
+
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
5460
|
+
}
|
|
5461
|
+
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
5462
|
+
await settleEvents?.(routeEvents);
|
|
5463
|
+
continue;
|
|
5464
|
+
}
|
|
5465
|
+
if (exitCode !== 0) {
|
|
4299
5466
|
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4300
5467
|
}
|
|
4301
|
-
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
4302
5468
|
await settleEvents?.(routeEvents);
|
|
4303
|
-
continue;
|
|
4304
5469
|
}
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
5470
|
+
};
|
|
5471
|
+
// Dispatch realtime work before awaiting its bulk residue. This lets an
|
|
5472
|
+
// established slow pass continue while the small scoped pull uses its
|
|
5473
|
+
// ref-counted lock carve-out; the residue remains serialized and cannot
|
|
5474
|
+
// starve the slow lane.
|
|
5475
|
+
await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
|
|
5476
|
+
await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
|
|
5477
|
+
}
|
|
5478
|
+
finally {
|
|
5479
|
+
pendingReceivers = Math.max(0, pendingReceivers - events.length);
|
|
5480
|
+
emitSyncWorkState();
|
|
5481
|
+
}
|
|
4317
5482
|
};
|
|
4318
5483
|
receiverSyncBatchFn.partition = (events, maxPaths) => {
|
|
4319
5484
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
@@ -4470,6 +5635,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4470
5635
|
if (stopped)
|
|
4471
5636
|
return shutdownPromise ?? Promise.resolve();
|
|
4472
5637
|
stopped = true;
|
|
5638
|
+
clearInterval(watcherPushStagesTimer);
|
|
5639
|
+
if (durableWatcherPersistTimer !== undefined)
|
|
5640
|
+
clearTimeout(durableWatcherPersistTimer);
|
|
5641
|
+
durableWatcherPersistTimer = undefined;
|
|
5642
|
+
if (syncWorkStateTimer !== undefined)
|
|
5643
|
+
clearTimeout(syncWorkStateTimer);
|
|
5644
|
+
syncWorkStateTimer = undefined;
|
|
5645
|
+
emitSyncWorkState(true);
|
|
4473
5646
|
for (const route of routePushRetryTimerHandles.keys()) {
|
|
4474
5647
|
clearRoutePushRetry(route);
|
|
4475
5648
|
}
|
|
@@ -4499,6 +5672,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4499
5672
|
coordinatorDisposal,
|
|
4500
5673
|
boundedEventSyncStartup,
|
|
4501
5674
|
eventSyncDisposal,
|
|
5675
|
+
persistDurableWatcherPaths(),
|
|
4502
5676
|
]).then(() => undefined);
|
|
4503
5677
|
try {
|
|
4504
5678
|
void receiver?.dispose();
|
|
@@ -4916,6 +6090,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4916
6090
|
lastCompletedFullReconcileWallAt = Date.now();
|
|
4917
6091
|
fullPassJournalHealthy = execution.accepted &&
|
|
4918
6092
|
isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
|
|
6093
|
+
if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
|
|
6094
|
+
clearDurableWatcherOverflow();
|
|
6095
|
+
}
|
|
6096
|
+
else if (fullPassJournalHealthy && pending.batch !== null) {
|
|
6097
|
+
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
6098
|
+
...pending.batch.paths.values(),
|
|
6099
|
+
]);
|
|
6100
|
+
}
|
|
4919
6101
|
if (fullPassJournalHealthy) {
|
|
4920
6102
|
fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
|
|
4921
6103
|
}
|
|
@@ -4973,7 +6155,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4973
6155
|
const catchupBatch = workingBatch;
|
|
4974
6156
|
const catchupGeneration = pending.generation;
|
|
4975
6157
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
4976
|
-
if (exit === 0
|
|
6158
|
+
if (exit === 0) {
|
|
6159
|
+
clearDurableWatcherOverflow();
|
|
6160
|
+
return;
|
|
6161
|
+
}
|
|
6162
|
+
if (exit === SCOPED_DRAIN_DEFERRED)
|
|
4977
6163
|
return;
|
|
4978
6164
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
4979
6165
|
interruptWait();
|
|
@@ -5081,6 +6267,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5081
6267
|
? exitCode === 0
|
|
5082
6268
|
: hasCompletedRunnerPass(result));
|
|
5083
6269
|
if (completedPostRepairPass) {
|
|
6270
|
+
if (!parallelCoordinatorEnabled &&
|
|
6271
|
+
pending.batch?.overflowed === true &&
|
|
6272
|
+
((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
|
|
6273
|
+
clearDurableWatcherOverflow();
|
|
6274
|
+
}
|
|
6275
|
+
else if (!parallelCoordinatorEnabled &&
|
|
6276
|
+
ranFullReconcile &&
|
|
6277
|
+
fullPassJournalHealthy &&
|
|
6278
|
+
pending.batch !== null) {
|
|
6279
|
+
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
6280
|
+
...pending.batch.paths.values(),
|
|
6281
|
+
]);
|
|
6282
|
+
}
|
|
5084
6283
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
5085
6284
|
// which follows journal repair. Its numeric seam has the same
|
|
5086
6285
|
// contract: zero means the injected pass completed. This preserves
|