@indigoai-us/hq-cloud 6.18.49-alpha.7 → 6.18.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +3 -5
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +13 -22
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -12
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +112 -675
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -27
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +15 -133
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +6 -1
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +217 -31
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +0 -12
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +4 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +2 -9
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +2 -34
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/outposts/session-host-command.d.ts +13 -0
- package/dist/outposts/session-host-command.d.ts.map +1 -1
- package/dist/outposts/session-host-command.js +34 -17
- package/dist/outposts/session-host-command.js.map +1 -1
- package/dist/outposts/session-host-connection.d.ts +2 -0
- package/dist/outposts/session-host-connection.d.ts.map +1 -1
- package/dist/outposts/session-host-connection.js +15 -0
- package/dist/outposts/session-host-connection.js.map +1 -1
- package/dist/outposts/session-host-server.d.ts +4 -0
- package/dist/outposts/session-host-server.d.ts.map +1 -1
- package/dist/outposts/session-host-server.js +42 -0
- package/dist/outposts/session-host-server.js.map +1 -1
- package/dist/session-log-capture.d.ts.map +1 -1
- package/dist/session-log-capture.js +140 -0
- package/dist/session-log-capture.js.map +1 -1
- package/dist/symlink-target-policy.d.ts +0 -11
- package/dist/symlink-target-policy.d.ts.map +1 -1
- package/dist/symlink-target-policy.js +0 -80
- package/dist/symlink-target-policy.js.map +1 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +12 -0
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +0 -4
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +14 -155
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +20 -6
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +160 -34
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/watcher.d.ts +3 -3
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +7 -15
- package/dist/watcher.js.map +1 -1
- package/package.json +2 -1
- package/dist/bin/watcher-catchup-skip-store.d.ts +0 -59
- package/dist/bin/watcher-catchup-skip-store.d.ts.map +0 -1
- package/dist/bin/watcher-catchup-skip-store.js +0 -278
- package/dist/bin/watcher-catchup-skip-store.js.map +0 -1
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
5
4
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
6
5
|
import * as v8 from "node:v8";
|
|
7
6
|
import { performance } from "node:perf_hooks";
|
|
@@ -16,17 +15,13 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
|
16
15
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
17
16
|
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
18
17
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
19
|
-
import { isEphemeralPath } from "../cli/share.js";
|
|
20
|
-
import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
|
|
21
|
-
import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
|
|
22
18
|
import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
|
|
23
19
|
import { createIgnoreFilter } from "../ignore.js";
|
|
24
|
-
import { wrapFilterWithPersonalVaultDefaults } from "../personal-vault-exclusions.js";
|
|
25
20
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
26
21
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
27
22
|
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
|
|
28
|
-
import { isPersonalVaultDeletionExcluded
|
|
29
|
-
import { NoopPushReceiver, ReceiverDispatchError,
|
|
23
|
+
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
24
|
+
import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
|
|
30
25
|
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
31
26
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
32
27
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
@@ -268,35 +263,12 @@ export class CooperativePassScheduler {
|
|
|
268
263
|
get hasActiveSlowRealtime() {
|
|
269
264
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
270
265
|
}
|
|
271
|
-
get isInCapYield() {
|
|
272
|
-
return this.capYieldDepth > 0;
|
|
273
|
-
}
|
|
274
266
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
275
267
|
get activeRealtimeWorkerCount() {
|
|
276
268
|
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
277
269
|
(this.activeFastRealtime === null ? 0 : 1) +
|
|
278
270
|
this.realtimeOccupants.size);
|
|
279
271
|
}
|
|
280
|
-
get diagnosticAdmissionSnapshot() {
|
|
281
|
-
const active = [
|
|
282
|
-
this.active,
|
|
283
|
-
this.activeFastRealtime,
|
|
284
|
-
...this.realtimeOccupants,
|
|
285
|
-
this.detachedChore,
|
|
286
|
-
].filter((entry) => entry !== null);
|
|
287
|
-
const holders = active.map((entry) => ({
|
|
288
|
-
kind: entry.transferDirection === "pull"
|
|
289
|
-
? "pull"
|
|
290
|
-
: entry.workClass === "chore"
|
|
291
|
-
? "chore"
|
|
292
|
-
: "push",
|
|
293
|
-
scope: entry.label.slice(0, 160),
|
|
294
|
-
}));
|
|
295
|
-
return {
|
|
296
|
-
queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
|
|
297
|
-
holders: holders.slice(0, 20),
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
272
|
pendingLaneCount(lane) {
|
|
301
273
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
302
274
|
}
|
|
@@ -1320,53 +1292,6 @@ export function isChoreLocalPush(paths, largeUpload) {
|
|
|
1320
1292
|
}
|
|
1321
1293
|
return count > 0;
|
|
1322
1294
|
}
|
|
1323
|
-
/** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
|
|
1324
|
-
export function watcherWarmupPathsByRoute(relativePaths) {
|
|
1325
|
-
const counts = new Map();
|
|
1326
|
-
for (const relativePath of relativePaths) {
|
|
1327
|
-
const route = routeChangeToTarget(relativePath);
|
|
1328
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1329
|
-
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1330
|
-
}
|
|
1331
|
-
const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1332
|
-
const result = {};
|
|
1333
|
-
for (const [key, count] of ranked.slice(0, 20))
|
|
1334
|
-
result[key] = count;
|
|
1335
|
-
const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
|
|
1336
|
-
if (other > 0)
|
|
1337
|
-
result.other = other;
|
|
1338
|
-
return result;
|
|
1339
|
-
}
|
|
1340
|
-
export function watcherWarmupPrefixesByRoute(relativePaths) {
|
|
1341
|
-
const counts = new Map();
|
|
1342
|
-
for (const relativePath of relativePaths) {
|
|
1343
|
-
const route = routeChangeToTarget(relativePath);
|
|
1344
|
-
const key = route ? routeKey(route) : "unrouted";
|
|
1345
|
-
const routePath = route?.kind === "company"
|
|
1346
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
1347
|
-
: relativePath;
|
|
1348
|
-
const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
|
|
1349
|
-
const routeCounts = counts.get(key) ?? new Map();
|
|
1350
|
-
routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
|
|
1351
|
-
counts.set(key, routeCounts);
|
|
1352
|
-
}
|
|
1353
|
-
const result = {};
|
|
1354
|
-
for (const [route, routeCounts] of [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)).slice(0, 20)) {
|
|
1355
|
-
const ranked = [...routeCounts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
|
|
1356
|
-
const top = {};
|
|
1357
|
-
for (const [prefix, count] of ranked.slice(0, 5))
|
|
1358
|
-
top[prefix] = count;
|
|
1359
|
-
const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
|
|
1360
|
-
if (other > 0)
|
|
1361
|
-
top.other = other;
|
|
1362
|
-
result[route] = top;
|
|
1363
|
-
}
|
|
1364
|
-
const otherRoutes = [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)).slice(20);
|
|
1365
|
-
if (otherRoutes.length > 0) {
|
|
1366
|
-
result.other = { other: otherRoutes.reduce((sum, [, routeCounts]) => sum + [...routeCounts.values()].reduce((routeSum, count) => routeSum + count, 0), 0) };
|
|
1367
|
-
}
|
|
1368
|
-
return result;
|
|
1369
|
-
}
|
|
1370
1295
|
/** A receiver event needs a known small upsert size before it can go realtime. */
|
|
1371
1296
|
export function isFastReceiverBatch(eventCount) {
|
|
1372
1297
|
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
@@ -1620,9 +1545,14 @@ function unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)
|
|
|
1620
1545
|
receiveCounts.every((count) => count !== undefined && Number.isFinite(count) && count >= 2);
|
|
1621
1546
|
});
|
|
1622
1547
|
}
|
|
1623
|
-
function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
|
|
1548
|
+
export function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
|
|
1624
1549
|
const exitCode = passExitCode(outcome);
|
|
1625
1550
|
const result = typeof outcome === "number" ? undefined : outcome.result;
|
|
1551
|
+
const httpStatus = [
|
|
1552
|
+
...(result?.errors ?? []),
|
|
1553
|
+
...(result?.transient ?? []),
|
|
1554
|
+
...(result?.companyFailures ?? []),
|
|
1555
|
+
].find((failure) => failure.httpStatus !== undefined)?.httpStatus;
|
|
1626
1556
|
const diagnostics = [
|
|
1627
1557
|
...(result?.errors ?? []).map(({ company, message }) => `${company}: ${message}`),
|
|
1628
1558
|
...(result?.transient ?? []).map(({ company, message }) => `${company}: ${message}`),
|
|
@@ -1630,11 +1560,15 @@ function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode)
|
|
|
1630
1560
|
];
|
|
1631
1561
|
const failureReason = `targeted pull failed with exit code ${exitCode}, accepted=${accepted}` +
|
|
1632
1562
|
(diagnostics.length === 0 ? "" : `: ${[...new Set(diagnostics)].join("; ")}`);
|
|
1633
|
-
const
|
|
1634
|
-
|
|
1635
|
-
(result?.transient?.length ?? 0) > 0
|
|
1636
|
-
|
|
1637
|
-
|
|
1563
|
+
const exitKind = exitCode === authRequiredExitCode
|
|
1564
|
+
? "auth_required"
|
|
1565
|
+
: exitCode === TRANSIENT_NETWORK_EXIT || (result?.transient?.length ?? 0) > 0
|
|
1566
|
+
? "transient_network"
|
|
1567
|
+
: undefined;
|
|
1568
|
+
const errorCode = ["ENOSPC", "EDQUOT", "EACCES", "EPERM"].find((code) => diagnostics.some((diagnostic) => new RegExp(`\\b${code}\\b`).test(diagnostic)));
|
|
1569
|
+
const transportTextMatch = exitKind === undefined && isReceiverTransportFailure(failureReason);
|
|
1570
|
+
const isTransportFailure = exitKind !== undefined || transportTextMatch;
|
|
1571
|
+
return new ReceiverDispatchError(isTransportFailure ? "transport" : "path-specific", failureReason, { exitCode, errorCode, exitKind, transportTextMatch, httpStatus, rawExit: typeof outcome === "number" });
|
|
1638
1572
|
}
|
|
1639
1573
|
/**
|
|
1640
1574
|
* Exit code for one scoped/catch-up route. A 404 company route returns
|
|
@@ -1829,10 +1763,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1829
1763
|
let recycleRequested = null;
|
|
1830
1764
|
const pendingAppliedUpserts = new Map();
|
|
1831
1765
|
const pendingAppliedDeletes = new Set();
|
|
1832
|
-
const uploadedAwaitingPublish = new Set();
|
|
1833
|
-
const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
|
|
1834
|
-
const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
|
|
1835
|
-
let uploadedAwaitingPublishDropLogged = false;
|
|
1836
1766
|
const rememberAppliedUpsert = (relativePath, contentHash) => {
|
|
1837
1767
|
pendingAppliedDeletes.delete(relativePath);
|
|
1838
1768
|
if (eventSyncLive && eventSync) {
|
|
@@ -1863,23 +1793,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1863
1793
|
pendingAppliedDeletes.delete(oldest);
|
|
1864
1794
|
}
|
|
1865
1795
|
};
|
|
1866
|
-
const flushPendingUploadEvents = () => {
|
|
1867
|
-
if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
|
|
1868
|
-
return;
|
|
1869
|
-
while (uploadedAwaitingPublish.size > 0) {
|
|
1870
|
-
const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
|
|
1871
|
-
const paths = new Map(uploadedPaths.map((relativePath) => [
|
|
1872
|
-
localPathForVaultKey(hqRoot, relativePath),
|
|
1873
|
-
relativePath,
|
|
1874
|
-
]));
|
|
1875
|
-
const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
|
|
1876
|
-
if (expanded.size > 0)
|
|
1877
|
-
eventSync.publishBatch({ paths: expanded });
|
|
1878
|
-
for (const relativePath of uploadedPaths) {
|
|
1879
|
-
uploadedAwaitingPublish.delete(relativePath);
|
|
1880
|
-
}
|
|
1881
|
-
}
|
|
1882
|
-
};
|
|
1883
1796
|
const flushPendingAppliedUpserts = () => {
|
|
1884
1797
|
if (!eventSyncLive || !eventSync)
|
|
1885
1798
|
return;
|
|
@@ -1891,7 +1804,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1891
1804
|
eventSync.markContentDeleted?.(relativePath);
|
|
1892
1805
|
}
|
|
1893
1806
|
pendingAppliedDeletes.clear();
|
|
1894
|
-
flushPendingUploadEvents();
|
|
1895
1807
|
};
|
|
1896
1808
|
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
1897
1809
|
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
@@ -1942,7 +1854,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1942
1854
|
refresh.unref?.();
|
|
1943
1855
|
}
|
|
1944
1856
|
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
1945
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal
|
|
1857
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
|
|
1946
1858
|
const passCheckpoint = pullSignal || receiverSignal
|
|
1947
1859
|
? Object.assign(async () => checkpoint?.(), {
|
|
1948
1860
|
shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
|
|
@@ -1972,43 +1884,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1972
1884
|
}
|
|
1973
1885
|
};
|
|
1974
1886
|
const invokePass = () => {
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
const complete = () => {
|
|
1980
|
-
watcherPushStage?.("pass return");
|
|
1981
|
-
activePassCount = Math.max(0, activePassCount - 1);
|
|
1982
|
-
emitSyncWorkState(true);
|
|
1983
|
-
};
|
|
1984
|
-
// Observe settlement without returning a chained promise to the pass
|
|
1985
|
-
// scheduler. The scheduler's synchronous test seam and watcher wake
|
|
1986
|
-
// ordering depend on the original pass promise's completion turn.
|
|
1987
|
-
void pass.then(complete, complete);
|
|
1988
|
-
return pass;
|
|
1989
|
-
};
|
|
1990
|
-
try {
|
|
1991
|
-
if (!runPassSupportsCooperativeCheckpoint) {
|
|
1992
|
-
if (!runPassSupportsQmdLifecycle)
|
|
1993
|
-
return trackPass(runPass(passArgvForRun));
|
|
1994
|
-
return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
|
|
1995
|
-
}
|
|
1996
|
-
// This callback is emitted by the production runner after journal
|
|
1997
|
-
// repair/seed and before fanout. It starts ingress early, but never
|
|
1998
|
-
// admits a second durable writer: queued realtime still waits for the
|
|
1999
|
-
// existing scheduler's cooperative hand-off checkpoint.
|
|
2000
|
-
const onJournalsReady = () => activateEventPushSurfaces();
|
|
2001
|
-
if (!runPassSupportsQmdLifecycle) {
|
|
2002
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
|
|
2003
|
-
}
|
|
2004
|
-
return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
|
|
1887
|
+
if (!runPassSupportsCooperativeCheckpoint) {
|
|
1888
|
+
if (!runPassSupportsQmdLifecycle)
|
|
1889
|
+
return runPass(passArgvForRun);
|
|
1890
|
+
return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
|
|
2005
1891
|
}
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
1892
|
+
// This callback is emitted by the production runner after journal
|
|
1893
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
1894
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
1895
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
1896
|
+
const onJournalsReady = () => activateEventPushSurfaces();
|
|
1897
|
+
if (!runPassSupportsQmdLifecycle) {
|
|
1898
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
|
|
2011
1899
|
}
|
|
1900
|
+
return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
|
|
2012
1901
|
};
|
|
2013
1902
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
2014
1903
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -2023,11 +1912,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2023
1912
|
return;
|
|
2024
1913
|
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
2025
1914
|
recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
|
|
2026
|
-
rememberUploadedPaths(outcome);
|
|
2027
|
-
if (typeof outcome !== "number") {
|
|
2028
|
-
acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
|
|
2029
|
-
}
|
|
2030
|
-
flushPendingUploadEvents();
|
|
2031
1915
|
};
|
|
2032
1916
|
// Pre-admit boundary observation.
|
|
2033
1917
|
heapCheckpoint?.();
|
|
@@ -2278,25 +2162,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2278
2162
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2279
2163
|
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
|
|
2280
2164
|
};
|
|
2281
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass
|
|
2282
|
-
if (watcherPushStage) {
|
|
2283
|
-
watcherPushStage("guarded scheduler admission requested");
|
|
2284
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2285
|
-
if (passScheduler.isInCapYield)
|
|
2286
|
-
watcherPushStage("CPU/heap cap wait");
|
|
2287
|
-
else if (admission.holders.some((holder) => holder.kind === "pull")) {
|
|
2288
|
-
watcherPushStage("pull-yield wait");
|
|
2289
|
-
}
|
|
2290
|
-
}
|
|
2165
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
|
|
2291
2166
|
telemetry?.mark("operationGate");
|
|
2292
2167
|
const admissionSignal = signal && receiverSignal
|
|
2293
2168
|
? AbortSignal.any([signal, receiverSignal])
|
|
2294
2169
|
: signal ?? receiverSignal;
|
|
2295
2170
|
return runGuardedTask((context) => {
|
|
2296
|
-
watcherPushStage?.("guarded scheduler admission granted");
|
|
2297
2171
|
telemetry?.mark("appendLock");
|
|
2298
2172
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2299
|
-
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal
|
|
2173
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2300
2174
|
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
|
|
2301
2175
|
};
|
|
2302
2176
|
/*
|
|
@@ -2418,7 +2292,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2418
2292
|
return {
|
|
2419
2293
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2420
2294
|
accepted,
|
|
2421
|
-
yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
|
|
2422
2295
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2423
2296
|
.filter((path) => path.status === "refused")
|
|
2424
2297
|
.map((path) => path.path)),
|
|
@@ -2616,172 +2489,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2616
2489
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
2617
2490
|
let drainedUntrackedDeletes = 0;
|
|
2618
2491
|
const pendingWatcherPaths = new Map();
|
|
2619
|
-
const durableWatcherPaths = new Map();
|
|
2620
|
-
const durableWatcherPathLimit = 10_000;
|
|
2621
|
-
let durableWatcherOverflowed = false;
|
|
2622
|
-
let durableWatcherPersistTimer;
|
|
2623
|
-
let durableWatcherPersistQueue = Promise.resolve();
|
|
2624
|
-
let durableWatcherPersistGeneration = 0;
|
|
2625
|
-
let durableWatcherPersistedGeneration = 0;
|
|
2626
|
-
const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
|
|
2627
|
-
const persistDurableWatcherPaths = () => {
|
|
2628
|
-
const generation = durableWatcherPersistGeneration;
|
|
2629
|
-
if (generation === durableWatcherPersistedGeneration)
|
|
2630
|
-
return durableWatcherPersistQueue;
|
|
2631
|
-
const snapshot = durableWatcherOverflowed
|
|
2632
|
-
? { format: 1, hqRoot, overflowed: true }
|
|
2633
|
-
: { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
|
|
2634
|
-
durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
|
|
2635
|
-
try {
|
|
2636
|
-
if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
|
|
2637
|
-
await fs.promises.rm(watcherPushStatePath, { force: true });
|
|
2638
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2639
|
-
return;
|
|
2640
|
-
}
|
|
2641
|
-
await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
|
|
2642
|
-
const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
|
|
2643
|
-
await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
|
|
2644
|
-
await fs.promises.rename(tempPath, watcherPushStatePath);
|
|
2645
|
-
durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
|
|
2646
|
-
}
|
|
2647
|
-
catch (err) {
|
|
2648
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2649
|
-
}
|
|
2650
|
-
}).catch((err) => {
|
|
2651
|
-
process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
|
|
2652
|
-
});
|
|
2653
|
-
return durableWatcherPersistQueue;
|
|
2654
|
-
};
|
|
2655
|
-
const scheduleDurableWatcherPersist = () => {
|
|
2656
|
-
durableWatcherPersistGeneration += 1;
|
|
2657
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
2658
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
2659
|
-
durableWatcherPersistTimer = setTimeout(() => {
|
|
2660
|
-
durableWatcherPersistTimer = undefined;
|
|
2661
|
-
void persistDurableWatcherPaths();
|
|
2662
|
-
}, 1_000);
|
|
2663
|
-
durableWatcherPersistTimer.unref?.();
|
|
2664
|
-
};
|
|
2665
|
-
const addDurableWatcherPath = (absolutePath, relativePath) => {
|
|
2666
|
-
if (durableWatcherOverflowed)
|
|
2667
|
-
return;
|
|
2668
|
-
if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
|
|
2669
|
-
durableWatcherPaths.clear();
|
|
2670
|
-
durableWatcherOverflowed = true;
|
|
2671
|
-
}
|
|
2672
|
-
else if (!durableWatcherOverflowed) {
|
|
2673
|
-
durableWatcherPaths.set(absolutePath, relativePath);
|
|
2674
|
-
}
|
|
2675
|
-
scheduleDurableWatcherPersist();
|
|
2676
|
-
};
|
|
2677
|
-
const clearDurableWatcherOverflow = () => {
|
|
2678
|
-
if (!durableWatcherOverflowed)
|
|
2679
|
-
return;
|
|
2680
|
-
durableWatcherOverflowed = false;
|
|
2681
|
-
durableWatcherPaths.clear();
|
|
2682
|
-
for (const [absolutePath, relativePath] of pendingWatcherPaths) {
|
|
2683
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
2684
|
-
}
|
|
2685
|
-
scheduleDurableWatcherPersist();
|
|
2686
|
-
};
|
|
2687
|
-
const acknowledgeDurableWatcherUploads = (relativePaths) => {
|
|
2688
|
-
let changed = false;
|
|
2689
|
-
const uploaded = new Set(relativePaths);
|
|
2690
|
-
for (const [absolutePath, relativePath] of durableWatcherPaths) {
|
|
2691
|
-
if (!uploaded.has(relativePath))
|
|
2692
|
-
continue;
|
|
2693
|
-
durableWatcherPaths.delete(absolutePath);
|
|
2694
|
-
changed = true;
|
|
2695
|
-
}
|
|
2696
|
-
if (changed)
|
|
2697
|
-
scheduleDurableWatcherPersist();
|
|
2698
|
-
};
|
|
2699
|
-
const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
|
|
2700
|
-
const stillPending = new Set(pendingWatcherPaths.values());
|
|
2701
|
-
acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
|
|
2702
|
-
};
|
|
2703
|
-
let activePassCount = 0;
|
|
2704
|
-
let inFlightWatcherPushes = 0;
|
|
2705
|
-
const watcherPushStages = new Map();
|
|
2706
|
-
let nextWatcherPushGroupId = 0;
|
|
2707
|
-
const setWatcherPushStage = (groupId, stage) => {
|
|
2708
|
-
const record = watcherPushStages.get(groupId);
|
|
2709
|
-
if (!record || record.stage === stage)
|
|
2710
|
-
return;
|
|
2711
|
-
record.stage = stage;
|
|
2712
|
-
record.stageEnteredAt = Date.now();
|
|
2713
|
-
process.stderr.write(`${JSON.stringify({
|
|
2714
|
-
type: "watcher-push-group-stage",
|
|
2715
|
-
diagnostic: true,
|
|
2716
|
-
groupId: record.groupId,
|
|
2717
|
-
route: record.route,
|
|
2718
|
-
paths: record.paths,
|
|
2719
|
-
stage: record.stage,
|
|
2720
|
-
stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
|
|
2721
|
-
})}\n`);
|
|
2722
|
-
};
|
|
2723
|
-
const startWatcherPushStageTracking = (route, paths) => {
|
|
2724
|
-
const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
|
|
2725
|
-
watcherPushStages.set(groupId, {
|
|
2726
|
-
groupId,
|
|
2727
|
-
route,
|
|
2728
|
-
paths,
|
|
2729
|
-
stage: "",
|
|
2730
|
-
stageEnteredAt: Date.now(),
|
|
2731
|
-
});
|
|
2732
|
-
return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
|
|
2733
|
-
};
|
|
2734
|
-
const emitWatcherPushInflightStages = () => {
|
|
2735
|
-
if (inFlightWatcherPushes <= 0)
|
|
2736
|
-
return;
|
|
2737
|
-
const now = Date.now();
|
|
2738
|
-
const entries = [...watcherPushStages.values()];
|
|
2739
|
-
const admission = passScheduler.diagnosticAdmissionSnapshot;
|
|
2740
|
-
process.stderr.write(`${JSON.stringify({
|
|
2741
|
-
type: "watcher-push-inflight-stages",
|
|
2742
|
-
diagnostic: true,
|
|
2743
|
-
groups: entries.slice(0, 20).map((record) => ({
|
|
2744
|
-
route: record.route,
|
|
2745
|
-
paths: record.paths,
|
|
2746
|
-
stage: record.stage,
|
|
2747
|
-
ageMs: Math.max(0, now - record.stageEnteredAt),
|
|
2748
|
-
})),
|
|
2749
|
-
other: Math.max(0, entries.length - 20),
|
|
2750
|
-
realtimeAdmissionQueueDepth: admission.queueDepth,
|
|
2751
|
-
holders: admission.holders,
|
|
2752
|
-
})}\n`);
|
|
2753
|
-
};
|
|
2754
|
-
const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
|
|
2755
|
-
watcherPushStagesTimer.unref?.();
|
|
2756
|
-
let pendingReceivers = 0;
|
|
2757
|
-
let lastSyncWorkStateEmitAt = 0;
|
|
2758
|
-
let syncWorkStateTimer;
|
|
2759
|
-
const emitSyncWorkState = (force = false) => {
|
|
2760
|
-
const now = Date.now();
|
|
2761
|
-
const elapsed = now - lastSyncWorkStateEmitAt;
|
|
2762
|
-
if (force || elapsed >= 250) {
|
|
2763
|
-
if (syncWorkStateTimer !== undefined)
|
|
2764
|
-
clearTimeout(syncWorkStateTimer);
|
|
2765
|
-
syncWorkStateTimer = undefined;
|
|
2766
|
-
lastSyncWorkStateEmitAt = now;
|
|
2767
|
-
const snapshot = {
|
|
2768
|
-
passActive: activePassCount > 0,
|
|
2769
|
-
queuedWatcherPushes: pendingWatcherPaths.size,
|
|
2770
|
-
inFlightWatcherPushes,
|
|
2771
|
-
pendingReceivers,
|
|
2772
|
-
};
|
|
2773
|
-
deps.onSyncWorkStateSnapshot?.(snapshot);
|
|
2774
|
-
stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
|
|
2775
|
-
return;
|
|
2776
|
-
}
|
|
2777
|
-
if (syncWorkStateTimer !== undefined)
|
|
2778
|
-
return;
|
|
2779
|
-
syncWorkStateTimer = setTimeout(() => {
|
|
2780
|
-
syncWorkStateTimer = undefined;
|
|
2781
|
-
emitSyncWorkState(true);
|
|
2782
|
-
}, Math.max(1, 250 - elapsed));
|
|
2783
|
-
syncWorkStateTimer.unref?.();
|
|
2784
|
-
};
|
|
2785
2492
|
const droppedUnresolvedWatcherSlugs = new Set();
|
|
2786
2493
|
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
2787
2494
|
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
@@ -2849,11 +2556,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2849
2556
|
}
|
|
2850
2557
|
}
|
|
2851
2558
|
if (batch) {
|
|
2852
|
-
if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
|
|
2853
|
-
durableWatcherPaths.clear();
|
|
2854
|
-
durableWatcherOverflowed = true;
|
|
2855
|
-
scheduleDurableWatcherPersist();
|
|
2856
|
-
}
|
|
2857
2559
|
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
2858
2560
|
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
2859
2561
|
return previousSequence === undefined ||
|
|
@@ -2862,7 +2564,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2862
2564
|
});
|
|
2863
2565
|
if (entries.length === 0)
|
|
2864
2566
|
return false;
|
|
2865
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2866
2567
|
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
2867
2568
|
pendingWatcherOriginalBatch =
|
|
2868
2569
|
pendingWatcherPaths.size === 0 &&
|
|
@@ -2874,13 +2575,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2874
2575
|
: null;
|
|
2875
2576
|
for (const [absolutePath, relativePath] of entries) {
|
|
2876
2577
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
2877
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
2878
2578
|
const change = batch.changes?.get(absolutePath);
|
|
2879
2579
|
if (change)
|
|
2880
2580
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2881
2581
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2882
2582
|
}
|
|
2883
|
-
emitSyncWorkState(queueWasEmpty);
|
|
2884
2583
|
if (batch.overflowed) {
|
|
2885
2584
|
pendingWatcherOverflowed = true;
|
|
2886
2585
|
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
@@ -2904,20 +2603,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2904
2603
|
// counterpart must be re-derived through the codec — a raw join would
|
|
2905
2604
|
// point at the impossible colon path on Windows.
|
|
2906
2605
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
2907
|
-
const queueWasEmpty = pendingWatcherPaths.size === 0;
|
|
2908
2606
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
2909
|
-
addDurableWatcherPath(absolutePath, changedRelPath);
|
|
2910
2607
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2911
2608
|
// A bare path signal carries no delete intent; drop any change record
|
|
2912
2609
|
// a prior batch left for this path so the stale intent cannot leak
|
|
2913
2610
|
// into the coalesced batch (#227).
|
|
2914
2611
|
pendingWatcherChanges.delete(absolutePath);
|
|
2915
|
-
emitSyncWorkState(queueWasEmpty);
|
|
2916
2612
|
return true;
|
|
2917
2613
|
}
|
|
2918
2614
|
pendingWatcherOriginalBatch = null;
|
|
2919
2615
|
pendingWatcherBareChange = true;
|
|
2920
|
-
emitSyncWorkState();
|
|
2921
2616
|
return true;
|
|
2922
2617
|
};
|
|
2923
2618
|
const takePendingWatcherChange = () => {
|
|
@@ -2962,7 +2657,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2962
2657
|
pendingWatcherDroppedRouteHints.clear();
|
|
2963
2658
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
2964
2659
|
pendingWatcherBatchGeneration = null;
|
|
2965
|
-
emitSyncWorkState();
|
|
2966
2660
|
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
2967
2661
|
};
|
|
2968
2662
|
const restorePendingWatcherPaths = (paths, generation, changes) => {
|
|
@@ -2976,54 +2670,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2976
2670
|
if (pendingWatcherChangeSequences.has(absolutePath))
|
|
2977
2671
|
continue;
|
|
2978
2672
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
2979
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
2980
2673
|
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
2981
2674
|
const change = changes?.get(absolutePath);
|
|
2982
2675
|
if (change)
|
|
2983
2676
|
pendingWatcherChanges.set(absolutePath, change);
|
|
2984
2677
|
}
|
|
2985
|
-
emitSyncWorkState();
|
|
2986
2678
|
};
|
|
2987
|
-
if (eventPush) {
|
|
2988
|
-
try {
|
|
2989
|
-
const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
|
|
2990
|
-
if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
|
|
2991
|
-
(!Array.isArray(stored.paths) && stored.overflowed !== true)) {
|
|
2992
|
-
throw new Error("stored watcher push state has an unsupported shape or root");
|
|
2993
|
-
}
|
|
2994
|
-
if (stored.overflowed === true) {
|
|
2995
|
-
durableWatcherOverflowed = true;
|
|
2996
|
-
pendingWatcherOverflowed = true;
|
|
2997
|
-
pendingWatcherWarmupCatchup = true;
|
|
2998
|
-
pendingWatcherDroppedRoutesUnknown = true;
|
|
2999
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3000
|
-
emitSyncWorkState(true);
|
|
3001
|
-
process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
|
|
3002
|
-
}
|
|
3003
|
-
for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
|
|
3004
|
-
if (typeof relativePath !== "string" || relativePath.length === 0)
|
|
3005
|
-
continue;
|
|
3006
|
-
const route = routeChangeToTarget(relativePath);
|
|
3007
|
-
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3008
|
-
continue;
|
|
3009
|
-
const absolutePath = localPathForVaultKey(hqRoot, relativePath);
|
|
3010
|
-
addDurableWatcherPath(absolutePath, relativePath);
|
|
3011
|
-
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
3012
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
3013
|
-
}
|
|
3014
|
-
if (pendingWatcherPaths.size > 0) {
|
|
3015
|
-
pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
|
|
3016
|
-
pendingWatcherWarmupCatchup = true;
|
|
3017
|
-
emitSyncWorkState(true);
|
|
3018
|
-
process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
|
|
3019
|
-
}
|
|
3020
|
-
}
|
|
3021
|
-
catch (err) {
|
|
3022
|
-
if (!(err instanceof Error && err.code === "ENOENT")) {
|
|
3023
|
-
process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
|
|
3024
|
-
}
|
|
3025
|
-
}
|
|
3026
|
-
}
|
|
3027
2679
|
const routePushFailureBackoff = new RoutePushFailureBackoff();
|
|
3028
2680
|
const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
|
|
3029
2681
|
const routePushRetryTimerHandles = new Map();
|
|
@@ -3874,6 +3526,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3874
3526
|
// a file and still be refused as stale (a download changed the directory
|
|
3875
3527
|
// mid-pass); the retry then finds the file already in the cloud and uploads
|
|
3876
3528
|
// nothing, so without this the file is never announced to peers.
|
|
3529
|
+
const uploadedAwaitingPublish = new Set();
|
|
3530
|
+
const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
|
|
3877
3531
|
const rememberUploadedPaths = (result) => {
|
|
3878
3532
|
if (result === undefined || typeof result === "number")
|
|
3879
3533
|
return;
|
|
@@ -3886,14 +3540,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3886
3540
|
const oldest = uploadedAwaitingPublish.values().next().value;
|
|
3887
3541
|
if (oldest !== undefined)
|
|
3888
3542
|
uploadedAwaitingPublish.delete(oldest);
|
|
3889
|
-
if (!uploadedAwaitingPublishDropLogged) {
|
|
3890
|
-
uploadedAwaitingPublishDropLogged = true;
|
|
3891
|
-
process.stderr.write(`${JSON.stringify({
|
|
3892
|
-
type: "event-sync-upload-buffer-overflow",
|
|
3893
|
-
cap: UPLOADED_AWAITING_PUBLISH_CAP,
|
|
3894
|
-
action: "drop-oldest",
|
|
3895
|
-
})}\n`);
|
|
3896
|
-
}
|
|
3897
3543
|
}
|
|
3898
3544
|
}
|
|
3899
3545
|
};
|
|
@@ -4009,39 +3655,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4009
3655
|
: "small";
|
|
4010
3656
|
const largeUpload = sizeClass !== "small";
|
|
4011
3657
|
const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
|
|
4012
|
-
const
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
4025
|
-
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
4026
|
-
? () => prepareWatcherChanges(admittedChanges)
|
|
4027
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced, undefined, routeId, watcherPushStage);
|
|
4028
|
-
}, sizeClass)
|
|
4029
|
-
: (() => {
|
|
4030
|
-
watcherPushStage("worker entry");
|
|
4031
|
-
return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
4032
|
-
? () => prepareWatcherChanges(group.changes)
|
|
4033
|
-
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
|
|
4034
|
-
result,
|
|
4035
|
-
accepted: true,
|
|
4036
|
-
coordinatorRefusedPaths: [],
|
|
4037
|
-
}));
|
|
4038
|
-
})());
|
|
4039
|
-
}
|
|
4040
|
-
finally {
|
|
4041
|
-
watcherPushStages.delete(groupId);
|
|
4042
|
-
inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
|
|
4043
|
-
emitSyncWorkState(true);
|
|
4044
|
-
}
|
|
3658
|
+
const execution = parallelCoordinatorEnabled && !chorePush
|
|
3659
|
+
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
|
|
3660
|
+
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
3661
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
3662
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
3663
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
3664
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
3665
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
|
|
3666
|
+
}, sizeClass)
|
|
3667
|
+
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
3668
|
+
? () => prepareWatcherChanges(group.changes)
|
|
3669
|
+
: undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
4045
3670
|
const result = execution.result;
|
|
4046
3671
|
rememberUploadedPaths(result);
|
|
4047
3672
|
const pushExit = passExitCode(result);
|
|
@@ -4060,7 +3685,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4060
3685
|
}
|
|
4061
3686
|
const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
|
|
4062
3687
|
relativePath.startsWith(`${refusedPath}/`));
|
|
4063
|
-
acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
|
|
4064
3688
|
const publishPaths = new Map();
|
|
4065
3689
|
for (const [absolutePath, relativePath] of group.paths) {
|
|
4066
3690
|
if (isRefusedPath(relativePath))
|
|
@@ -4100,11 +3724,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4100
3724
|
deferPushAfterAdmissionTimeout(group, routeId);
|
|
4101
3725
|
}
|
|
4102
3726
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
4103
|
-
acknowledgeDurableWatcherUploads(group.relPaths);
|
|
4104
3727
|
routePushFailureBackoff.succeeded(routeId);
|
|
4105
3728
|
clearRoutePushRetry(routeId);
|
|
4106
3729
|
if (group.route.kind === "company") {
|
|
4107
|
-
deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
|
|
4108
3730
|
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
4109
3731
|
}
|
|
4110
3732
|
}
|
|
@@ -4166,7 +3788,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4166
3788
|
const execution = parallelCoordinatorEnabled
|
|
4167
3789
|
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
4168
3790
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4169
|
-
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced
|
|
3791
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
4170
3792
|
})
|
|
4171
3793
|
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
4172
3794
|
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
@@ -4218,7 +3840,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4218
3840
|
addRoute(hint);
|
|
4219
3841
|
const skipCompanies = resolveSkipCompanies();
|
|
4220
3842
|
if (parallelCoordinatorEnabled) {
|
|
4221
|
-
let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
|
|
4222
3843
|
for (const route of routes.values()) {
|
|
4223
3844
|
if (route.kind === "company" &&
|
|
4224
3845
|
skipCompanies.has(route.slug.toLowerCase())) {
|
|
@@ -4247,11 +3868,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4247
3868
|
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
4248
3869
|
interruptWait();
|
|
4249
3870
|
}
|
|
4250
|
-
else {
|
|
4251
|
-
routesRemaining -= 1;
|
|
4252
|
-
if (routesRemaining === 0)
|
|
4253
|
-
clearDurableWatcherOverflow();
|
|
4254
|
-
}
|
|
4255
3871
|
});
|
|
4256
3872
|
}
|
|
4257
3873
|
return 0;
|
|
@@ -4488,7 +4104,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4488
4104
|
}
|
|
4489
4105
|
}
|
|
4490
4106
|
watcher.start();
|
|
4491
|
-
let catchupPathStatWarningReported = false;
|
|
4492
4107
|
const reportWatcherReady = () => {
|
|
4493
4108
|
if (stopped || watcher === null)
|
|
4494
4109
|
return;
|
|
@@ -4529,147 +4144,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4529
4144
|
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
|
|
4530
4145
|
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
4531
4146
|
return;
|
|
4532
|
-
const
|
|
4533
|
-
const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
|
|
4534
|
-
const eligiblePaths = new Map();
|
|
4535
|
-
const eligibleChanges = new Map();
|
|
4536
|
-
const excludedByReason = {};
|
|
4537
|
-
const excludedByReasonByRoute = {};
|
|
4538
|
-
const ignoreFilter = createIgnoreFilter(hqRoot);
|
|
4539
|
-
const personalFilterExclusions = new Map();
|
|
4540
|
-
const personalPushFilter = wrapFilterWithPersonalVaultDefaults(ignoreFilter, hqRoot, (relativePath, match) => personalFilterExclusions.set(relativePath, match.id));
|
|
4541
|
-
const clearedResolvedCompanies = new Set();
|
|
4542
|
-
const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
|
|
4543
|
-
const backupClass = (relativePath) => {
|
|
4544
|
-
if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
|
|
4545
|
-
return "attioCleanup";
|
|
4546
|
-
if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
|
|
4547
|
-
return "readModelRollout";
|
|
4548
|
-
return undefined;
|
|
4549
|
-
};
|
|
4550
|
-
const exclude = (reason, routeName) => {
|
|
4551
|
-
excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
|
|
4552
|
-
const routeReasons = excludedByReasonByRoute[routeName] ??= {};
|
|
4553
|
-
routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
|
|
4554
|
-
};
|
|
4555
|
-
for (const [absolutePath, relativePath] of catchup.batch.paths) {
|
|
4556
|
-
const route = routeChangeToTarget(relativePath);
|
|
4557
|
-
const routeName = route ? routeKey(route) : "unrouted";
|
|
4558
|
-
const ontologyBackupClass = backupClass(relativePath);
|
|
4559
|
-
if (ontologyBackupClass) {
|
|
4560
|
-
exclude("local-rule:ontology-backup", routeName);
|
|
4561
|
-
ontologyBackupPaths[ontologyBackupClass] += 1;
|
|
4562
|
-
continue;
|
|
4563
|
-
}
|
|
4564
|
-
if (route?.kind === "company") {
|
|
4565
|
-
const resolvedNow = cloudCompanySlugs.has(route.slug) ||
|
|
4566
|
-
skipStore?.hasCurrentGrantCompany(route.slug) ||
|
|
4567
|
-
skipStore?.hasCurrentGrantVersion(route.slug) ||
|
|
4568
|
-
(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
|
|
4569
|
-
if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
|
|
4570
|
-
skipStore?.clearUnresolvedCompany(route.slug);
|
|
4571
|
-
clearedResolvedCompanies.add(route.slug);
|
|
4572
|
-
}
|
|
4573
|
-
if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
|
|
4574
|
-
exclude("unresolved-company", routeName);
|
|
4575
|
-
continue;
|
|
4576
|
-
}
|
|
4577
|
-
}
|
|
4578
|
-
if (route?.kind === "company" &&
|
|
4579
|
-
!cloudCompanySlugs.has(route.slug) &&
|
|
4580
|
-
!skipStore?.hasCurrentGrantCompany(route.slug) &&
|
|
4581
|
-
!skipStore?.hasCurrentGrantVersion(route.slug) &&
|
|
4582
|
-
!(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
|
|
4583
|
-
exclude("local-only-company", routeName);
|
|
4584
|
-
continue;
|
|
4585
|
-
}
|
|
4586
|
-
const company = route?.kind === "company" ? route.slug : "personal";
|
|
4587
|
-
const routePath = route?.kind === "company"
|
|
4588
|
-
? relativePath.slice(`companies/${route.slug}/`.length)
|
|
4589
|
-
: relativePath;
|
|
4590
|
-
if (isEphemeralPath(routePath)) {
|
|
4591
|
-
exclude("push-filter:conflict-mirror", routeName);
|
|
4592
|
-
continue;
|
|
4593
|
-
}
|
|
4594
|
-
const acceptedByPushFilter = route?.kind === "personal"
|
|
4595
|
-
? personalPushFilter(absolutePath, false)
|
|
4596
|
-
: ignoreFilter(absolutePath, false);
|
|
4597
|
-
if (!acceptedByPushFilter) {
|
|
4598
|
-
const exclusionId = route?.kind === "personal"
|
|
4599
|
-
? personalFilterExclusions.get(relativePath)
|
|
4600
|
-
: undefined;
|
|
4601
|
-
exclude(exclusionId
|
|
4602
|
-
? `push-filter:personal:${exclusionId}`
|
|
4603
|
-
: "push-filter:ignore", routeName);
|
|
4604
|
-
continue;
|
|
4605
|
-
}
|
|
4606
|
-
if (route && isLocallyIneligibleForPush(routePath, {
|
|
4607
|
-
boardClientOwned: true,
|
|
4608
|
-
ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
|
|
4609
|
-
})) {
|
|
4610
|
-
const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
|
|
4611
|
-
exclude(`local-rule:${pathClass}`, routeName);
|
|
4612
|
-
continue;
|
|
4613
|
-
}
|
|
4614
|
-
try {
|
|
4615
|
-
const stat = fs.lstatSync(absolutePath);
|
|
4616
|
-
if (route && stat.isSymbolicLink()) {
|
|
4617
|
-
const scopeRoot = route.kind === "company"
|
|
4618
|
-
? path.join(hqRoot, "companies", route.slug)
|
|
4619
|
-
: relativePath.startsWith("personal/")
|
|
4620
|
-
? path.join(hqRoot, "personal")
|
|
4621
|
-
: hqRoot;
|
|
4622
|
-
const inspection = inspectLocalSymlinkScope({
|
|
4623
|
-
absoluteLinkPath: absolutePath,
|
|
4624
|
-
linkPath: relativePath,
|
|
4625
|
-
scopeRoot,
|
|
4626
|
-
scope: route.kind === "company"
|
|
4627
|
-
? { kind: "company", companySlug: route.slug }
|
|
4628
|
-
: { kind: "personal" },
|
|
4629
|
-
});
|
|
4630
|
-
if (inspection.refused) {
|
|
4631
|
-
exclude("symlink", routeName);
|
|
4632
|
-
continue;
|
|
4633
|
-
}
|
|
4634
|
-
}
|
|
4635
|
-
if (skipStore?.matches(company, routePath, stat)) {
|
|
4636
|
-
exclude("record-match", routeName);
|
|
4637
|
-
continue;
|
|
4638
|
-
}
|
|
4639
|
-
}
|
|
4640
|
-
catch (error) {
|
|
4641
|
-
// Fail open: unreadable state or a filesystem race must not suppress a candidate.
|
|
4642
|
-
if (!catchupPathStatWarningReported) {
|
|
4643
|
-
catchupPathStatWarningReported = true;
|
|
4644
|
-
process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
|
|
4645
|
-
}
|
|
4646
|
-
}
|
|
4647
|
-
eligiblePaths.set(absolutePath, relativePath);
|
|
4648
|
-
const change = catchup.batch.changes?.get(absolutePath);
|
|
4649
|
-
if (change)
|
|
4650
|
-
eligibleChanges.set(absolutePath, change);
|
|
4651
|
-
}
|
|
4652
|
-
const queued = addPendingWatcherChange(undefined, {
|
|
4653
|
-
paths: eligiblePaths,
|
|
4654
|
-
changes: eligibleChanges,
|
|
4655
|
-
}, {
|
|
4147
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
4656
4148
|
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
4657
4149
|
});
|
|
4658
4150
|
pendingWatcherWarmupCatchup ||= queued;
|
|
4659
|
-
const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
|
|
4660
4151
|
process.stderr.write(JSON.stringify({
|
|
4661
4152
|
type: "watcher-warmup-catchup",
|
|
4662
4153
|
reason: "watch-set-ready",
|
|
4663
|
-
pathsFound:
|
|
4664
|
-
candidatePathsByRoute: watcherWarmupPathsByRoute(catchup.batch.paths.values()),
|
|
4665
|
-
pathsByRoute: watcherWarmupPathsByRoute(eligiblePaths.values()),
|
|
4666
|
-
pathPrefixesByRoute: watcherWarmupPrefixesByRoute(catchup.batch.paths.values()),
|
|
4667
|
-
excludedByReason: {
|
|
4668
|
-
...excludedByReason,
|
|
4669
|
-
...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
|
|
4670
|
-
},
|
|
4671
|
-
excludedByReasonByRoute,
|
|
4672
|
-
ontologyBackupPaths,
|
|
4154
|
+
pathsFound: catchup.batch.paths.size,
|
|
4673
4155
|
scannedPaths: catchup.scannedPaths,
|
|
4674
4156
|
elapsedMs: Date.now() - startedAt,
|
|
4675
4157
|
queued,
|
|
@@ -4736,16 +4218,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4736
4218
|
const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
4737
4219
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
4738
4220
|
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
4739
|
-
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal
|
|
4221
|
+
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
4740
4222
|
});
|
|
4741
4223
|
const pathCount = events.length;
|
|
4742
4224
|
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
4743
4225
|
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
4744
4226
|
if (execution.admissionTimeout)
|
|
4745
4227
|
throw execution.admissionTimeout;
|
|
4746
|
-
if (execution.yieldedToPush) {
|
|
4747
|
-
throw new RealtimePushYieldError();
|
|
4748
|
-
}
|
|
4749
4228
|
const exitCode = passExitCode(execution.result);
|
|
4750
4229
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
4751
4230
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -4767,82 +4246,74 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4767
4246
|
const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
|
|
4768
4247
|
if (stopped)
|
|
4769
4248
|
return;
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
4777
|
-
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
if (lane) {
|
|
4787
|
-
receiverLaneByBatch.delete(events);
|
|
4788
|
-
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4789
|
-
return;
|
|
4790
|
-
}
|
|
4791
|
-
// Direct embedders may invoke the engine without first going through
|
|
4792
|
-
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4793
|
-
// delivery-wide partition the receiver would expose.
|
|
4794
|
-
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4795
|
-
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4796
|
-
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4249
|
+
// The injectable receiver seam and the production SQS receiver both
|
|
4250
|
+
// arrive here only after a successful receive. Keep the cadence tight
|
|
4251
|
+
// until the receiver later reports an empty outer long-poll.
|
|
4252
|
+
onReceiveActivity(true);
|
|
4253
|
+
// A received wake/high-water signal means there is bounded work now.
|
|
4254
|
+
// It wakes the scheduler but is still funneled through runGuarded below,
|
|
4255
|
+
// so it cannot overlap the pass that may currently own the operation
|
|
4256
|
+
// lock.
|
|
4257
|
+
// Apply the caps once across the whole delivery, then retain the residue
|
|
4258
|
+
// in the normal lane. The receiver already coalesces to one latest event
|
|
4259
|
+
// per path, so the partition cannot split revisions of one path.
|
|
4260
|
+
if (parallelCoordinatorEnabled) {
|
|
4261
|
+
const lane = receiverLaneByBatch.get(events);
|
|
4262
|
+
if (lane) {
|
|
4263
|
+
receiverLaneByBatch.delete(events);
|
|
4264
|
+
await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4797
4265
|
return;
|
|
4798
4266
|
}
|
|
4267
|
+
// Direct embedders may invoke the engine without first going through
|
|
4268
|
+
// SqsPushReceiver. Preserve that seam while applying the same
|
|
4269
|
+
// delivery-wide partition the receiver would expose.
|
|
4799
4270
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
}
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
}
|
|
4829
|
-
if (exitCode !== 0) {
|
|
4271
|
+
await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4272
|
+
await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
|
|
4273
|
+
return;
|
|
4274
|
+
}
|
|
4275
|
+
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
4276
|
+
const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
|
|
4277
|
+
const eventsByRoute = new Map();
|
|
4278
|
+
for (const event of laneEvents) {
|
|
4279
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
4280
|
+
// The receiver is an independent ingress from the local watcher.
|
|
4281
|
+
// Apply the same route-selection boundary before it can enqueue a
|
|
4282
|
+
// fast pull: a --skip-personal runner must acknowledge, not pull,
|
|
4283
|
+
// a personal-namespace delivery.
|
|
4284
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
4285
|
+
continue;
|
|
4286
|
+
const key = routeKey(route);
|
|
4287
|
+
const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
|
|
4288
|
+
group.events.push(event);
|
|
4289
|
+
if (event.kind === "upsert")
|
|
4290
|
+
group.upsertPaths.push(event.relativePath);
|
|
4291
|
+
eventsByRoute.set(key, group);
|
|
4292
|
+
}
|
|
4293
|
+
for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
|
|
4294
|
+
const paths = routeEvents.map((event) => event.relativePath);
|
|
4295
|
+
const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
|
|
4296
|
+
const exitCode = passExitCode(result);
|
|
4297
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
4298
|
+
if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
|
|
4830
4299
|
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4831
4300
|
}
|
|
4301
|
+
warnUnresolvedReceiverCompany(route, routeEvents.length);
|
|
4832
4302
|
await settleEvents?.(routeEvents);
|
|
4303
|
+
continue;
|
|
4833
4304
|
}
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
|
|
4305
|
+
if (exitCode !== 0) {
|
|
4306
|
+
throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
|
|
4307
|
+
}
|
|
4308
|
+
await settleEvents?.(routeEvents);
|
|
4309
|
+
}
|
|
4310
|
+
};
|
|
4311
|
+
// Dispatch realtime work before awaiting its bulk residue. This lets an
|
|
4312
|
+
// established slow pass continue while the small scoped pull uses its
|
|
4313
|
+
// ref-counted lock carve-out; the residue remains serialized and cannot
|
|
4314
|
+
// starve the slow lane.
|
|
4315
|
+
await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
|
|
4316
|
+
await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
|
|
4846
4317
|
};
|
|
4847
4318
|
receiverSyncBatchFn.partition = (events, maxPaths) => {
|
|
4848
4319
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
@@ -4999,14 +4470,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4999
4470
|
if (stopped)
|
|
5000
4471
|
return shutdownPromise ?? Promise.resolve();
|
|
5001
4472
|
stopped = true;
|
|
5002
|
-
clearInterval(watcherPushStagesTimer);
|
|
5003
|
-
if (durableWatcherPersistTimer !== undefined)
|
|
5004
|
-
clearTimeout(durableWatcherPersistTimer);
|
|
5005
|
-
durableWatcherPersistTimer = undefined;
|
|
5006
|
-
if (syncWorkStateTimer !== undefined)
|
|
5007
|
-
clearTimeout(syncWorkStateTimer);
|
|
5008
|
-
syncWorkStateTimer = undefined;
|
|
5009
|
-
emitSyncWorkState(true);
|
|
5010
4473
|
for (const route of routePushRetryTimerHandles.keys()) {
|
|
5011
4474
|
clearRoutePushRetry(route);
|
|
5012
4475
|
}
|
|
@@ -5036,7 +4499,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5036
4499
|
coordinatorDisposal,
|
|
5037
4500
|
boundedEventSyncStartup,
|
|
5038
4501
|
eventSyncDisposal,
|
|
5039
|
-
persistDurableWatcherPaths(),
|
|
5040
4502
|
]).then(() => undefined);
|
|
5041
4503
|
try {
|
|
5042
4504
|
void receiver?.dispose();
|
|
@@ -5454,14 +4916,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5454
4916
|
lastCompletedFullReconcileWallAt = Date.now();
|
|
5455
4917
|
fullPassJournalHealthy = execution.accepted &&
|
|
5456
4918
|
isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
|
|
5457
|
-
if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
|
|
5458
|
-
clearDurableWatcherOverflow();
|
|
5459
|
-
}
|
|
5460
|
-
else if (fullPassJournalHealthy && pending.batch !== null) {
|
|
5461
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
5462
|
-
...pending.batch.paths.values(),
|
|
5463
|
-
]);
|
|
5464
|
-
}
|
|
5465
4919
|
if (fullPassJournalHealthy) {
|
|
5466
4920
|
fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
|
|
5467
4921
|
}
|
|
@@ -5519,11 +4973,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5519
4973
|
const catchupBatch = workingBatch;
|
|
5520
4974
|
const catchupGeneration = pending.generation;
|
|
5521
4975
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
5522
|
-
if (exit === 0)
|
|
5523
|
-
clearDurableWatcherOverflow();
|
|
5524
|
-
return;
|
|
5525
|
-
}
|
|
5526
|
-
if (exit === SCOPED_DRAIN_DEFERRED)
|
|
4976
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
5527
4977
|
return;
|
|
5528
4978
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
5529
4979
|
interruptWait();
|
|
@@ -5631,19 +5081,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
5631
5081
|
? exitCode === 0
|
|
5632
5082
|
: hasCompletedRunnerPass(result));
|
|
5633
5083
|
if (completedPostRepairPass) {
|
|
5634
|
-
if (!parallelCoordinatorEnabled &&
|
|
5635
|
-
pending.batch?.overflowed === true &&
|
|
5636
|
-
((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
|
|
5637
|
-
clearDurableWatcherOverflow();
|
|
5638
|
-
}
|
|
5639
|
-
else if (!parallelCoordinatorEnabled &&
|
|
5640
|
-
ranFullReconcile &&
|
|
5641
|
-
fullPassJournalHealthy &&
|
|
5642
|
-
pending.batch !== null) {
|
|
5643
|
-
acknowledgeDurableWatcherPathsCoveredByFullPass([
|
|
5644
|
-
...pending.batch.paths.values(),
|
|
5645
|
-
]);
|
|
5646
|
-
}
|
|
5647
5084
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
5648
5085
|
// which follows journal repair. Its numeric seam has the same
|
|
5649
5086
|
// contract: zero means the injected pass completed. This preserves
|