@indigoai-us/hq-cloud 6.18.35 → 6.18.37

package/dist/bin/sync-runner-watch-loop.js

@@ -9,7 +9,8 @@ import { targetedJournalSessionCensusSizes } from "../cli/sync.js";
 import { VaultClient } from "../index.js";
 import { pickCanonicalPersonEntity } from "../vault-client.js";
 import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
-import { describeError } from "../lib/describe-error.js";
+import { describeError, pushFailureTelemetryLabel, sanitizePushFailureText, } from "../lib/describe-error.js";
+import { RoutePushFailureBackoff, systemPushRetryScheduler, } from "./push-failure-backoff.js";
 import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
 import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
 import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
@@ -1690,7 +1691,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         refresh.unref?.();
     }
     const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
-    const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry) => {
+    const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
+        const passCheckpoint = pullSignal || receiverSignal
+            ? Object.assign(async () => checkpoint?.(), {
+                shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
+                shouldAbort: () => pullSignal?.aborted === true ||
+                    receiverSignal?.aborted === true ||
+                    checkpoint?.shouldAbort?.() === true,
+                runChoreNetworkTransfer: checkpoint?.runChoreNetworkTransfer,
+            })
+            : checkpoint;
         const reportedAppliedUpserts = new Set();
         const reportedAppliedDeletes = new Set();
         const recordDurableAppliedUpserts = (upserts) => {
@@ -1722,9 +1732,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             // existing scheduler's cooperative hand-off checkpoint.
             const onJournalsReady = () => activateEventPushSurfaces();
             if (!runPassSupportsQmdLifecycle) {
-                return runPass(passArgvForRun, checkpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
+                return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
             }
-            return runPass(passArgvForRun, checkpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
+            return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
         };
         // Observe the heap at pass boundaries — a gated checkpoint immediately before
         // admitting a pass and immediately after it returns. A pass that runs a long
@@ -1987,13 +1997,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
     // chore ownership. Realtime normally waits for a chore hand-off; the
     // watcher fast path explicitly opts into its established scoped exception.
     passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowConcurrentHandoff, transferDirection, transferSizeClass);
-    const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false) => {
+    const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal) => {
         telemetry?.mark("operationGate");
+        const admissionSignal = signal && receiverSignal
+            ? AbortSignal.any([signal, receiverSignal])
+            : signal ?? receiverSignal;
         return runGuardedTask((context) => {
             telemetry?.mark("appendLock");
             telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
-            return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry);
-        }, lane, workClass, allowFastRealtimeOvertake, signal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass);
+            return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
+        }, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass);
     };
     /*
      * Area storage and the parallel coordinator default to AUTO. AUTO turns
@@ -2465,15 +2478,43 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         pendingWatcherBatchGeneration = null;
         return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
     };
-    const restorePendingWatcherPaths = (paths, generation) => {
+    const restorePendingWatcherPaths = (paths, generation, changes) => {
         pendingWatcherBatchGeneration ??=
             generation ?? ++nextWatcherBatchGeneration;
         pendingWatcherOriginalBatch = null;
         for (const [absolutePath, relativePath] of paths.entries()) {
+            // A newer watcher event may already be queued for this path. Keep its
+            // change record instead of restoring stale delete metadata from the
+            // batch that just finished.
+            if (pendingWatcherChangeSequences.has(absolutePath))
+                continue;
             pendingWatcherPaths.set(absolutePath, relativePath);
             pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
+            const change = changes?.get(absolutePath);
+            if (change)
+                pendingWatcherChanges.set(absolutePath, change);
         }
     };
+    const routePushFailureBackoff = new RoutePushFailureBackoff();
+    const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
+    const routePushRetryTimerHandles = new Map();
+    const clearRoutePushRetry = (route) => {
+        const handle = routePushRetryTimerHandles.get(route);
+        if (handle === undefined)
+            return;
+        pushRetryScheduler.clearTimeout(handle);
+        routePushRetryTimerHandles.delete(route);
+    };
+    const scheduleRoutePushRetry = (route, delayMs) => {
+        if (routePushRetryTimerHandles.has(route))
+            return;
+        const handle = pushRetryScheduler.setTimeout(() => {
+            routePushRetryTimerHandles.delete(route);
+            if (!stopped)
+                interruptWait();
+        }, delayMs);
+        routePushRetryTimerHandles.set(route, handle);
+    };
     /**
      * Journal roots the bulk-intent quarantine has refused for the current
      * delete episode, keyed `slug\0journalKey`.
@@ -3332,7 +3373,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         }
         return false;
     };
-    const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
+    let emitScopedPushFailureTelemetry = null;
+    const runScopedDrain = async (batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
         if (batch)
             await resolveDeleteSnapshotCaptures(batch);
         const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
@@ -3343,6 +3385,52 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             const relativePath = group.paths.get(absolutePath);
             return relativePath !== undefined && admittedPaths.some((scope) => relativePath === scope || relativePath.startsWith(`${scope}/`));
         }));
+        const pushFailureText = (result, route, caughtError) => {
+            if (caughtError !== undefined)
+                return describeError(caughtError);
+            if (typeof result === "number")
+                return `sync pass exited ${result}`;
+            const failures = [
+                ...(result.result?.companyFailures ?? []),
+                ...(result.result?.errors ?? []),
+                ...(result.result?.transient ?? []),
+            ];
+            const routeFailure = route.kind === "company"
+                ? failures.find((failure) => failure.company.toLowerCase() === route.slug.toLowerCase())
+                : undefined;
+            return routeFailure?.message ?? failures[0]?.message ??
+                `sync pass exited ${result.exitCode}`;
+        };
+        const recordPushFailure = (route, exitCode, message) => {
+            const routeId = routeKey(route);
+            const companyRoot = route.kind === "company"
+                ? path.join(hqRoot, "companies", route.slug)
+                : hqRoot;
+            const errorText = sanitizePushFailureText(message, companyRoot);
+            const state = routePushFailureBackoff.failed(routeId, monotonicNow());
+            scheduleRoutePushRetry(routeId, state.delayMs);
+            if (!state.shouldReport)
+                return;
+            const event = {
+                route: routeId,
+                exitCode,
+                consecutiveFailures: state.consecutiveFailures,
+                nextRetryDelayMs: state.delayMs,
+                errorSummary: pushFailureTelemetryLabel(errorText),
+            };
+            const diagnostic = {
+                type: "scoped-push-failure",
+                ...event,
+                error: errorText,
+            };
+            (deps.stderr ?? process.stderr).write(`${JSON.stringify(diagnostic)}\n`);
+            if (deps.emitScopedPushFailureTelemetry) {
+                deps.emitScopedPushFailureTelemetry(event);
+            }
+            else {
+                emitScopedPushFailureTelemetry?.(event);
+            }
+        };
         if (watcherDrainRunsPush && grouped) {
             for (const group of grouped.values()) {
                 // Paused workspaces: drop the scoped push without uploading or
@@ -3350,6 +3438,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 if (isPausedRoute(group.route)) {
                     continue;
                 }
+                const routeId = routeKey(group.route);
+                const retryState = routePushFailureBackoff.state(routeId);
+                if (retryState && retryState.retryAt > monotonicNow()) {
+                    scheduleRoutePushRetry(routeId, retryState.retryAt - monotonicNow());
+                    restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
+                    continue;
+                }
+                clearRoutePushRetry(routeId);
                 try {
                     const sessionLogBackup = group.relPaths.every(isSessionLogsPushOnlyPath);
                     const lane = sessionLogBackup
@@ -3384,6 +3480,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                     rememberUploadedPaths(result);
                     const pushExit = passExitCode(result);
                     if (execution.accepted && pushExit === 0) {
+                        routePushFailureBackoff.succeeded(routeId);
+                        clearRoutePushRetry(routeId);
                         const pushPathResults = typeof result === "number" ? [] : (result.result?.pushPathResults ?? []);
                         const refusedPaths = new Set(pushPathResults
                             .filter((pathResult) => pathResult.status === "refused")
@@ -3431,21 +3529,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                         }
                     }
                     else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
+                        routePushFailureBackoff.succeeded(routeId);
+                        clearRoutePushRetry(routeId);
                         if (group.route.kind === "company") {
                             logDroppedUnresolvedWatcherRoute(group.route.slug);
                         }
                     }
                     else {
-                        restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
-                        if (wakeOnPushFailure)
+                        restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
+                        if (pushExit !== 0) {
+                            recordPushFailure(group.route, pushExit, pushFailureText(result, group.route));
+                        }
+                        else if (_wakeOnPushFailure) {
                             interruptWait();
+                        }
                     }
                 }
                 catch (err) {
-                    restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
-                    process.stderr.write(`watch scoped push failed, will retry next tick: ${describeError(err)}\n`);
-                    if (wakeOnPushFailure)
-                        interruptWait();
+                    recordPushFailure(group.route, 1, pushFailureText(1, group.route, err));
+                    restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
                 }
             }
         }
@@ -3534,6 +3636,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                     if (watcherDrainRunsPush) {
                         const push = await runGuarded(buildScopedPushArgv(route, [], passArgv));
                         const pushExit = scopedRouteDrainExit({ result: push, accepted: true });
+                        if (pushExit === 0) {
+                            const routeId = routeKey(route);
+                            routePushFailureBackoff.succeeded(routeId);
+                            clearRoutePushRetry(routeId);
+                        }
                         if (pushExit !== 0)
                             return push;
                     }
@@ -3562,10 +3669,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             if (watcherDrainRunsPush) {
                 // Whole-route push (no --scope-path): the dropped paths are unknown,
                 // so the route's full walk is the narrowest sound scope.
+                const push = await runGuarded(buildScopedPushArgv(route, [], passArgv));
                 const exit = scopedRouteDrainExit({
-                    result: await runGuarded(buildScopedPushArgv(route, [], passArgv)),
+                    result: push,
                     accepted: true,
                 });
+                if (exit === 0) {
+                    const routeId = routeKey(route);
+                    routePushFailureBackoff.succeeded(routeId);
+                    clearRoutePushRetry(routeId);
+                }
                 if (exit !== 0 && worstExit === 0)
                     worstExit = exit;
             }
@@ -3631,6 +3744,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             authToken: getAccessToken,
             region: runtime.region,
         });
+        emitScopedPushFailureTelemetry = (event) => {
+            void emitCloudTelemetry(watcherTelemetryClient, {
+                eventName: "sync_runner_scoped_push_failed",
+                source: "sync_runner_watch_loop",
+                properties: event,
+            }, {
+                claims,
+                log: (message) => (deps.stderr ?? process.stderr).write(`hq-sync-runner: ${message}\n`),
+            });
+        };
         const createWatcher = deps.createWatcher ??
             ((opts) => new TreeWatcher({
                 hqRoot: opts.hqRoot,
@@ -3862,7 +3985,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         // a known-small event can be slow solely because an earlier event exhausted
         // the delivery-wide fast budget.
         const receiverLaneByBatch = new WeakMap();
-        const drainParallelReceiverLane = async (lane, laneEvents, receiveCountsByPath) => {
+        const drainParallelReceiverLane = async (lane, laneEvents, receiveCountsByPath, settleEvents, receiverSignal) => {
             const eventsByRoute = new Map();
             for (const event of laneEvents) {
                 const route = routeChangeToTarget(event.relativePath);
@@ -3877,15 +4000,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             // particular, a later route's fast receipt must never wait for an
             // earlier route's slow pull to finish.
             const routeGroups = [...eventsByRoute.values()];
-            const completed = await Promise.all(routeGroups.map(({ route, events }) => {
-                return runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
+            const settlements = await Promise.allSettled(routeGroups.map(async ({ route, events }) => {
+                const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
                     const slicePaths = slice.paths.map((path) => path.canonicalPath);
                     const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
-                    return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced));
+                    return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
                 });
-            }));
-            for (const [index, execution] of completed.entries()) {
-                const { route, events } = routeGroups[index];
                 const pathCount = events.length;
                 const exitCode = passExitCode(execution.result);
                 if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
@@ -3893,14 +4013,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                         throw targetedReceiverDispatchError(execution.result, execution.accepted, runtime.authRequiredPassExit);
                     }
                     warnUnresolvedReceiverCompany(route, pathCount);
-                    continue;
                 }
-                if (!execution.accepted || exitCode !== 0) {
+                else if (!execution.accepted || exitCode !== 0) {
                     throw targetedReceiverDispatchError(execution.result, execution.accepted, runtime.authRequiredPassExit);
                 }
-            }
+                // The receiver can acknowledge this tenant's receipts now, while
+                // independent route groups continue to settle.
+                await settleEvents?.(events);
+            }));
+            const failedSettlement = settlements.find((settlement) => settlement.status === "rejected");
+            if (failedSettlement)
+                throw failedSettlement.reason;
         };
-        const receiverSyncBatchFn = async ({ events, approximateReceiveCountsByPath, }) => {
+        const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
             if (stopped)
                 return;
             // The injectable receiver seam and the production SQS receiver both
@@ -3918,19 +4043,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 const lane = receiverLaneByBatch.get(events);
                 if (lane) {
                     receiverLaneByBatch.delete(events);
-                    await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath);
+                    await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
                     return;
                 }
                 // Direct embedders may invoke the engine without first going through
                 // SqsPushReceiver. Preserve that seam while applying the same
                 // delivery-wide partition the receiver would expose.
                 const lanes = partitionByKnownSizes(events, receiverEventSize);
-                await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath);
-                await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath);
+                await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
+                await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
                 return;
             }
             const lanes = partitionByKnownSizes(events, receiverEventSize);
-            const drainLane = async (lane, laneEvents, receiveCountsByPath) => {
+            const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
                 const eventsByRoute = new Map();
                 for (const event of laneEvents) {
                     const route = routeChangeToTarget(event.relativePath);
@@ -3949,26 +4074,28 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 }
                 for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
                     const paths = routeEvents.map((event) => event.relativePath);
-                    const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime"));
+                    const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
                     const exitCode = passExitCode(result);
                     if (exitCode === UNRESOLVED_COMPANY_EXIT) {
                         if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
                             throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
                         }
                         warnUnresolvedReceiverCompany(route, routeEvents.length);
+                        await settleEvents?.(routeEvents);
                         continue;
                     }
                     if (exitCode !== 0) {
                         throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
                     }
+                    await settleEvents?.(routeEvents);
                 }
             };
             // Dispatch realtime work before awaiting its bulk residue. This lets an
             // established slow pass continue while the small scoped pull uses its
             // ref-counted lock carve-out; the residue remains serialized and cannot
             // starve the slow lane.
-            await drainLane("fast", lanes.fast, approximateReceiveCountsByPath);
-            await drainLane("slow", lanes.slow, approximateReceiveCountsByPath);
+            await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
+            await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
         };
         receiverSyncBatchFn.partition = (events, maxPaths) => {
             const lanes = partitionByKnownSizes(events, receiverEventSize);
@@ -4110,6 +4237,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         if (stopped)
             return shutdownPromise ?? Promise.resolve();
         stopped = true;
+        for (const route of routePushRetryTimerHandles.keys()) {
+            clearRoutePushRetry(route);
+        }
         qmdShutdownController.abort();
         eventSyncShutdownDeadlineMs =
             Date.now() + EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS;