@indigoai-us/hq-cloud 6.18.91 → 6.18.93

package/dist/bin/sync-runner-watch-loop.js

@@ -3757,6 +3757,38 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         }
     }
     const routePushFailureBackoff = new RoutePushFailureBackoff();
+    /**
+     * Same 5 s doubling to 5 min schedule, for a route's scoped (watcher) pull.
+     * Kept apart from the push state so a failing pull never holds an upload
+     * and a failing push never resets the pull's step. Retry timers use the
+     * `pull:<route>` key in the shared scheduler.
+     */
+    const routePullFailureBackoff = new RoutePushFailureBackoff();
+    /**
+     * Routes with a scoped pull running now. Coordinator watcher lanes and the
+     * pending-batch drain can run concurrently; without this, both could pass
+     * the backoff check before either pull failed and run the same route's pull
+     * twice.
+     */
+    const scopedPullsInFlight = new Set();
+    /**
+     * Pending watcher paths whose push already succeeded and that wait only
+     * for their route's scoped pull (backing off, running, or deferred by
+     * admission). Later drains skip the push for them, so a route whose pull
+     * keeps failing does not re-run a no-op push pass on every wake. A fresh
+     * watcher event for a path removes it, so a new edit is still pushed.
+     */
+    const pullOnlyWatcherPaths = new Set();
+    /**
+     * One set per running scoped drain, holding paths that got a fresh watcher
+     * event while that drain ran. Each set lives only as long as its drain.
+     */
+    const activeDrainFreshPaths = new Set();
+    const noteFreshWatcherEvent = (absolutePath) => {
+        pullOnlyWatcherPaths.delete(absolutePath);
+        for (const freshPaths of activeDrainFreshPaths)
+            freshPaths.add(absolutePath);
+    };
     const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
     const routePushRetryTimerHandles = new Map();
     const clearRoutePushRetry = (route) => {
@@ -4612,7 +4644,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         return false;
     };
     let emitScopedPushFailureTelemetry = null;
-    const runScopedDrain = async (batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
+    const runScopedDrainOnce = async (freshDuringThisDrain, batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
         if (batch) {
             // Read each routed quarantine shard once for this watcher batch. Filter
             // before resolving delete snapshots or submitting work to the coordinator
@@ -4694,6 +4726,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 emitScopedPushFailureTelemetry?.(event);
             }
         };
+        /**
+         * Routes whose push did not run in this drain because the route's failure
+         * backoff held it. Their paths are back in the pending batch with a retry
+         * scheduled, so the paired scoped pull for the
+         * same paths waits for that retry. Without this, every wake of the loop
+         * (a receiver pull for any company, a retry timer, another watcher batch)
+         * re-took the restored paths, skipped the held push and still ran the
+         * pull leg. A route with a deterministic error then produced one errored
+         * runner session and `company_sync_leg_completed` per wake, all day,
+         * however long the push backoff was. A push that runs and fails still
+         * gets its pull in the same drain, as before, so a pull that would clear
+         * the failure is not starved: at most one pull per push attempt.
+         */
+        const routesAwaitingPushRetry = new Set();
         /**
          * The push never ran: a chore held the slot past the admission deadline.
          * Put the paths back and retry without counting a failure, so the route
@@ -4704,8 +4750,51 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
             scheduleRoutePushRetry(routeId, REALTIME_ADMISSION_RETRY_MS);
         };
+        /** Paths whose push succeeded (or needed nothing) in this drain. */
+        const pushSettledThisDrain = new Set();
+        /**
+         * Keep a group's paths pending for its pull only. A path qualifies when
+         * its push is already settled and no fresh watcher event arrived for it
+         * since this drain started.
+         */
+        const keepForPullOnly = (group) => {
+            for (const absolutePath of group.paths.keys()) {
+                if (freshDuringThisDrain.has(absolutePath))
+                    continue;
+                if (!watcherDrainRunsPush ||
+                    pushSettledThisDrain.has(absolutePath) ||
+                    pullOnlyWatcherPaths.has(absolutePath)) {
+                    pullOnlyWatcherPaths.add(absolutePath);
+                }
+            }
+            restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
+        };
+        /** The part of a group that still needs a push, or null when none does. */
+        const withoutPullOnlyPaths = (group) => {
+            const pushPaths = [...group.paths.entries()].filter(([absolutePath]) => !pullOnlyWatcherPaths.has(absolutePath));
+            if (pushPaths.length === group.paths.size)
+                return group;
+            if (pushPaths.length === 0)
+                return null;
+            const paths = new Map(pushPaths);
+            const changes = new Map([...group.changes.entries()].filter(([absolutePath]) => paths.has(absolutePath)));
+            return {
+                route: group.route,
+                paths,
+                changes,
+                relPaths: [...new Set(paths.values())],
+                deleteScopeRoots: [
+                    ...new Set([...changes.values()].flatMap((item) => (item.deleteSnapshots ?? []).flatMap((snapshot) => snapshot.deleteScopeRoot === undefined ? [] : [snapshot.deleteScopeRoot]))),
+                ],
+            };
+        };
         if (watcherDrainRunsPush && grouped) {
-            for (const group of grouped.values()) {
+            for (const routeGroup of grouped.values()) {
+                // Paths already pushed that wait only for their pull skip the push;
+                // the pull half below still covers them.
+                const group = withoutPullOnlyPaths(routeGroup);
+                if (group === null)
+                    continue;
                 // Paused workspaces: drop the scoped push without uploading or
                 // publishing event-sync metadata (path/hash must not leak while Off).
                 if (isPausedRoute(group.route)) {
@@ -4716,6 +4805,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 if (retryState && retryState.retryAt > monotonicNow()) {
                     scheduleRoutePushRetry(routeId, retryState.retryAt - monotonicNow());
                     restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
+                    routesAwaitingPushRetry.add(routeId);
                     continue;
                 }
                 clearRoutePushRetry(routeId);
@@ -4824,6 +4914,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                     if (execution.accepted && pushExit === 0) {
                         routePushFailureBackoff.succeeded(routeId);
                         clearRoutePushRetry(routeId);
+                        for (const absolutePath of group.paths.keys())
+                            pushSettledThisDrain.add(absolutePath);
                         const pushPathResults = typeof result === "number" ? [] : (result.result?.pushPathResults ?? []);
                         const refusedPaths = new Set(pushPathResults
                             .filter((pathResult) => pathResult.status === "refused")
@@ -4883,6 +4975,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                     }
                     else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
                         acknowledgeDurableWatcherUploads(group.relPaths);
+                        for (const absolutePath of group.paths.keys())
+                            pushSettledThisDrain.add(absolutePath);
                         routePushFailureBackoff.succeeded(routeId);
                         clearRoutePushRetry(routeId);
                         if (group.route.kind === "company") {
@@ -4928,6 +5022,39 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 // Paused workspaces sync in NO direction while Off.
                 if (isPausedRoute(group.route))
                     continue;
+                // The push for these paths is pending a scheduled retry; the pull
+                // runs with it then. Report the drain as deferred, not failed, so
+                // callers neither re-queue the paths again nor wake the loop early.
+                if (routesAwaitingPushRetry.has(routeKey(group.route))) {
+                    // The push half restored the paths it held. Paths that were
+                    // waiting only for this pull were not part of that push; keep them.
+                    keepForPullOnly(group);
+                    deferred = true;
+                    continue;
+                }
+                // A failed scoped pull backs off per route on the same schedule as
+                // the push. Until its retry is due, keep the paths pending and skip
+                // the pull. Before this, a failed pull made the callers restore the
+                // paths and wake the loop at once, so a route whose pull always
+                // failed re-ran it every few seconds for as long as the machine was
+                // awake (usage-only-pricing US-001).
+                const pullRouteId = routeKey(group.route);
+                const pullRetryKey = `pull:${pullRouteId}`;
+                const pullRetryState = routePullFailureBackoff.state(pullRouteId);
+                if (pullRetryState && pullRetryState.retryAt > monotonicNow()) {
+                    keepForPullOnly(group);
+                    scheduleRoutePushRetry(pullRetryKey, pullRetryState.retryAt - monotonicNow());
+                    deferred = true;
+                    continue;
+                }
+                if (scopedPullsInFlight.has(pullRouteId)) {
+                    // The running pull decides the backoff. Keep these paths and look
+                    // again after the admission retry interval.
+                    keepForPullOnly(group);
+                    scheduleRoutePushRetry(pullRetryKey, REALTIME_ADMISSION_RETRY_MS);
+                    deferred = true;
+                    continue;
+                }
                 const exactFileUpserts = [...group.paths.entries()].flatMap(([absolutePath, relativePath]) => {
                     const kind = group.changes.get(absolutePath)?.kind;
                     // Only create/change events are file upserts. Keep unlinks on the
@@ -4948,26 +5075,50 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 const lane = isFastLocalBatch(group.paths.keys(), group.changes)
                     ? "fast"
                     : "slow";
-                const execution = parallelCoordinatorEnabled
-                    ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
+                scopedPullsInFlight.add(pullRouteId);
+                const execution = await (parallelCoordinatorEnabled
+                    ? runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
                         const slicePaths = slice.paths.map((path) => path.canonicalPath);
                         return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route), undefined, coordinatorFenced === true);
                     })
-                    : await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
+                    : runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
                         if (error instanceof RealtimeAdmissionTimeoutError) {
                             return { accepted: false, admissionTimeout: error };
                         }
                         throw error;
-                    });
+                    })).finally(() => {
+                    scopedPullsInFlight.delete(pullRouteId);
+                });
                 if ("admissionTimeout" in execution && execution.admissionTimeout) {
                     // The pull never ran. Keep its paths and retry, without the
                     // pass-failure handling the shared exit code would get.
-                    restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
+                    keepForPullOnly(group);
                     scheduleRoutePushRetry(`pull:${routeKey(group.route)}`, REALTIME_ADMISSION_RETRY_MS);
                     deferred = true;
                     continue;
                 }
                 const exit = scopedRouteDrainExit(execution);
+                if (exit === 0) {
+                    routePullFailureBackoff.succeeded(pullRouteId);
+                    clearRoutePushRetry(pullRetryKey);
+                    for (const absolutePath of group.paths.keys())
+                        pullOnlyWatcherPaths.delete(absolutePath);
+                }
+                else if (exit !== runtime.authRequiredPassExit) {
+                    // Auth-required keeps its exit so the loop can park for sign-in.
+                    // Every other failure keeps the paths, schedules the retry and
+                    // reports the drain deferred, so no caller re-queues the paths or
+                    // wakes the loop early.
+                    const state = routePullFailureBackoff.failed(pullRouteId, monotonicNow());
+                    keepForPullOnly(group);
+                    scheduleRoutePushRetry(pullRetryKey, state.delayMs);
+                    if (state.shouldReport) {
+                        (deps.stderr ?? process.stderr).write(`hq-sync-runner: scoped pull for ${pullRouteId} failed (exit=${exit}, ` +
+                            `consecutive=${state.consecutiveFailures}); retrying in ${state.delayMs}ms\n`);
+                    }
+                    deferred = true;
+                    continue;
+                }
                 if (exit !== 0 && worstExit === 0)
                     worstExit = exit;
             }
@@ -4979,6 +5130,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         // when a peer change arrives between those reconciles.
         return 0;
     };
+    const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
+        const freshDuringThisDrain = new Set();
+        activeDrainFreshPaths.add(freshDuringThisDrain);
+        try {
+            return await runScopedDrainOnce(freshDuringThisDrain, batch, watcherBatchGeneration, wakeOnPushFailure, allowFastRealtimeOvertake);
+        }
+        finally {
+            activeDrainFreshPaths.delete(freshDuringThisDrain);
+        }
+    };
     /**
      * Overflow drain: the watcher hit its backlog cap, so per-path detail is
      * gone — but when every dropped path still carried route evidence
@@ -5219,6 +5380,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 catch {
                     /* diagnostics must never affect sync */
                 }
+                // A fresh event means the path needs a push again, even if it was
+                // waiting only for its route's pull.
+                if (batch) {
+                    for (const absolutePath of batch.paths.keys())
+                        noteFreshWatcherEvent(absolutePath);
+                }
+                else if (changedRelPath) {
+                    noteFreshWatcherEvent(localPathForVaultKey(hqRoot, changedRelPath));
+                }
                 // Peel a bounded realtime sub-batch from a mixed watcher batch while a
                 // bulk pass is in flight. The residue enters the ordinary pending drain
                 // immediately; a latest path appears in exactly one of these maps.