@indigoai-us/hq-cloud 6.15.80 → 6.15.81

package/dist/bin/sync-runner-watch-loop.js

@@ -15,6 +15,9 @@ import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watche
 import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournals, readJournal, writeJournal, } from "../journal.js";
 import { NoopPushReceiver, } from "../sync/push-receiver.js";
 import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
+import { isParallelCoordinatorEnabled } from "../sync/feature-flags.js";
+import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
+import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
 import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
 import { resolveSkipCompanies } from "./sync-runner-planning.js";
 const FULL_RECONCILE_EVERY_TICKS = 10;
@@ -62,6 +65,16 @@ export const DEFAULT_FAST_LANE_MAX_BYTES = 4 * 1024 * 1024;
  * leaving session logs and other transfer-heavy files in the slow lane.
  */
 export const DEFAULT_FAST_LANE_MAX_FILE_BYTES = 256 * 1024;
+/**
+ * A coordinator submission must not contain both a directory and one of its
+ * descendants: they are one mutation scope, and the lease manager correctly
+ * rejects internally-conflicting submissions. Prefer the widest scope so an
+ * unlinkDir following child unlink events remains admissible.
+ */
+export function normalizeCoordinatorPaths(paths) {
+    const ordered = [...new Set(paths)].sort((left, right) => left.length - right.length || left.localeCompare(right));
+    return ordered.filter((candidate, index) => !ordered.slice(0, index).some((ancestor) => candidate.startsWith(`${ancestor}/`)));
+}
 function positiveEnvInt(name, fallback) {
     const value = Number.parseInt(process.env[name] ?? "", 10);
     return Number.isFinite(value) && value > 0 ? value : fallback;
@@ -443,6 +456,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
     // Fast work may bypass the lock only while this process *actually owns* it;
     // otherwise it must participate in normal lock acquisition.
     let processOwnsOperationLock = false;
+    // Compatibility chores still mutate whole routes/vaults. While one is
+    // executing, a coordinator realtime task must not take the normal fast-path
+    // lock carve-out: it has no bounded chore lease to prove independence.
+    let coordinatorWholeMutationChores = 0;
     let operationLockUsers = 0;
     let resolveNoOperationLockUsers = null;
     const acquireOperationLockUser = () => {
@@ -469,7 +486,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         // its keyed journal delta and per-object mutation are independent from the
         // slow pass's bulk paths, and the slow pass already owns the root lock.
         // Never bypass the lock when no slow pass is active.
-        if (lane === "fast" && processOwnsOperationLock) {
+        if (lane === "fast" &&
+            processOwnsOperationLock &&
+            coordinatorWholeMutationChores === 0) {
             acquireOperationLockUser();
             try {
                 prepare?.();
@@ -731,6 +750,109 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         });
     };
     const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
+    /*
+     * The bridge is intentionally opt-in.  The legacy guarded pass arrays stay
+     * exactly as they are until an outpost enables HQ_SYNC_PARALLEL_COORDINATOR.
+     * Once enabled, the coordinator owns admission and path fencing; the small
+     * inline adapters are only the compatibility execution seam while the
+     * runner's real slice executors are being moved into child workers.
+     */
+    const parallelCoordinatorEnabled = isParallelCoordinatorEnabled(process.env.HQ_SYNC_PARALLEL_COORDINATOR);
+    const coordinatorExecutions = new WeakMap();
+    const coordinatorWorker = async (slice) => {
+        const execution = coordinatorExecutions.get(slice.work);
+        if (!execution) {
+            return {
+                paths: slice.paths.map((item) => ({
+                    path: item.canonicalPath,
+                    epoch: item.epoch,
+                    status: "retryable",
+                    reason: "coordinator execution was not registered",
+                })),
+            };
+        }
+        if (slice.work.class === "chore")
+            coordinatorWholeMutationChores += 1;
+        try {
+            execution.result = await execution.run(slice);
+        }
+        finally {
+            if (slice.work.class === "chore")
+                coordinatorWholeMutationChores -= 1;
+        }
+        const exitCode = passExitCode(execution.result);
+        return {
+            paths: slice.paths.map((item) => ({
+                path: item.canonicalPath,
+                epoch: item.epoch,
+                status: exitCode === 0 ? "durable" : "retryable",
+                ...(exitCode === 0
+                    ? {}
+                    : { reason: `sync pass exited ${exitCode}` }),
+            })),
+        };
+    };
+    const coordinator = parallelCoordinatorEnabled
+        ? new DefaultParallelSyncCoordinator({
+            realtimeWorker: new InlineSyncWorkerAdapter(coordinatorWorker),
+            choreWorker: new InlineSyncWorkerAdapter(coordinatorWorker),
+            identifyPath: (canonicalPath) => {
+                const route = routeChangeToTarget(canonicalPath);
+                if (!route)
+                    throw new Error(`coordinator cannot route ${canonicalPath}`);
+                return route.kind === "company"
+                    ? { targetUid: `company:${route.slug}`, areaId: `company:${route.slug}` }
+                    : { targetUid: "personal", areaId: "personal" };
+            },
+            // Compatibility chores still use whole-vault/whole-route executors,
+            // which cannot yet name sound bounded mutation paths. Their execution
+            // therefore remains behind the runner's guarded operation queue until
+            // those executors can provide concrete scopes.
+            planChorePaths: () => [],
+        })
+        : null;
+    const runCoordinatorWork = async (work, task) => {
+        if (!coordinator) {
+            return {
+                result: await task({
+                    id: "legacy-direct",
+                    work,
+                    paths: [],
+                    signal: new AbortController().signal,
+                }),
+                accepted: true,
+            };
+        }
+        const execution = { run: task };
+        coordinatorExecutions.set(work, execution);
+        try {
+            const outcome = await coordinator.submit(work);
+            const accepted = outcome.paths.every((path) => path.status === "durable");
+            return {
+                result: execution.result ?? 1,
+                accepted,
+            };
+        }
+        finally {
+            coordinatorExecutions.delete(work);
+        }
+    };
+    let nextCoordinatorCausality = 0;
+    const realtimeWork = (source, direction, paths) => ({
+        class: "realtime",
+        source,
+        direction,
+        paths: normalizeCoordinatorPaths(paths),
+        causality: `${source}-${++nextCoordinatorCausality}`,
+    });
+    const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
+    const runRealtimePass = (source, direction, paths, task) => runCoordinatorWork(realtimeWork(source, direction, paths), task);
+    const enqueueChore = (kind, task, onSettled) => {
+        void runCoordinatorWork(choreWork(kind), task).then(onSettled).catch((error) => {
+            process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
+            onSettled({ result: 1, accepted: false });
+        });
+    };
     let watcher = null;
     let eventPushSurfacesActivated = false;
     let watcherUnavailableWarned = false;
@@ -1417,6 +1539,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         const skipCompanies = grouped ? resolveSkipCompanies() : null;
         const isPausedRoute = (route) => route.kind === "company" &&
             (skipCompanies?.has(route.slug.toLowerCase()) ?? false);
+        const changesForAdmittedPaths = (group, admittedPaths) => new Map([...group.changes.entries()].filter(([absolutePath]) => {
+            const relativePath = group.paths.get(absolutePath);
+            return relativePath !== undefined && admittedPaths.some((scope) => relativePath === scope || relativePath.startsWith(`${scope}/`));
+        }));
         if (watcherDrainRunsPush && grouped) {
             for (const group of grouped.values()) {
                 // Paused workspaces: drop the scoped push without uploading or
@@ -1424,15 +1550,24 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 if (isPausedRoute(group.route)) {
                     continue;
                 }
-                const scopedArgv = buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots);
                 try {
                     const lane = isFastLocalBatch(group.paths.keys(), group.changes)
                         ? "fast"
                         : "slow";
-                    const result = await runGuarded(scopedArgv, group.changes.size > 0
-                        ? () => prepareWatcherChanges(group.changes)
-                        : undefined, lane);
-                    if (passExitCode(result) === 0) {
+                    const execution = parallelCoordinatorEnabled
+                        ? await runRealtimePass("watcher", "push", group.relPaths, async (slice) => {
+                            const admittedPaths = slice.paths.map((path) => path.canonicalPath);
+                            const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
+                            const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
+                            return runGuarded(scopedArgv, admittedChanges.size > 0
+                                ? () => prepareWatcherChanges(admittedChanges)
+                                : undefined, lane);
+                        })
+                        : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
+                                ? () => prepareWatcherChanges(group.changes)
+                                : undefined, lane), accepted: true };
+                    const result = execution.result;
+                    if (execution.accepted && passExitCode(result) === 0) {
                         const refusedPaths = new Set(typeof result === "number"
                             ? []
                             : (result.result?.pushPathResults ?? [])
@@ -1492,7 +1627,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                         continue;
                     ranPersonalDrain = true;
                 }
-                const exit = passExitCode(await runGuarded(pullArgv, undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"));
+                const execution = parallelCoordinatorEnabled
+                    ? await runRealtimePass("watcher", "pull", group.relPaths, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"))
+                    : { result: await runGuarded(pullArgv, undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"), accepted: true };
+                const exit = execution.accepted ? passExitCode(execution.result) : 1;
                 if (exit !== 0 && worstExit === 0)
                     worstExit = exit;
             }
@@ -1527,6 +1665,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         for (const hint of batch.droppedRouteHints ?? [])
             addRoute(hint);
         const skipCompanies = resolveSkipCompanies();
+        if (parallelCoordinatorEnabled) {
+            for (const route of routes.values()) {
+                if (route.kind === "company" &&
+                    skipCompanies.has(route.slug.toLowerCase())) {
+                    continue;
+                }
+                enqueueChore("backlog", async () => {
+                    if (watcherDrainRunsPush) {
+                        const push = await runGuarded(buildScopedPushArgv(route, [], passArgv));
+                        if (passExitCode(push) !== 0)
+                            return push;
+                    }
+                    return drainRunsPull
+                        ? runGuarded(buildRoutePullArgv(route, passArgv))
+                        : 0;
+                }, (execution) => {
+                    if (!execution.accepted || passExitCode(execution.result) !== 0) {
+                        process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
+                        interruptWait();
+                    }
+                });
+            }
+            return 0;
+        }
         let worstExit = 0;
         for (const route of routes.values()) {
             // Paused workspaces sync in NO direction while Off.
@@ -1647,6 +1809,29 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 const lanes = selectedBatch && !selectedBatch.overflowed
                     ? partitionLocalBatch(selectedBatch)
                     : null;
+                if (parallelCoordinatorEnabled && lanes) {
+                    // Preserve the ingress partition/debounce, but both partitions now
+                    // submit semantic realtime work.  In particular a fresh large file
+                    // never waits for a whole-backlog chore merely because it exceeded
+                    // the old fast byte cap.
+                    for (const lane of [lanes.fast, lanes.slow]) {
+                        if (!lane)
+                            continue;
+                        const generation = ++nextWatcherBatchGeneration;
+                        void runScopedDrain(lane, generation, true).then((exit) => {
+                            if (exit === 0)
+                                return;
+                            restorePendingWatcherPaths(lane.paths, generation);
+                            interruptWait();
+                        }).catch((err) => {
+                            restorePendingWatcherPaths(lane.paths, generation);
+                            process.stderr.write(`watch coordinator scoped push failed, will retry next tick: ${describeError(err)}\n`);
+                            interruptWait();
+                        });
+                    }
+                    realtimeState.scheduler?.signal("watcher");
+                    return;
+                }
                 if (lanes?.fast &&
                     activeSlowPass !== null) {
                     const generation = ++nextWatcherBatchGeneration;
@@ -1768,6 +1953,28 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             // Apply the caps once across the whole delivery, then retain the residue
             // in the normal lane. The receiver already coalesces to one latest event
             // per path, so the partition cannot split revisions of one path.
+            if (parallelCoordinatorEnabled) {
+                // #423's coalescing/caps remain ingress protection only.  A newly
+                // delivered single large path is realtime work, not a slow-pass item;
+                // byte size is handled by the transfer budget below the coordinator.
+                const pathsByRoute = new Map();
+                for (const event of events) {
+                    const route = routeChangeToTarget(event.relativePath);
+                    if (!route || !isWatchRouteSelected(route, watchRouteSelection))
+                        continue;
+                    const key = routeKey(route);
+                    const group = pathsByRoute.get(key) ?? { route, paths: [] };
+                    group.paths.push(event.relativePath);
+                    pathsByRoute.set(key, group);
+                }
+                for (const { route, paths } of pathsByRoute.values()) {
+                    const execution = await runRealtimePass("receiver", "pull", paths, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, "slow"));
+                    if (!execution.accepted || passExitCode(execution.result) !== 0) {
+                        throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
+                    }
+                }
+                return;
+            }
             const lanes = partitionByKnownSizes(events, receiverEventSize);
             const drainLane = async (lane, laneEvents) => {
                 const pathsByRoute = new Map();
@@ -1800,6 +2007,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             await drainLane("slow", lanes.slow);
         };
         receiverSyncBatchFn.partition = (events) => {
+            if (parallelCoordinatorEnabled)
+                return events.length > 0 ? [events] : [];
             const lanes = partitionByKnownSizes(events, receiverEventSize);
             return [lanes.fast, lanes.slow].filter((lane) => lane.length > 0);
         };
@@ -1821,10 +2030,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 runGuarded: async (drain) => {
                     let outcome;
                     try {
-                        outcome = await runGuardedTask(() => runTaskWithLock(async () => {
-                            await drain();
-                            return 0;
-                        }));
+                        if (parallelCoordinatorEnabled) {
+                            const execution = await runRealtimePass("wake", "pull", [], async () => runGuardedTask(() => runTaskWithLock(async () => {
+                                await drain();
+                                return 0;
+                            })));
+                            outcome = execution.accepted ? execution.result : 1;
+                        }
+                        else {
+                            outcome = await runGuardedTask(() => runTaskWithLock(async () => {
+                                await drain();
+                                return 0;
+                            }));
+                        }
                     }
                     catch (err) {
                         // Scheduler clients may have already been disposed when a queued
@@ -1899,11 +2117,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             });
         }
     }
+    let coordinatorShutdown = null;
     const shutdown = () => {
         if (stopped)
-            return;
+            return coordinatorShutdown ?? Promise.resolve();
         stopped = true;
         resolveStoppedQueue();
+        // Reject queued coordinator work before tearing down ingress.  Active
+        // work receives its abort signal; receiver receipts remain unacknowledged
+        // unless their durable pull has already completed.
+        coordinatorShutdown = coordinator?.shutdown() ?? Promise.resolve();
         try {
             void receiver?.dispose();
         }
@@ -1931,10 +2154,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             /* ignore */
         }
         resolveStopped?.();
+        return coordinatorShutdown;
     };
     const onShutdownSignal = deps.onShutdownSignal ??
         ((handler) => {
-            const wrapped = () => handler();
+            const wrapped = () => { void handler(); };
             process.on("SIGTERM", wrapped);
             process.on("SIGINT", wrapped);
             return () => {
@@ -1948,8 +2172,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
     let nextDueAt = monotonicNow();
     let pollTick = 0;
     let lastCompletedFullReconcileAt = null;
+    let fullChorePending = false;
+    let fullChoreFollowupPending = false;
+    let coordinatorAuthRequired = false;
     try {
         while (!stopped) {
+            if (coordinatorAuthRequired)
+                return 0;
             pollTick++;
             const pending = takePendingWatcherChange();
             // The quarantine is scoped to the delete episode it refused, and only
@@ -2020,10 +2249,58 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             };
             let result;
             let ranFullReconcile = false;
+            let enqueuedFullChore = false;
             if (shouldRunFull) {
-                ranFullReconcile = true;
                 emitDecision("full", reasons);
-                result = await runGuarded(passArgv);
+                if (parallelCoordinatorEnabled) {
+                    // Cadence/full/backlog work is admitted to the reserved chore worker
+                    // and deliberately never awaited by the watch loop.  The completion
+                    // callback, rather than submission, advances the full-reconcile
+                    // clock and turns on event surfaces.
+                    if (!fullChorePending) {
+                        fullChorePending = true;
+                        enqueueChore("full-reconcile", () => runGuarded(passArgv), (execution) => {
+                            fullChorePending = false;
+                            const exit = passExitCode(execution.result);
+                            if (exit === runtime.authRequiredPassExit) {
+                                coordinatorAuthRequired = true;
+                                interruptWait();
+                                return;
+                            }
+                            if (execution.accepted && exit === 0) {
+                                lastCompletedFullReconcileAt = monotonicNow();
+                                activateEventPushSurfaces();
+                                if (fullChoreFollowupPending) {
+                                    fullChoreFollowupPending = false;
+                                    interruptWait();
+                                }
+                                return;
+                            }
+                            process.stderr.write(`hq-sync-runner: full reconcile chore failed (exit=${exit}); will retry on cadence\n`);
+                            if (fullChoreFollowupPending) {
+                                fullChoreFollowupPending = false;
+                                interruptWait();
+                            }
+                        });
+                    }
+                    else {
+                        // A full pass can have already scanned this route. Preserve any
+                        // incoming evidence and make the completion schedule one more
+                        // reconcile rather than dropping the taken batch.
+                        if (pending.batch) {
+                            restorePendingWatcherPaths(pending.batch.paths, pending.generation);
+                        }
+                        if (pending.bare)
+                            pendingWatcherBareChange = true;
+                        fullChoreFollowupPending ||= pending.batch !== null || pending.bare;
+                    }
+                    enqueuedFullChore = true;
+                    result = 0;
+                }
+                else {
+                    ranFullReconcile = true;
+                    result = await runGuarded(passArgv);
+                }
             }
             else if (pending.batch === null) {
                 // There is no watcher batch and no full-reconcile trigger. Do not
@@ -2104,9 +2381,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             else if (exitCode !== 0) {
                 return exitCode;
             }
-            const completedPostRepairPass = typeof result === "number"
+            const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
                 ? exitCode === 0
-                : result.result !== undefined;
+                : result.result !== undefined);
             if (completedPostRepairPass) {
                 // The production adapter returns RunnerPassResult only after fanout,
                 // which follows journal repair. Its numeric seam has the same
@@ -2118,9 +2395,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             if (Date.now() >= nextBackupPruneAt) {
                 // Advance before invoking: a failed/unavailable backstop cannot become
                 // an every-tick retry loop. It is diagnostic only and never restarts
-                // the sync daemon. Keep it in the same deferred lock queue as every
-                // other watch task so an already-waiting rescue owns this handoff too.
+                // the sync daemon. Under coordinator mode it is a reserved-worker
+                // chore, never a dependency of realtime admission.
                 nextBackupPruneAt = Date.now() + 24 * 60 * 60 * 1000;
+                if (parallelCoordinatorEnabled) {
+                    enqueueChore("rotation", async () => runGuardedTask(() => runTaskWithLock(async () => {
+                        await runBackupPrune(hqRoot);
+                        return 0;
+                    })), (execution) => {
+                        if (!execution.accepted || passExitCode(execution.result) !== 0) {
+                            process.stderr.write("hq-sync-runner: backup retention chore failed; will retry tomorrow\n");
+                        }
+                    });
+                    continue;
+                }
                 try {
                     await runGuardedTask(() => runTaskWithLock(async () => {
                         await runBackupPrune(hqRoot);
@@ -2160,7 +2448,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         return 0;
     }
     finally {
-        shutdown();
+        await shutdown();
         detachSignal?.();
     }
 }