@indigoai-us/hq-cloud 6.18.14-alpha.2 → 6.18.14

package/dist/bin/sync-runner-watch-loop.js

@@ -35,6 +35,7 @@ import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
 import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, scopePathForRoute, } from "./sync-runner-watch-routes.js";
 import { resolveSkipCompanies } from "./sync-runner-planning.js";
 const FULL_RECONCILE_EVERY_TICKS = 10;
+const warnedUnresolvedReceiverCompanies = new Set();
 /** Maximum time shutdown waits for event-sync startup before abandoning it. */
 export const EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS = 5_000;
 /** A deferred full chore must get the next admission after this long. */
@@ -1318,6 +1319,17 @@ function isBulkIntentOverride() {
 function passExitCode(outcome) {
     return typeof outcome === "number" ? outcome : outcome.exitCode;
 }
+export function shouldAcknowledgeParallelUnresolvedCompany(accepted, exitCode) {
+    return accepted && exitCode === UNRESOLVED_COMPANY_EXIT;
+}
+function unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath) {
+    return events.every((event) => {
+        const receiveCounts = receiveCountsByPath?.get(event.relativePath);
+        return receiveCounts !== undefined &&
+            receiveCounts.length > 0 &&
+            receiveCounts.every((count) => count !== undefined && Number.isFinite(count) && count >= 2);
+    });
+}
 function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
     const exitCode = passExitCode(outcome);
     const result = typeof outcome === "number" ? undefined : outcome.result;
@@ -1376,18 +1388,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
     const stdout = deps.stdout ?? process.stdout;
     const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
-    const writeMaintenanceProgress = () => {
-        stdout.write(JSON.stringify({
-            type: "maintenance-progress",
-            company: "watch",
-            bytesProcessed: 0,
-            totalBytes: 0,
-        }) + "\n");
-    };
     const monotonicNow = deps.monotonicNow ?? (() => performance.now());
     const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
-    const qmdShutdownController = new AbortController();
-    const runPassSupportsQmdLifecycle = deps.runPass === undefined || deps.runPassSupportsQmdLifecycle === true;
     const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
     const pollIdx = argv.indexOf("--poll-remote-ms");
     // An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
@@ -1595,20 +1597,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             }
         };
         const invokePass = () => {
-            if (!runPassSupportsCooperativeCheckpoint) {
-                if (!runPassSupportsQmdLifecycle)
-                    return runPass(passArgvForRun);
-                return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
-            }
+            if (!runPassSupportsCooperativeCheckpoint)
+                return runPass(passArgvForRun);
             // This callback is emitted by the production runner after journal
             // repair/seed and before fanout. It starts ingress early, but never
             // admits a second durable writer: queued realtime still waits for the
             // existing scheduler's cooperative hand-off checkpoint.
-            const onJournalsReady = () => activateEventPushSurfaces();
-            if (!runPassSupportsQmdLifecycle) {
-                return runPass(passArgvForRun, checkpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
-            }
-            return runPass(passArgvForRun, checkpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
+            return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
         };
         // Observe the heap at pass boundaries — a gated checkpoint immediately before
         // admitting a pass and immediately after it returns. A pass that runs a long
@@ -1801,14 +1796,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         resolveInterruptWait = null;
         resolve?.();
     };
-    let reportWatcherLiveness = () => undefined;
     const waitForNextPass = async (delayMs) => {
         if (interruptGeneration !== observedInterruptGeneration) {
             observedInterruptGeneration = interruptGeneration;
             return;
         }
         const heartbeatTimer = setInterval(() => {
-            writeMaintenanceProgress();
+            stdout.write(JSON.stringify({
+                type: "maintenance-progress",
+                company: "watch",
+                bytesProcessed: 0,
+                totalBytes: 0,
+            }) + "\n");
             reportWatcherLiveness();
         }, idleHeartbeatIntervalMs);
         heartbeatTimer.unref?.();
@@ -2109,7 +2108,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
      * liveness heartbeat as well so an operator can distinguish a live watcher
      * from a long-running poll-only process without inspecting /proc.
      */
-    reportWatcherLiveness = () => {
+    const reportWatcherLiveness = () => {
         if (!eventPush || stopped)
             return;
         reattemptWatcherDiscoveryIfDue();
@@ -3308,6 +3307,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
     };
     let receiver = null;
     let receiverHasQueuedWork = false;
+    const warnUnresolvedReceiverCompany = (route, pathCount) => {
+        if (route.kind !== "company" ||
+            warnedUnresolvedReceiverCompanies.has(route.slug))
+            return;
+        warnedUnresolvedReceiverCompanies.add(route.slug);
+        (deps.stderr ?? process.stderr).write(`${JSON.stringify({
+            level: "warn",
+            event: "receiver.unresolved-company",
+            companySlug: route.slug,
+            pathCount,
+        })}\n`);
+    };
     const onReceiveActivity = (hasMessages) => {
         receiverHasQueuedWork = hasMessages;
         if (hasMessages)
@@ -3555,7 +3566,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         // a known-small event can be slow solely because an earlier event exhausted
         // the delivery-wide fast budget.
         const receiverLaneByBatch = new WeakMap();
-        const drainParallelReceiverLane = async (lane, laneEvents) => {
+        const drainParallelReceiverLane = async (lane, laneEvents, receiveCountsByPath) => {
             const eventsByRoute = new Map();
             for (const event of laneEvents) {
                 const route = routeChangeToTarget(event.relativePath);
@@ -3569,20 +3580,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             // Submit every route in this lane before awaiting any result. In
             // particular, a later route's fast receipt must never wait for an
             // earlier route's slow pull to finish.
-            const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => {
+            const routeGroups = [...eventsByRoute.values()];
+            const completed = await Promise.all(routeGroups.map(({ route, events }) => {
                 return runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
                     const slicePaths = slice.paths.map((path) => path.canonicalPath);
                     const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
                     return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced));
                 });
             }));
-            for (const execution of completed) {
-                if (!execution.accepted || passExitCode(execution.result) !== 0) {
+            for (const [index, execution] of completed.entries()) {
+                const { route, events } = routeGroups[index];
+                const pathCount = events.length;
+                const exitCode = passExitCode(execution.result);
+                if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
+                    if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
+                        throw targetedReceiverDispatchError(execution.result, execution.accepted, runtime.authRequiredPassExit);
+                    }
+                    warnUnresolvedReceiverCompany(route, pathCount);
+                    continue;
+                }
+                if (!execution.accepted || exitCode !== 0) {
                     throw targetedReceiverDispatchError(execution.result, execution.accepted, runtime.authRequiredPassExit);
                 }
             }
         };
-        const receiverSyncBatchFn = async ({ events }) => {
+        const receiverSyncBatchFn = async ({ events, approximateReceiveCountsByPath, }) => {
             if (stopped)
                 return;
             // The injectable receiver seam and the production SQS receiver both
@@ -3600,19 +3622,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 const lane = receiverLaneByBatch.get(events);
                 if (lane) {
                     receiverLaneByBatch.delete(events);
-                    await drainParallelReceiverLane(lane, events);
+                    await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath);
                     return;
                 }
                 // Direct embedders may invoke the engine without first going through
                 // SqsPushReceiver. Preserve that seam while applying the same
                 // delivery-wide partition the receiver would expose.
                 const lanes = partitionByKnownSizes(events, receiverEventSize);
-                await drainParallelReceiverLane("fast", lanes.fast);
-                await drainParallelReceiverLane("slow", lanes.slow);
+                await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath);
+                await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath);
                 return;
             }
             const lanes = partitionByKnownSizes(events, receiverEventSize);
-            const drainLane = async (lane, laneEvents) => {
+            const drainLane = async (lane, laneEvents, receiveCountsByPath) => {
                 const eventsByRoute = new Map();
                 for (const event of laneEvents) {
                     const route = routeChangeToTarget(event.relativePath);
@@ -3632,7 +3654,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
                     const paths = routeEvents.map((event) => event.relativePath);
                     const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime"));
-                    if (passExitCode(result) !== 0) {
+                    const exitCode = passExitCode(result);
+                    if (exitCode === UNRESOLVED_COMPANY_EXIT) {
+                        if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
+                            throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
+                        }
+                        warnUnresolvedReceiverCompany(route, routeEvents.length);
+                        continue;
+                    }
+                    if (exitCode !== 0) {
                         throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
                     }
                 }
@@ -3641,23 +3671,21 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             // established slow pass continue while the small scoped pull uses its
             // ref-counted lock carve-out; the residue remains serialized and cannot
             // starve the slow lane.
-            await drainLane("fast", lanes.fast);
-            await drainLane("slow", lanes.slow);
+            await drainLane("fast", lanes.fast, approximateReceiveCountsByPath);
+            await drainLane("slow", lanes.slow, approximateReceiveCountsByPath);
         };
-        receiverSyncBatchFn.partition = (events, maxPaths) => {
+        receiverSyncBatchFn.partition = (events) => {
             const lanes = partitionByKnownSizes(events, receiverEventSize);
-            const partitionMaxPaths = Math.max(1, maxPaths ?? events.length);
-            return ["fast", "slow"].flatMap((lane) => {
-                const laneEvents = lanes[lane];
-                const chunks = [];
-                for (let offset = 0; offset < laneEvents.length; offset += partitionMaxPaths) {
-                    const chunk = laneEvents.slice(offset, offset + partitionMaxPaths);
-                    if (parallelCoordinatorEnabled)
-                        receiverLaneByBatch.set(chunk, lane);
-                    chunks.push(chunk);
-                }
-                return chunks;
-            });
+            if (parallelCoordinatorEnabled) {
+                return ["fast", "slow"].flatMap((lane) => {
+                    const laneEvents = lanes[lane];
+                    if (laneEvents.length === 0)
+                        return [];
+                    receiverLaneByBatch.set(laneEvents, lane);
+                    return [laneEvents];
+                });
+            }
+            return [lanes.fast, lanes.slow].filter((lane) => lane.length > 0);
         };
         const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
         receiver = createReceiver({
@@ -3784,7 +3812,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         if (stopped)
             return shutdownPromise ?? Promise.resolve();
         stopped = true;
-        qmdShutdownController.abort();
         eventSyncShutdownDeadlineMs =
             Date.now() + EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS;
         resolveStoppedQueue();
@@ -3977,7 +4004,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 if (recycleRequested !== null)
                     return;
                 recycleRequested = record;
-                qmdShutdownController.abort();
                 // Additive, non-error NDJSON protocol event for the desktop supervisor.
                 stdout.write(`${JSON.stringify({ type: "heap-recycle", ...record })}\n`);
                 // Compact numeric line, worded to match none of hq-desktop-app's fatal