@indigoai-us/hq-cloud 6.14.49 → 6.15.0

package/src/bin/sync-runner-watch-loop.ts

@@ -1,6 +1,7 @@
 import * as os from "os";
 import * as path from "path";
 import * as fs from "fs";
+import { performance } from "node:perf_hooks";
 import { inventoryManagedSnapshots } from "../backup-prune.js";
 import { VaultClient } from "../index.js";
 import { pickCanonicalPersonEntity } from "../vault-client.js";
@@ -44,6 +45,7 @@ import {
   buildScopedDrainPullArgv,
   buildScopedPushArgv,
   buildTargetedPullArgv,
+  isWatchRouteSelected,
   routeChangeToTarget,
   routeKey,
   type WatchRoute,
@@ -91,9 +93,35 @@ export function adaptivePollMs(
   return Math.round(floorMs + ratio * (ceilMs - floorMs));
 }
 
+/**
+ * Advance the cadence from its previous monotonic deadline, rather than from
+ * the end of a pass. A slow pass is allowed one immediate catch-up pass; the
+ * returned deadline is always the first future slot, so missed slots cannot
+ * turn into an unbounded burst.
+ */
+export function advanceMonotonicDeadline(
+  previousDueAt: number,
+  now: number,
+  intervalMs: number,
+): { nextDueAt: number; delayMs: number } {
+  let nextDueAt = previousDueAt + intervalMs;
+  if (nextDueAt > now) {
+    return { nextDueAt, delayMs: nextDueAt - now };
+  }
+
+  // The pass crossed at least one deadline. Run exactly one catch-up now,
+  // then retain only the first future cadence slot.
+  do {
+    nextDueAt += intervalMs;
+  } while (nextDueAt <= now);
+  return { nextDueAt, delayMs: 0 };
+}
+
 export interface ParsedLoopArgs {
   hqRoot: string;
   lockTimeoutSec?: number;
+  /** Resolved parser flag; environment policy is applied by resolveSkipPersonal. */
+  skipPersonal?: boolean;
   /**
    * The run's resolved sync direction, straight from `parseArgs` (the single
    * source of truth — same default as the parser). The drain reads this to
@@ -220,6 +248,7 @@ export async function runWatchLoop(
   const sleep =
     deps.sleep ??
     ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const monotonicNow = deps.monotonicNow ?? (() => performance.now());
   const runPass =
     deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
   const pollIdx = argv.indexOf("--poll-remote-ms");
@@ -232,6 +261,22 @@ export async function runWatchLoop(
   const cpuCount = os.cpus().length;
   const eventPush = argv.includes("--event-push");
   const companiesMode = argv.includes("--companies");
+  const companyMode = argv.includes("--company");
+  const personalMode = argv.includes("--personal");
+  // Keep this at the watcher ingress rather than importing the runner's
+  // resolver: sync-runner imports this loop, and a reverse value import would
+  // create a runtime cycle. These are the parser's documented truthy values.
+  const skipPersonalEnv = (process.env.HQ_SYNC_SKIP_PERSONAL ?? "").toLowerCase();
+  const watchRouteSelection = {
+    companies: companiesMode,
+    company: companyMode,
+    personal: personalMode,
+    skipPersonal:
+      parsed.skipPersonal === true ||
+      skipPersonalEnv === "1" ||
+      skipPersonalEnv === "true" ||
+      skipPersonalEnv === "yes",
+  };
   const hqRoot = parsed.hqRoot;
   // Delayed and daily: this must never be attached to the 15s poll cadence.
   // Stage 0 remains dry-run only; the standalone hq-backup-prune command is
@@ -323,6 +368,45 @@ export async function runWatchLoop(
   };
 
   let stopped = false;
+  let resolveStopped: (() => void) | null = null;
+  const stoppedSignal = new Promise<void>((resolve) => {
+    resolveStopped = resolve;
+  });
+  // A signal can arrive while a guarded pass is running, before the next wait
+  // has registered. Keep a monotonic generation so that event is retained and
+  // consumed by the next scheduler wait instead of being lost.
+  let interruptGeneration = 0;
+  let observedInterruptGeneration = 0;
+  let resolveInterruptWait: (() => void) | null = null;
+  const interruptWait = (): void => {
+    if (stopped) return;
+    interruptGeneration += 1;
+    const resolve = resolveInterruptWait;
+    resolveInterruptWait = null;
+    resolve?.();
+  };
+  const waitForNextPass = async (delayMs: number): Promise<void> => {
+    if (interruptGeneration !== observedInterruptGeneration) {
+      observedInterruptGeneration = interruptGeneration;
+      return;
+    }
+
+    await Promise.race([
+      sleep(delayMs),
+      new Promise<void>((resolve) => {
+        resolveInterruptWait = resolve;
+        // Do not lose an event that arrived between the first generation
+        // check and registration of this wait.
+        if (interruptGeneration !== observedInterruptGeneration) {
+          resolveInterruptWait = null;
+          resolve();
+        }
+      }),
+      stoppedSignal,
+    ]);
+    resolveInterruptWait = null;
+    observedInterruptGeneration = interruptGeneration;
+  };
   let activePass: Promise<number | RunnerPassOutcome> | null = null;
   type QueuedPass = {
     argv: string[];
@@ -384,6 +468,13 @@ export async function runWatchLoop(
   const pendingWatcherPaths = new Map<string, string>();
   const pendingWatcherChanges = new Map<string, TreeChange>();
   let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
+  // Every coalesced watcher batch gets one durable in-process generation.
+  // A wake can cause the scheduler to revisit a drained batch, but an event
+  // announcement is an external side effect and must occur at most once for
+  // that batch after its targeted push succeeds.
+  let nextWatcherBatchGeneration = 0;
+  let pendingWatcherBatchGeneration: number | null = null;
+  const lastPublishedWatcherBatchGenerationByRoute = new Map<string, number>();
   let pendingWatcherBareChange = false;
   let pendingWatcherOverflowed = false;
   let pendingWatcherDroppedPaths = 0;
@@ -391,7 +482,30 @@ export async function runWatchLoop(
   const addPendingWatcherChange = (
     changedRelPath?: string,
     batch?: TreeChangeBatch,
-  ): void => {
+  ): boolean => {
+    if (batch) {
+      const paths = new Map<string, string>();
+      const changes = new Map<string, TreeChange>();
+      for (const [absolutePath, relativePath] of batch.paths.entries()) {
+        const route = routeChangeToTarget(relativePath);
+        if (!route || !isWatchRouteSelected(route, watchRouteSelection)) continue;
+        paths.set(absolutePath, relativePath);
+        const change = batch.changes?.get(absolutePath);
+        if (change) changes.set(absolutePath, change);
+      }
+      if (paths.size === 0) return false;
+      batch = {
+        ...batch,
+        paths,
+        changes,
+      };
+    } else if (changedRelPath) {
+      const route = routeChangeToTarget(changedRelPath);
+      if (!route || !isWatchRouteSelected(route, watchRouteSelection)) {
+        return false;
+      }
+    }
+    pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
     if (batch) {
       pendingWatcherOriginalBatch =
         pendingWatcherPaths.size === 0 &&
@@ -410,7 +524,7 @@ export async function runWatchLoop(
         pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
         pendingWatcherDroppedBytes += batch.droppedBytes ?? 0;
       }
-      return;
+      return true;
     }
     if (changedRelPath) {
       pendingWatcherOriginalBatch = null;
@@ -424,14 +538,16 @@ export async function runWatchLoop(
       // a prior batch left for this path so the stale intent cannot leak
       // into the coalesced batch (#227).
       pendingWatcherChanges.delete(absolutePath);
-      return;
+      return true;
     }
     pendingWatcherOriginalBatch = null;
     pendingWatcherBareChange = true;
+    return true;
   };
   const takePendingWatcherChange = (): {
     batch: TreeChangeBatch | null;
     bare: boolean;
+    generation: number | null;
   } => {
     const batch =
       pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
@@ -452,6 +568,7 @@ export async function runWatchLoop(
             }
           : null;
     const bare = pendingWatcherBareChange;
+    const generation = pendingWatcherBatchGeneration;
     pendingWatcherPaths.clear();
     pendingWatcherChanges.clear();
     pendingWatcherOriginalBatch = null;
@@ -459,9 +576,15 @@ export async function runWatchLoop(
     pendingWatcherOverflowed = false;
     pendingWatcherDroppedPaths = 0;
     pendingWatcherDroppedBytes = 0;
-    return { batch, bare };
+    pendingWatcherBatchGeneration = null;
+    return { batch, bare, generation };
   };
-  const restorePendingWatcherPaths = (paths: Map<string, string>): void => {
+  const restorePendingWatcherPaths = (
+    paths: Map<string, string>,
+    generation: number | null,
+  ): void => {
+    pendingWatcherBatchGeneration ??=
+      generation ?? ++nextWatcherBatchGeneration;
     pendingWatcherOriginalBatch = null;
     for (const [absolutePath, relativePath] of paths.entries()) {
       pendingWatcherPaths.set(absolutePath, relativePath);
@@ -512,7 +635,7 @@ export async function runWatchLoop(
     relativePath: string,
   ): { slug: string; key: string } | null => {
     const route = routeChangeToTarget(relativePath);
-    if (!route) return null;
+    if (!route || !isWatchRouteSelected(route, watchRouteSelection)) return null;
     if (route.kind === "personal") {
       return { slug: PERSONAL_VAULT_JOURNAL_SLUG, key: relativePath };
     }
@@ -786,14 +909,17 @@ export async function runWatchLoop(
     return grouped;
   };
 
-  const publishBatchIfAvailable = (paths: Map<string, string>): void => {
+  const publishBatchIfAvailable = (paths: Map<string, string>): boolean => {
     if (!stopped && eventSync && paths.size > 0) {
       eventSync.publishBatch({ paths: new Map(paths) });
+      return true;
     }
+    return false;
   };
 
   const runScopedDrain = async (
     batch: TreeChangeBatch | null,
+    watcherBatchGeneration: number | null,
   ): Promise<number> => {
     if (drainRunsPush && batch && batch.paths.size > 0) {
       const skipCompanies = resolveSkipCompanies();
@@ -828,26 +954,35 @@ export async function runWatchLoop(
                     .filter((pathResult) => pathResult.status === "refused")
                     .map((pathResult) => pathResult.relativePath),
             );
-            publishBatchIfAvailable(
-              new Map(
-                [...group.paths.entries()].filter(
-                  ([absolutePath, relativePath]) => {
-                    if (refusedPaths.has(relativePath)) return false;
-                    if (group.changes.get(absolutePath)?.kind !== "unlinkDir") {
-                      return true;
-                    }
-                    return ![...refusedPaths].some((refusedPath) =>
-                      refusedPath.startsWith(`${relativePath}/`),
-                    );
-                  },
+            const route = routeKey(group.route);
+            const generation = watcherBatchGeneration ?? 0;
+            if (
+              lastPublishedWatcherBatchGenerationByRoute.get(route) !== generation
+            ) {
+              const published = publishBatchIfAvailable(
+                new Map(
+                  [...group.paths.entries()].filter(
+                    ([absolutePath, relativePath]) => {
+                      if (refusedPaths.has(relativePath)) return false;
+                      if (group.changes.get(absolutePath)?.kind !== "unlinkDir") {
+                        return true;
+                      }
+                      return ![...refusedPaths].some((refusedPath) =>
+                        refusedPath.startsWith(`${relativePath}/`),
+                      );
+                    },
+                  ),
                 ),
-              ),
-            );
+              );
+              if (published) {
+                lastPublishedWatcherBatchGenerationByRoute.set(route, generation);
+              }
+            }
           } else {
-            restorePendingWatcherPaths(group.paths);
+            restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
           }
         } catch (err) {
-          restorePendingWatcherPaths(group.paths);
+          restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
           process.stderr.write(
             `watch scoped push failed, will retry next tick: ${describeError(err)}\n`,
           );
@@ -901,12 +1036,17 @@ export async function runWatchLoop(
 
     watcher.onChange((changedRelPath, batch) => {
       if (stopped) return;
-      addPendingWatcherChange(changedRelPath, batch);
+      if (addPendingWatcherChange(changedRelPath, batch)) interruptWait();
     });
     watcher.start();
 
     const receiverSyncFn: SyncEngineFn = async (ctx) => {
       if (stopped) return;
+      // A received wake/high-water signal means there is bounded work now.
+      // It wakes the scheduler but is still funneled through runGuarded below,
+      // so it cannot overlap the pass that may currently own the operation
+      // lock.
+      interruptWait();
       const route = routeChangeToTarget(ctx.event.relativePath);
       if (!route) return;
       const targetedArgv = buildTargetedPullArgv(route, passArgv);
@@ -921,8 +1061,7 @@ export async function runWatchLoop(
     receiver = createReceiver({ syncFn: receiverSyncFn, hqRoot });
     void Promise.resolve(receiver.start()).catch(() => undefined);
 
-    const email = claims?.email;
-    if (resolveEventSync(email, process.env.HQ_SYNC_EVENT_SYNC)) {
+    if (resolveEventSync(process.env.HQ_SYNC_EVENT_SYNC)) {
       const startES = deps.startEventSync ?? defaultStartEventSync;
       void (async () => {
         const eventSyncClient = new VaultClient({
@@ -962,10 +1101,6 @@ export async function runWatchLoop(
     }
   }
 
-  let resolveStopped: (() => void) | null = null;
-  const stoppedSignal = new Promise<void>((resolve) => {
-    resolveStopped = resolve;
-  });
   const shutdown = (): void => {
     if (stopped) return;
     stopped = true;
@@ -1001,6 +1136,9 @@ export async function runWatchLoop(
     });
   detachSignal = onShutdownSignal(shutdown);
 
+  // The initial reconcile is due immediately. Later passes advance from this
+  // deadline, not from pass completion, so cadence stays monotonic.
+  let nextDueAt = monotonicNow();
   let pollTick = 0;
   try {
     while (!stopped) {
@@ -1022,7 +1160,7 @@ export async function runWatchLoop(
         pollTick % FULL_RECONCILE_EVERY_TICKS === 0;
       const result = shouldRunFull
         ? await runGuarded(passArgv)
-        : await runScopedDrain(pending.batch);
+        : await runScopedDrain(pending.batch, pending.generation);
       // This batch's episode has now been prepared and pushed, so a later
       // removal under the same path is judged fresh — a root that never
       // expired would silently refuse ordinary deletes from here on.
@@ -1081,7 +1219,13 @@ export async function runWatchLoop(
       }
       const nextPollMs =
         explicitPollMs ?? adaptivePollMs(sampleLoadAvg(), cpuCount);
-      await Promise.race([sleep(nextPollMs), stoppedSignal]);
+      const schedule = advanceMonotonicDeadline(
+        nextDueAt,
+        monotonicNow(),
+        nextPollMs,
+      );
+      nextDueAt = schedule.nextDueAt;
+      await waitForNextPass(schedule.delayMs);
     }
     return 0;
   } finally {

package/src/bin/sync-runner-watch-routes.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from "vitest";
+import {
+  isWatchRouteSelected,
+  routeChangeToTarget,
+  type WatchRouteSelection,
+} from "./sync-runner-watch-routes.js";
+
+const companies: WatchRouteSelection = {
+  companies: true,
+  company: false,
+  personal: false,
+  skipPersonal: false,
+};
+
+describe("watch route selection", () => {
+  it("routes personal and company changes for an ordinary companies run", () => {
+    const personal = routeChangeToTarget("knowledge/me.md");
+    const company = routeChangeToTarget("companies/indigo/knowledge/team.md");
+
+    expect(personal).toEqual({ kind: "personal" });
+    expect(company).toEqual({ kind: "company", slug: "indigo" });
+    expect(isWatchRouteSelected(personal!, companies)).toBe(true);
+    expect(isWatchRouteSelected(company!, companies)).toBe(true);
+  });
+
+  it("rejects personal changes before queueing when companies skips personal", () => {
+    const personal = routeChangeToTarget("knowledge/me.md");
+
+    expect(
+      isWatchRouteSelected(personal!, { ...companies, skipPersonal: true }),
+    ).toBe(false);
+  });
+
+  it("keeps route selection scoped for personal and single-company runs", () => {
+    const personal = { kind: "personal" } as const;
+    const company = { kind: "company", slug: "indigo" } as const;
+
+    expect(
+      isWatchRouteSelected(personal, {
+        companies: false,
+        company: false,
+        personal: true,
+        skipPersonal: false,
+      }),
+    ).toBe(true);
+    expect(
+      isWatchRouteSelected(company, {
+        companies: false,
+        company: false,
+        personal: true,
+        skipPersonal: false,
+      }),
+    ).toBe(false);
+    expect(
+      isWatchRouteSelected(personal, {
+        companies: false,
+        company: true,
+        personal: false,
+        skipPersonal: false,
+      }),
+    ).toBe(false);
+    expect(
+      isWatchRouteSelected(company, {
+        companies: false,
+        company: true,
+        personal: false,
+        skipPersonal: false,
+      }),
+    ).toBe(true);
+  });
+});

package/src/bin/sync-runner-watch-routes.ts

@@ -5,6 +5,18 @@ export type WatchRoute =
   | { kind: "company"; slug: string }
   | { kind: "personal" };
 
+/**
+ * The sync target selected for a watcher run. Watcher events must be rejected
+ * at ingress when they cannot belong to this target; deferring this decision
+ * until the pass planner would still queue a needless guarded pass.
+ */
+export interface WatchRouteSelection {
+  companies: boolean;
+  company: boolean;
+  personal: boolean;
+  skipPersonal: boolean;
+}
+
 /**
  * Route a changed relative path to the push target that owns it.
  *
@@ -25,6 +37,21 @@ export function routeChangeToTarget(relPath: string): WatchRoute | null {
   return { kind: "personal" };
 }
 
+/**
+ * Whether a routed watcher change belongs to the command's selected sync
+ * target. This is deliberately evaluated before watcher work is queued, so a
+ * disabled personal scope cannot create a pass or event-sync publication.
+ */
+export function isWatchRouteSelected(
+  route: WatchRoute,
+  selection: WatchRouteSelection,
+): boolean {
+  if (route.kind === "personal") {
+    return selection.personal || (selection.companies && !selection.skipPersonal);
+  }
+  return selection.companies || selection.company;
+}
+
 /**
  * Build the argv for a targeted push pass from a routed change.
  */