@indigoai-us/hq-cloud 6.11.12 → 6.11.13

package/src/bin/sync-runner-watch-loop.ts

@@ -0,0 +1,443 @@
+import * as path from "path";
+import { VaultClient } from "../index.js";
+import { pickCanonicalPersonEntity } from "../vault-client.js";
+import {
+  acquireOperationLock,
+  withOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+} from "../operation-lock.js";
+import { describeError } from "../lib/describe-error.js";
+import { getOrCreateMachineId } from "../lib/machine-id.js";
+import {
+  TreeWatcher,
+  WatchPushDriver,
+  systemClock,
+  type TreeChangeBatch,
+} from "../watcher.js";
+import {
+  NoopPushReceiver,
+  type PushReceiver,
+  type SyncEngineFn,
+} from "../sync/push-receiver.js";
+import {
+  resolveEventSync,
+  startEventSync as defaultStartEventSync,
+  type EventSyncHandles,
+} from "../sync/event-sync.js";
+import {
+  buildFullFanoutPushArgv,
+  buildTargetedPullArgv,
+  buildTargetedPushArgv,
+  routeChangeToTarget,
+  routesForBatch,
+  type WatchRoute,
+} from "./sync-runner-watch-routes.js";
+import type { RunnerLoopDeps, WatcherSurface } from "./sync-runner.js";
+
+export interface ParsedLoopArgs {
+  hqRoot: string;
+  lockTimeoutSec?: number;
+}
+
+export interface WatchLoopRuntime {
+  runPassWithOperationLockAlreadyHeld: (passArgv: string[]) => Promise<number>;
+  defaultGetIdTokenClaims: () => { email?: string } | null;
+  defaultGetAccessToken: () => Promise<string>;
+  apiUrl: string;
+  region: string;
+}
+
+export async function runOneShotWithOperationLock(
+  argv: string[],
+  parsed: ParsedLoopArgs,
+  deps: RunnerLoopDeps,
+  runtime: WatchLoopRuntime,
+): Promise<number> {
+  const runOnce =
+    deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
+  try {
+    return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
+      timeoutSec: parsed.lockTimeoutSec,
+    });
+  } catch (err) {
+    if (err instanceof OperationLockedError) {
+      process.stderr.write(err.message + "\n");
+      return OPERATION_LOCKED_EXIT;
+    }
+    throw err;
+  }
+}
+
+export async function runWatchLoop(
+  argv: string[],
+  parsed: ParsedLoopArgs,
+  deps: RunnerLoopDeps,
+  runtime: WatchLoopRuntime,
+): Promise<number> {
+  const sleep =
+    deps.sleep ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const runPass =
+    deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
+  const pollIdx = argv.indexOf("--poll-remote-ms");
+  const pollMs =
+    pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
+  const eventPush = argv.includes("--event-push");
+  const companiesMode = argv.includes("--companies");
+  const hqRoot = parsed.hqRoot;
+
+  const passArgv = argv.filter((a, i) => {
+    if (a === "--watch") return false;
+    if (a === "--poll-remote-ms") return false;
+    if (a === "--event-push") return false;
+    if (i > 0 && argv[i - 1] === "--poll-remote-ms") return false;
+    return true;
+  });
+
+  if (parsed.lockTimeoutSec === 0) {
+    try {
+      const handle = acquireOperationLock(hqRoot, "sync", {
+        timeoutSec: 0,
+        onWaitStart: () => undefined,
+      });
+      handle.release();
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
+      throw err;
+    }
+  }
+
+  const runPassWithLock = async (passArgvForRun: string[]): Promise<number> => {
+    try {
+      const handle = acquireOperationLock(hqRoot, "sync", {
+        timeoutSec: 0,
+        onWaitStart: () => undefined,
+      });
+      try {
+        return await runPass(passArgvForRun);
+      } finally {
+        handle.release();
+      }
+    } catch (err) {
+      if (!(err instanceof OperationLockedError)) throw err;
+      if (parsed.lockTimeoutSec === 0) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
+    }
+
+    try {
+      return await withOperationLock(
+        hqRoot,
+        "sync",
+        () => runPass(passArgvForRun),
+        { timeoutSec: parsed.lockTimeoutSec },
+      );
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
+      throw err;
+    }
+  };
+
+  let stopped = false;
+  let activePass: Promise<number> | null = null;
+  type QueuedPass = {
+    argv: string[];
+    resolve: (code: number) => void;
+    reject: (err: unknown) => void;
+  };
+  const pendingPasses: QueuedPass[] = [];
+  const resolveStoppedQueue = (): void => {
+    while (pendingPasses.length > 0) {
+      pendingPasses.shift()?.resolve(0);
+    }
+  };
+  const drainQueuedPasses = (): void => {
+    if (activePass !== null) return;
+    if (stopped) {
+      resolveStoppedQueue();
+      return;
+    }
+    const next = pendingPasses.shift();
+    if (!next) return;
+    const current = startGuardedPass(next.argv);
+    void current.then(next.resolve, next.reject);
+  };
+  const startGuardedPass = (passArgvForRun: string[]): Promise<number> => {
+    const current = stopped
+      ? Promise.resolve(0)
+      : runPassWithLock(passArgvForRun);
+    activePass = current;
+    void current
+      .finally(() => {
+        if (activePass === current) {
+          activePass = null;
+          drainQueuedPasses();
+        }
+      })
+      .catch(() => undefined);
+    return current;
+  };
+  const runGuarded = (passArgvForRun: string[]): Promise<number> => {
+    if (activePass === null && pendingPasses.length === 0) {
+      return startGuardedPass(passArgvForRun);
+    }
+
+    return new Promise<number>((resolve, reject) => {
+      pendingPasses.push({ argv: passArgvForRun, resolve, reject });
+      drainQueuedPasses();
+    });
+  };
+
+  let watcher: WatcherSurface | null = null;
+  let driver: WatchPushDriver | null = null;
+  let detachSignal: (() => void) | null = null;
+  const pendingWatcherPaths = new Map<string, string>();
+  let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
+  let pendingWatcherBareChange = false;
+  let pendingWatcherOverflowed = false;
+  let pendingWatcherDroppedPaths = 0;
+  let pendingWatcherDroppedBytes = 0;
+  const addPendingWatcherChange = (
+    changedRelPath?: string,
+    batch?: TreeChangeBatch,
+  ): void => {
+    if (batch) {
+      pendingWatcherOriginalBatch =
+        pendingWatcherPaths.size === 0 &&
+        !pendingWatcherBareChange &&
+        !pendingWatcherOverflowed &&
+        !batch.overflowed
+          ? batch
+          : null;
+      for (const [absolutePath, relativePath] of batch.paths.entries()) {
+        pendingWatcherPaths.set(absolutePath, relativePath);
+      }
+      if (batch.overflowed) {
+        pendingWatcherOverflowed = true;
+        pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
+        pendingWatcherDroppedBytes += batch.droppedBytes ?? 0;
+      }
+      return;
+    }
+    if (changedRelPath) {
+      pendingWatcherOriginalBatch = null;
+      pendingWatcherPaths.set(path.join(hqRoot, changedRelPath), changedRelPath);
+      return;
+    }
+    pendingWatcherOriginalBatch = null;
+    pendingWatcherBareChange = true;
+  };
+  const takePendingWatcherChange = (): {
+    rel: string | null;
+    batch: TreeChangeBatch | null;
+  } => {
+    const batch =
+      pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
+        ? pendingWatcherOriginalBatch
+        : pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
+          ? {
+              paths: new Map(pendingWatcherPaths),
+              ...(pendingWatcherOverflowed
+                ? {
+                    overflowed: true,
+                    droppedPaths: pendingWatcherDroppedPaths,
+                    droppedBytes: pendingWatcherDroppedBytes,
+                  }
+                : {}),
+            }
+          : null;
+    const rel = [...pendingWatcherPaths.values()][0] ?? null;
+    const fallbackRel = rel ?? (pendingWatcherBareChange ? null : null);
+    pendingWatcherPaths.clear();
+    pendingWatcherOriginalBatch = null;
+    pendingWatcherBareChange = false;
+    pendingWatcherOverflowed = false;
+    pendingWatcherDroppedPaths = 0;
+    pendingWatcherDroppedBytes = 0;
+    return { rel: fallbackRel, batch };
+  };
+
+  let receiver: PushReceiver | null = null;
+  let eventSync: EventSyncHandles | null = null;
+
+  if (eventPush) {
+    const clock = deps.clock ?? systemClock;
+    const debounceMs = 2000;
+    const createWatcher =
+      deps.createWatcher ??
+      ((opts) =>
+        new TreeWatcher({
+          hqRoot: opts.hqRoot,
+          debounceMs: opts.debounceMs,
+          clock: opts.clock,
+          personalMode: !companiesMode,
+        }));
+    watcher = createWatcher({ hqRoot, debounceMs, clock });
+
+    driver = new WatchPushDriver({
+      debounceMs: 0,
+      clock,
+      push: async () => {
+        if (stopped) return;
+        const { rel, batch: batchForPublish } = takePendingWatcherChange();
+        const batchRoutes = batchForPublish
+          ? routesForBatch(batchForPublish)
+          : new Map<string, WatchRoute>();
+        let targetedArgv: string[];
+        if (batchForPublish?.overflowed || batchRoutes.size > 1) {
+          targetedArgv = buildFullFanoutPushArgv(passArgv);
+        } else {
+          const route =
+            batchRoutes.size === 1
+              ? [...batchRoutes.values()][0]
+              : rel
+                ? routeChangeToTarget(rel)
+                : { kind: "personal" as const };
+          if (!route) return;
+          targetedArgv = buildTargetedPushArgv(route, passArgv);
+        }
+        const result = await runGuarded(targetedArgv);
+        if (
+          result === 0 &&
+          !stopped &&
+          eventSync &&
+          !batchForPublish?.overflowed
+        ) {
+          const batch: TreeChangeBatch | null =
+            batchForPublish ??
+            (rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
+          if (batch) eventSync.publishBatch(batch);
+        }
+      },
+    });
+
+    watcher.onChange((changedRelPath, batch) => {
+      if (stopped) return;
+      addPendingWatcherChange(changedRelPath, batch);
+      driver?.notifyChange();
+    });
+    watcher.start();
+
+    const receiverSyncFn: SyncEngineFn = async (ctx) => {
+      if (stopped) return;
+      const route = routeChangeToTarget(ctx.event.relativePath);
+      if (!route) return;
+      const targetedArgv = buildTargetedPullArgv(route, passArgv);
+      const result = await runGuarded(targetedArgv);
+      if (result !== 0) {
+        throw new Error(`targeted pull failed with exit code ${result}`);
+      }
+    };
+    const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
+    receiver = createReceiver({ syncFn: receiverSyncFn, hqRoot });
+    void Promise.resolve(receiver.start()).catch(() => undefined);
+
+    const getClaims =
+      deps.getIdTokenClaims ?? runtime.defaultGetIdTokenClaims;
+    const email = getClaims()?.email;
+    if (resolveEventSync(email, process.env.HQ_SYNC_EVENT_SYNC)) {
+      const getAccessToken =
+        deps.getAccessToken ?? runtime.defaultGetAccessToken;
+      const startES = deps.startEventSync ?? defaultStartEventSync;
+      void (async () => {
+        const handles = await startES({
+          hqRoot,
+          apiUrl: runtime.apiUrl,
+          authToken: getAccessToken,
+          deviceId: getOrCreateMachineId(hqRoot),
+          resolveTenantId: async () => {
+            const client = new VaultClient({
+              apiUrl: runtime.apiUrl,
+              authToken: getAccessToken,
+              region: runtime.region,
+            });
+            const persons = await client.entity.listByType("person");
+            const pick = pickCanonicalPersonEntity(persons);
+            if (!pick?.uid) {
+              throw new Error("no canonical person entity for this account");
+            }
+            return pick.uid;
+          },
+          syncFn: receiverSyncFn,
+          log: (m) => process.stderr.write(`${m}\n`),
+        });
+        if (!handles) return;
+        if (stopped) {
+          void handles.dispose();
+          return;
+        }
+        eventSync = handles;
+      })().catch((err) => {
+        process.stderr.write(
+          `event-sync: wiring failed, continuing poll-only: ${describeError(err)}\n`,
+        );
+      });
+    }
+  }
+
+  let resolveStopped: (() => void) | null = null;
+  const stoppedSignal = new Promise<void>((resolve) => {
+    resolveStopped = resolve;
+  });
+  const shutdown = (): void => {
+    if (stopped) return;
+    stopped = true;
+    resolveStoppedQueue();
+    try {
+      void receiver?.dispose();
+    } catch {
+      /* ignore */
+    }
+    try {
+      void eventSync?.dispose();
+    } catch {
+      /* ignore */
+    }
+    eventSync = null;
+    try {
+      driver?.dispose();
+    } catch {
+      /* ignore */
+    }
+    try {
+      watcher?.dispose();
+    } catch {
+      /* ignore */
+    }
+    resolveStopped?.();
+  };
+  const onShutdownSignal =
+    deps.onShutdownSignal ??
+    ((handler: () => void) => {
+      const wrapped = () => handler();
+      process.on("SIGTERM", wrapped);
+      process.on("SIGINT", wrapped);
+      return () => {
+        process.off("SIGTERM", wrapped);
+        process.off("SIGINT", wrapped);
+      };
+    });
+  detachSignal = onShutdownSignal(shutdown);
+
+  try {
+    while (!stopped) {
+      const result = await runGuarded(passArgv);
+      if (result !== 0) {
+        return result;
+      }
+      await Promise.race([sleep(pollMs), stoppedSignal]);
+    }
+    return 0;
+  } finally {
+    shutdown();
+    detachSignal?.();
+  }
+}

package/src/bin/sync-runner-watch-routes.ts

@@ -0,0 +1,86 @@
+import * as path from "path";
+import type { TreeChangeBatch } from "../watcher.js";
+
+export type WatchRoute =
+  | { kind: "company"; slug: string }
+  | { kind: "personal" };
+
+/**
+ * Route a changed relative path to the push target that owns it.
+ *
+ * - `companies/<slug>/...` -> a single-company route for `<slug>`.
+ * - anything else under hqRoot -> the personal route.
+ */
+export function routeChangeToTarget(relPath: string): WatchRoute | null {
+  const norm = relPath.split(path.sep).join("/").replace(/^\.\//, "");
+  if (norm === "" || norm.startsWith("..")) return null;
+  const segments = norm.split("/").filter((s) => s.length > 0);
+  if (segments.length === 0) return null;
+  if (segments[0] === "companies") {
+    if (segments.length < 2 || segments[1].length === 0) return null;
+    return { kind: "company", slug: segments[1] };
+  }
+  return { kind: "personal" };
+}
+
+/**
+ * Build the argv for a targeted push pass from a routed change.
+ */
+export function buildTargetedPushArgv(
+  route: WatchRoute,
+  baseArgv: string[],
+): string[] {
+  const carried = carriedFlags(baseArgv);
+  if (route.kind === "company") {
+    return ["--company", route.slug, "--direction", "push", ...carried];
+  }
+  return ["--companies", "--direction", "push", ...carried];
+}
+
+export function buildFullFanoutPushArgv(baseArgv: string[]): string[] {
+  return ["--companies", "--direction", "push", ...carriedFlags(baseArgv)];
+}
+
+/**
+ * Build the argv for a targeted pull pass from a routed change.
+ */
+export function buildTargetedPullArgv(
+  route: WatchRoute,
+  baseArgv: string[],
+): string[] {
+  const carried = carriedFlags(baseArgv);
+  if (route.kind === "company") {
+    return ["--company", route.slug, "--direction", "pull", ...carried];
+  }
+  return ["--companies", "--direction", "pull", ...carried];
+}
+
+function routeKey(route: WatchRoute): string {
+  return route.kind === "company" ? `company:${route.slug}` : "personal";
+}
+
+export function routesForBatch(batch: TreeChangeBatch): Map<string, WatchRoute> {
+  const routes = new Map<string, WatchRoute>();
+  for (const relPath of batch.paths.values()) {
+    const route = routeChangeToTarget(relPath);
+    if (!route) continue;
+    routes.set(routeKey(route), route);
+  }
+  return routes;
+}
+
+/**
+ * Extract the `--hq-root` / `--on-conflict` pair-flags from a base argv so a
+ * re-targeted pass inherits the same root and conflict policy.
+ */
+function carriedFlags(baseArgv: string[]): string[] {
+  const carried: string[] = [];
+  for (let i = 0; i < baseArgv.length; i++) {
+    const a = baseArgv[i];
+    if (a === "--hq-root" || a === "--on-conflict") {
+      carried.push(a);
+      if (baseArgv[i + 1] !== undefined) carried.push(baseArgv[++i]);
+    }
+  }
+  return carried;
+}