@indigoai-us/hq-cloud 6.13.5 → 6.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cloud",
-  "version": "6.13.5",
+  "version": "6.14.0",
   "description": "HQ by Indigo cloud sync engine — bidirectional S3 sync for mobile access",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/bin/sync-runner-company.ts

@@ -6,6 +6,13 @@ import type { ConflictStrategy } from "../cli/conflict.js";
 import type { UploadAuthor } from "../s3.js";
 import type { VaultServiceConfig } from "../index.js";
 import { resolvePullScope, type PullScope } from "../sync/pull-scope.js";
+import {
+  coalescePrefixes,
+  pathToScopePrefix,
+  toScopePrefixEntries,
+  type ScopePrefixEntry,
+  type ScopePrefixInput,
+} from "../prefix-coalesce.js";
 import { describeError } from "../lib/describe-error.js";
 import type {
   DeletePropagationPolicy,
@@ -42,6 +49,7 @@ export async function executeCompanyFanout(options: {
   shareFn: (options: ShareOptions) => Promise<ShareResult>;
   resolveDeletePolicy: () => DeletePropagationPolicy;
   emit: (event: RunnerEvent) => void;
+  scopePaths?: string[];
 }): Promise<CompanyFanoutResult> {
   const doPush =
     options.direction === "push" || options.direction === "both";
@@ -196,6 +204,10 @@ export async function executeCompanyFanout(options: {
 
       if (doPush) {
         activePhase = "push";
+        const pushPrefixSet = effectivePushPrefixSet(
+          scope.prefixSet,
+          options.scopePaths ?? [],
+        );
         const pushPaths =
           target.personalMode === true
             ? computePersonalVaultPaths(options.hqRoot, { teamSyncedSlugs })
@@ -224,8 +236,8 @@ export async function executeCompanyFanout(options: {
           ...(decommissionPrefixes && decommissionPrefixes.length > 0
             ? { decommissionPrefixes }
             : {}),
-          ...(scope.prefixSet !== undefined
-            ? { prefixSet: scope.prefixSet }
+          ...(pushPrefixSet !== undefined
+            ? { prefixSet: pushPrefixSet }
             : {}),
         });
       }
@@ -364,6 +376,53 @@ export async function executeCompanyFanout(options: {
   return { stateByCompany, errors, allConflicts };
 }
 
+function effectivePushPrefixSet(
+  aclPrefixSet: readonly ScopePrefixInput[] | undefined,
+  scopePaths: readonly string[],
+): string[] | undefined {
+  if (scopePaths.length === 0) {
+    return aclPrefixSet === undefined
+      ? undefined
+      : coalescePrefixes(aclPrefixSet);
+  }
+
+  const requested = scopePaths.flatMap((p) => [
+    pathToScopePrefix(p),
+    pathToScopePrefix(p.endsWith("/") ? p : p + "/"),
+  ]);
+  if (aclPrefixSet === undefined) {
+    return coalescePrefixes(requested);
+  }
+
+  const aclEntries = toScopePrefixEntries(aclPrefixSet);
+  const requestedEntries = toScopePrefixEntries(requested);
+  const intersection: ScopePrefixEntry[] = [];
+  for (const requestedEntry of requestedEntries) {
+    for (const aclEntry of aclEntries) {
+      if (entryCoversEntry(aclEntry, requestedEntry)) {
+        intersection.push(requestedEntry);
+      } else if (entryCoversEntry(requestedEntry, aclEntry)) {
+        intersection.push(aclEntry);
+      }
+    }
+  }
+  return coalescePrefixes(intersection);
+}
+
+function entryCoversEntry(
+  entry: ScopePrefixEntry,
+  candidate: ScopePrefixEntry,
+): boolean {
+  if (entry.prefix === "") return true;
+  if (entry.match === "prefix") {
+    return (
+      candidate.prefix === entry.prefix ||
+      candidate.prefix.startsWith(entry.prefix)
+    );
+  }
+  return candidate.match === "exact" && candidate.prefix === entry.prefix;
+}
+
 function normalizePullChangedPaths(
   target: RunnerTarget,
   changedPaths: string[] | undefined,

package/src/bin/sync-runner-watch-loop.ts

@@ -12,7 +12,6 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
 import { getOrCreateMachineId } from "../lib/machine-id.js";
 import {
   TreeWatcher,
-  WatchPushDriver,
   systemClock,
   type TreeChangeBatch,
 } from "../watcher.js";
@@ -27,18 +26,28 @@ import {
   type EventSyncHandles,
 } from "../sync/event-sync.js";
 import {
-  buildFullFanoutPushArgv,
+  buildScopedDrainPullArgv,
+  buildScopedPushArgv,
   buildTargetedPullArgv,
-  buildTargetedPushArgv,
   routeChangeToTarget,
-  routesForBatch,
+  routeKey,
   type WatchRoute,
 } from "./sync-runner-watch-routes.js";
-import type { RunnerLoopDeps, WatcherSurface } from "./sync-runner.js";
+import type { Direction, RunnerLoopDeps, WatcherSurface } from "./sync-runner.js";
+
+const FULL_RECONCILE_EVERY_TICKS = 10;
 
 export interface ParsedLoopArgs {
   hqRoot: string;
   lockTimeoutSec?: number;
+  /**
+   * The run's resolved sync direction, straight from `parseArgs` (the single
+   * source of truth — same default as the parser). The drain reads this to
+   * decide whether scoped pushes and/or the pull leg run; it must NOT re-parse
+   * `--direction` from argv, which mis-defaults an omitted flag and disagrees
+   * with the parser on duplicate flags.
+   */
+  direction?: Direction;
 }
 
 export interface WatchLoopRuntime {
@@ -92,9 +101,19 @@ export async function runWatchLoop(
     if (a === "--watch") return false;
     if (a === "--poll-remote-ms") return false;
     if (a === "--event-push") return false;
+    if (a === "--scope-path") return false;
     if (i > 0 && argv[i - 1] === "--poll-remote-ms") return false;
+    if (i > 0 && argv[i - 1] === "--scope-path") return false;
     return true;
   });
+  // Authoritative direction from parseArgs — never re-derive from argv (that
+  // mis-defaults an omitted --direction to "both" and picks the wrong
+  // occurrence on duplicates). Fall back to the parser's own default ("pull").
+  const originalDirection: Direction = parsed.direction ?? "pull";
+  const drainRunsPush =
+    originalDirection === "both" || originalDirection === "push";
+  const drainRunsPull =
+    originalDirection === "both" || originalDirection === "pull";
 
   if (parsed.lockTimeoutSec === 0) {
     try {
@@ -198,7 +217,6 @@ export async function runWatchLoop(
   };
 
   let watcher: WatcherSurface | null = null;
-  let driver: WatchPushDriver | null = null;
   let detachSignal: (() => void) | null = null;
   const pendingWatcherPaths = new Map<string, string>();
   let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
@@ -237,8 +255,8 @@ export async function runWatchLoop(
     pendingWatcherBareChange = true;
   };
   const takePendingWatcherChange = (): {
-    rel: string | null;
     batch: TreeChangeBatch | null;
+    bare: boolean;
   } => {
     const batch =
       pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
@@ -255,15 +273,81 @@ export async function runWatchLoop(
                 : {}),
             }
           : null;
-    const rel = [...pendingWatcherPaths.values()][0] ?? null;
-    const fallbackRel = rel ?? (pendingWatcherBareChange ? null : null);
+    const bare = pendingWatcherBareChange;
     pendingWatcherPaths.clear();
     pendingWatcherOriginalBatch = null;
     pendingWatcherBareChange = false;
     pendingWatcherOverflowed = false;
     pendingWatcherDroppedPaths = 0;
     pendingWatcherDroppedBytes = 0;
-    return { rel: fallbackRel, batch };
+    return { batch, bare };
+  };
+  const restorePendingWatcherPaths = (paths: Map<string, string>): void => {
+    pendingWatcherOriginalBatch = null;
+    for (const [absolutePath, relativePath] of paths.entries()) {
+      pendingWatcherPaths.set(absolutePath, relativePath);
+    }
+  };
+
+  const groupBatchByRoute = (
+    batch: TreeChangeBatch,
+  ): Map<
+    string,
+    { route: WatchRoute; paths: Map<string, string>; relPaths: string[] }
+  > => {
+    const grouped = new Map<
+      string,
+      { route: WatchRoute; paths: Map<string, string>; relPaths: string[] }
+    >();
+    for (const [absolutePath, relativePath] of batch.paths.entries()) {
+      const route = routeChangeToTarget(relativePath);
+      if (!route) continue;
+      const key = routeKey(route);
+      const existing =
+        grouped.get(key) ??
+        { route, paths: new Map<string, string>(), relPaths: [] };
+      existing.paths.set(absolutePath, relativePath);
+      existing.relPaths = [...new Set([...existing.paths.values()])];
+      grouped.set(key, existing);
+    }
+    return grouped;
+  };
+
+  const publishBatchIfAvailable = (paths: Map<string, string>): void => {
+    if (!stopped && eventSync && paths.size > 0) {
+      eventSync.publishBatch({ paths: new Map(paths) });
+    }
+  };
+
+  const runScopedDrain = async (
+    batch: TreeChangeBatch | null,
+  ): Promise<number> => {
+    if (drainRunsPush && batch && batch.paths.size > 0) {
+      const grouped = groupBatchByRoute(batch);
+      for (const group of grouped.values()) {
+        const scopedArgv = buildScopedPushArgv(
+          group.route,
+          group.relPaths,
+          passArgv,
+        );
+        try {
+          const result = await runGuarded(scopedArgv);
+          if (result === 0) {
+            publishBatchIfAvailable(group.paths);
+          } else {
+            restorePendingWatcherPaths(group.paths);
+          }
+        } catch (err) {
+          restorePendingWatcherPaths(group.paths);
+          process.stderr.write(
+            `watch scoped push failed, will retry next tick: ${describeError(err)}\n`,
+          );
+        }
+      }
+    }
+
+    if (!drainRunsPull) return 0;
+    return runGuarded(buildScopedDrainPullArgv(passArgv));
   };
 
   let receiver: PushReceiver | null = null;
@@ -283,47 +367,9 @@ export async function runWatchLoop(
         }));
     watcher = createWatcher({ hqRoot, debounceMs, clock });
 
-    driver = new WatchPushDriver({
-      debounceMs: 0,
-      clock,
-      push: async () => {
-        if (stopped) return;
-        const { rel, batch: batchForPublish } = takePendingWatcherChange();
-        const batchRoutes = batchForPublish
-          ? routesForBatch(batchForPublish)
-          : new Map<string, WatchRoute>();
-        let targetedArgv: string[];
-        if (batchForPublish?.overflowed || batchRoutes.size > 1) {
-          targetedArgv = buildFullFanoutPushArgv(passArgv);
-        } else {
-          const route =
-            batchRoutes.size === 1
-              ? [...batchRoutes.values()][0]
-              : rel
-                ? routeChangeToTarget(rel)
-                : { kind: "personal" as const };
-          if (!route) return;
-          targetedArgv = buildTargetedPushArgv(route, passArgv);
-        }
-        const result = await runGuarded(targetedArgv);
-        if (
-          result === 0 &&
-          !stopped &&
-          eventSync &&
-          !batchForPublish?.overflowed
-        ) {
-          const batch: TreeChangeBatch | null =
-            batchForPublish ??
-            (rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
-          if (batch) eventSync.publishBatch(batch);
-        }
-      },
-    });
-
     watcher.onChange((changedRelPath, batch) => {
       if (stopped) return;
       addPendingWatcherChange(changedRelPath, batch);
-      driver?.notifyChange();
     });
     watcher.start();
 
@@ -403,11 +449,6 @@ export async function runWatchLoop(
       /* ignore */
     }
     eventSync = null;
-    try {
-      driver?.dispose();
-    } catch {
-      /* ignore */
-    }
     try {
       watcher?.dispose();
     } catch {
@@ -428,9 +469,19 @@ export async function runWatchLoop(
     });
   detachSignal = onShutdownSignal(shutdown);
 
+  let pollTick = 0;
   try {
     while (!stopped) {
-      const result = await runGuarded(passArgv);
+      pollTick++;
+      const pending = takePendingWatcherChange();
+      const shouldRunFull =
+        pollTick === 1 ||
+        pending.bare ||
+        pending.batch?.overflowed === true ||
+        pollTick % FULL_RECONCILE_EVERY_TICKS === 0;
+      const result = shouldRunFull
+        ? await runGuarded(passArgv)
+        : await runScopedDrain(pending.batch);
       if (result === TRANSIENT_NETWORK_EXIT) {
         // The machine was briefly offline (the discovery fetch failed after
         // retries). That is NOT a crash — stay alive and retry on the next

package/src/bin/sync-runner-watch-routes.ts

@@ -37,10 +37,48 @@ export function buildTargetedPushArgv(
   return ["--companies", "--direction", "push", ...carried];
 }
 
+export function buildScopedPushArgv(
+  route: WatchRoute,
+  relPaths: readonly string[],
+  baseArgv: string[],
+): string[] {
+  const scopeArgs = scopedPathArgs(route, relPaths);
+  const carried = carriedFlags(baseArgv);
+  if (route.kind === "company") {
+    return [
+      "--company",
+      route.slug,
+      "--direction",
+      "push",
+      ...scopeArgs,
+      ...carried,
+    ];
+  }
+  return ["--personal", "--direction", "push", ...scopeArgs, ...carried];
+}
+
 export function buildFullFanoutPushArgv(baseArgv: string[]): string[] {
   return ["--companies", "--direction", "push", ...carriedFlags(baseArgv)];
 }
 
+export function buildScopedDrainPullArgv(baseArgv: string[]): string[] {
+  const carried = carriedFlags(baseArgv);
+  if (baseArgv.includes("--personal")) {
+    return ["--personal", "--direction", "pull", ...carried];
+  }
+  const companyIdx = baseArgv.indexOf("--company");
+  if (companyIdx >= 0 && baseArgv[companyIdx + 1] !== undefined) {
+    return [
+      "--company",
+      baseArgv[companyIdx + 1],
+      "--direction",
+      "pull",
+      ...carried,
+    ];
+  }
+  return ["--companies", "--direction", "pull", ...carried];
+}
+
 /**
  * Build the argv for a targeted pull pass from a routed change.
  */
@@ -55,7 +93,7 @@ export function buildTargetedPullArgv(
   return ["--companies", "--direction", "pull", ...carried];
 }
 
-function routeKey(route: WatchRoute): string {
+export function routeKey(route: WatchRoute): string {
   return route.kind === "company" ? `company:${route.slug}` : "personal";
 }
 
@@ -69,6 +107,24 @@ export function routesForBatch(batch: TreeChangeBatch): Map<string, WatchRoute>
   return routes;
 }
 
+function scopedPathArgs(route: WatchRoute, relPaths: readonly string[]): string[] {
+  const args: string[] = [];
+  for (const relPath of relPaths) {
+    const scopePath = scopePathForRoute(route, relPath);
+    if (scopePath === null) continue;
+    args.push("--scope-path", scopePath);
+  }
+  return args;
+}
+
+function scopePathForRoute(route: WatchRoute, relPath: string): string | null {
+  const norm = relPath.split(path.sep).join("/").replace(/^\.\//, "");
+  if (route.kind === "personal") return norm;
+  const prefix = `companies/${route.slug}/`;
+  if (norm.startsWith(prefix)) return norm.slice(prefix.length);
+  return null;
+}
+
 /**
  * Extract the `--hq-root` / `--on-conflict` pair-flags from a base argv so a
  * re-targeted pass inherits the same root and conflict policy.