@indigoai-us/hq-cloud 6.14.8 → 6.14.9

package/src/cli/sync.ts

@@ -26,6 +26,7 @@ import {
   headRemoteFile,
   primeObjectTransport,
   toPosixKey,
+  classifyVaultKey,
 } from "../s3.js";
 import type { DownloadModeWarning, RemoteFile } from "../s3.js";
 import {
@@ -348,12 +349,10 @@ export type SyncProgressEvent =
     }
   | {
       /**
-       * Emitted by the PULL leg once per remote key skipped because it is
-       * invalid for the vault's scope — today, a `companies/…`-prefixed key in
-       * a company-scoped vault (the server rejects these with
-       * INVALID_KEY_COMPANIES_SCOPED; see hq-pro files-acl validateObjectKey).
-       * Such keys are bucket poisoning from a client with a stale doubled
-       * local tree writing via direct S3 (incident 2026-07-11). Like
+       * Emitted by the PULL leg once per remote key skipped because the server
+       * will never presign it (for example, a `companies/…`-prefixed key in a
+       * company-scoped vault or a key with control characters). Such keys are
+       * bucket poisoning from an outdated direct-S3 client. Like
        * `skip-size-limit`, this is deliberately NOT `type: "error"` — one bad
        * object must never error a whole company sync (erroring gated
        * heartbeat toolset refreshes fleet-wide in the incident). Surfaced for
@@ -363,6 +362,8 @@ export type SyncProgressEvent =
        */
       type: "skip-invalid-scoped-key";
       path: string;
+      /** Server-compatible invalid-key code, such as INVALID_KEY_CONTROL_CHARS. */
+      errorCode?: string;
     };
 
 export interface SyncOptions {
@@ -816,7 +817,7 @@ async function syncWithOperationLockHeld(
   const plan = planPull(run);
 
   emitPullPlan(run.emit, plan);
-  reportInvalidScopedKeys(run, plan);
+  reportInvalidRemoteKeys(run, plan);
 
   const scopePlan = planScopeShrink(run);
   const scopeRun = executeScopeShrink(run, scopePlan);
@@ -951,30 +952,30 @@ function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
 }
 
 /**
- * Surface scope-invalid remote keys the planner skipped (see the
- * `invalidScopedKeys` doc on PullPlan). One warning event per key on the
+ * Surface permanently-unpresignable remote keys the planner skipped (see the
+ * `invalidRemoteKeys` doc on PullPlan). One warning event per key on the
  * existing sync event channel (deliberately NOT `type: "error"` — see the
  * `skip-invalid-scoped-key` event doc), plus one best-effort cloud telemetry
  * event on the channel sync already uses (emitCloudTelemetry). Per policy
  * hq-alert-baseline-calibration the documented healthy baseline is ZERO
- * INVALID_KEY_COMPANIES_SCOPED occurrences fleet-wide, so consumers can alert
- * on any non-zero count with no threshold tuning.
+ * invalid-key occurrences fleet-wide, so consumers can alert on any non-zero
+ * count with no threshold tuning.
  */
-function reportInvalidScopedKeys(run: PullRunContext, plan: PullPlan): void {
-  if (plan.invalidScopedKeys.length === 0) return;
-  for (const key of plan.invalidScopedKeys) {
-    run.emit({ type: "skip-invalid-scoped-key", path: key });
+function reportInvalidRemoteKeys(run: PullRunContext, plan: PullPlan): void {
+  if (plan.invalidRemoteKeys.length === 0) return;
+  const countsByErrorCode = new Map<string, number>();
+  for (const { path, errorCode } of plan.invalidRemoteKeys) {
+    run.emit({ type: "skip-invalid-scoped-key", path, errorCode });
+    countsByErrorCode.set(errorCode, (countsByErrorCode.get(errorCode) ?? 0) + 1);
+  }
+  for (const [errorCode, count] of countsByErrorCode) {
+    void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
+      eventName: "sync_invalid_scoped_key_skipped",
+      source: "hq-sync",
+      companyUid: run.ctx.uid,
+      properties: { count, errorCode, leg: "pull" },
+    }, { claims: run.options.telemetryClaims });
   }
-  void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
-    eventName: "sync_invalid_scoped_key_skipped",
-    source: "hq-sync",
-    companyUid: run.ctx.uid,
-    properties: {
-      count: plan.invalidScopedKeys.length,
-      errorCode: "INVALID_KEY_COMPANIES_SCOPED",
-      leg: "pull",
-    },
-  }, { claims: run.options.telemetryClaims });
 }
 
 /**
@@ -1991,14 +1992,13 @@ interface PullPlan {
    */
   filesToTombstoneDelete: number;
   /**
-   * Remote keys skipped because they are invalid for the vault's scope
-   * (company mode only: literal `companies/…` keys — bucket poisoning from a
-   * doubled-tree client via direct S3). Counted in `filesExcludedByPolicy`;
-   * carried by key so the caller can log one warning per key and emit
-   * telemetry. Healthy baseline is an empty array (policy
+   * Remote keys skipped because the server will never presign them. Counted in
+   * `filesExcludedByPolicy`; carried with the server-compatible error code so
+   * the caller can log one warning per key and emit telemetry. Healthy
+   * baseline is an empty array (policy
    * hq-alert-baseline-calibration: zero expected, any occurrence is signal).
    */
-  invalidScopedKeys: string[];
+  invalidRemoteKeys: Array<{ path: string; errorCode: string }>;
 }
 
 /**
@@ -2075,9 +2075,9 @@ function computePullPlan(
   excludePrefixes: readonly ScopePrefixInput[] = [],
 ): PullPlan {
   const items: PullPlanItem[] = [];
-  // Company-mode remote keys invalid for the vault's scope (`companies/…` in a
-  // company bucket) — collected so the caller can warn + emit telemetry.
-  const invalidScopedKeys: string[] = [];
+  // Remote keys the server will never presign — collected so the caller can
+  // warn + emit telemetry without allowing one poison object to fail a pull.
+  const invalidRemoteKeys: Array<{ path: string; errorCode: string }> = [];
   // Clean/current/non-divergent files missing locally — collected here, then
   // resolved post-loop by the bulk-asymmetry decision (tombstone vs. restore).
   const intentionalDeleteCandidates: Array<{
@@ -2086,6 +2086,23 @@ function computePullPlan(
   }> = [];
 
   for (const remoteFile of remoteFiles) {
+    const invalidKey = classifyVaultKey(
+      remoteFile.key,
+      personalMode ? "personal" : "company",
+    );
+    if (invalidKey !== null) {
+      invalidRemoteKeys.push({
+        path: remoteFile.key,
+        errorCode: invalidKey.code,
+      });
+      items.push({
+        action: "skip-excluded-policy",
+        remoteFile,
+        localPath: companyRoot,
+      });
+      continue;
+    }
+
     const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
     if (localPath === null) {
       items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
@@ -2103,23 +2120,6 @@ function computePullPlan(
       continue;
     }
 
-    // Scope-invalid key filter (incident 2026-07-11). A company vault bucket
-    // is already company-scoped, so a literal `companies/…` key can only be
-    // bucket poisoning — a client with a stale doubled local tree
-    // (companies/{slug}/companies/{slug}/…) pushing via the direct-S3 STS
-    // transport, which bypasses the server validator. The presign download
-    // API rejects these keys (INVALID_KEY_COMPANIES_SCOPED), so attempting to
-    // download them errors the WHOLE company sync. Skip them here — one bad
-    // object must never take down a company sync — and record them on the
-    // plan so the caller can warn + emit telemetry (baseline is zero; any
-    // occurrence is signal). Mirrors the personalMode/malformed-key filters
-    // above. Upload-side counterpart: validateVaultUploadKey in s3.ts.
-    if (!personalMode && remoteFile.key.startsWith("companies/")) {
-      invalidScopedKeys.push(remoteFile.key);
-      items.push({ action: "skip-excluded-policy", remoteFile, localPath });
-      continue;
-    }
-
     if (
       personalMode &&
       remoteFile.key.startsWith("companies/") &&
@@ -2680,7 +2680,7 @@ function computePullPlan(
     filesOutOfScope,
     filesToTombstoneDelete,
     tombstones,
-    invalidScopedKeys,
+    invalidRemoteKeys,
   };
 }
 
@@ -2765,7 +2765,7 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     );
   } else if (event.type === "skip-invalid-scoped-key") {
     console.warn(
-      `  ! ${event.path} skipped — invalid 'companies/'-prefixed key in a company-scoped vault (bucket poisoning; INVALID_KEY_COMPANIES_SCOPED)`,
+      `  ! ${event.path} skipped — invalid vault key (${event.errorCode ?? "INVALID_KEY_COMPANIES_SCOPED"})`,
     );
   }
 }

package/src/manifest-reconcile.test.ts

@@ -0,0 +1,107 @@
+import { afterEach, describe, expect, it } from "vitest";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import yaml from "js-yaml";
+
+import { reconcileCompanyManifest } from "./manifest-reconcile.js";
+import type { EntityInfo } from "./vault-client.js";
+
+let hqRoot: string | undefined;
+
+afterEach(() => {
+  if (hqRoot) {
+    fs.rmSync(hqRoot, { recursive: true, force: true });
+    hqRoot = undefined;
+  }
+});
+
+describe("reconcileCompanyManifest", () => {
+  it("atomically restores one successful cloud entry after the personal leg without replacing existing manifest state", async () => {
+    hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-manifest-reconcile-"));
+    const companiesDir = path.join(hqRoot, "companies");
+    fs.mkdirSync(path.join(companiesDir, "acme"), { recursive: true });
+    fs.mkdirSync(path.join(companiesDir, "failed"), { recursive: true });
+    fs.mkdirSync(path.join(companiesDir, "deleted"), { recursive: true });
+
+    // This is the older manifest the personal-vault leg pulled after the
+    // cloud-company legs completed. The final merge must retain this state and
+    // add only the live, successful cloud target.
+    const manifestPath = path.join(companiesDir, "manifest.yaml");
+    fs.writeFileSync(
+      manifestPath,
+      `version: 3\nsettings:\n  keep: true\ncompanies:\n  local-only:\n    name: Local Only\n    repos:\n      - repos/local\n  preserved:\n    name: Keep Me\n    cloud_uid: cmp_preserved\n`,
+    );
+
+    const entities: Record<string, EntityInfo> = {
+      cmp_acme: {
+        uid: "cmp_acme",
+        slug: "acme",
+        type: "company",
+        status: "active",
+        name: "Acme Inc",
+        bucketName: "hq-vault-cmp-acme",
+        createdAt: "2026-01-01T00:00:00Z",
+      },
+      cmp_failed: {
+        uid: "cmp_failed",
+        slug: "failed",
+        type: "company",
+        status: "active",
+        name: "Failed Corp",
+        bucketName: "hq-vault-cmp-failed",
+        createdAt: "2026-01-01T00:00:00Z",
+      },
+      cmp_deleted: {
+        uid: "cmp_deleted",
+        slug: "deleted",
+        type: "company",
+        status: "deleted",
+        name: "Deleted Corp",
+        bucketName: "hq-vault-cmp-deleted",
+        createdAt: "2026-01-01T00:00:00Z",
+      },
+    };
+    const options = {
+      hqRoot,
+      targets: [
+        { uid: "cmp_acme", slug: "acme" },
+        { uid: "cmp_failed", slug: "failed" },
+        { uid: "cmp_deleted", slug: "deleted" },
+        { uid: "cmp_missing-directory", slug: "missing-directory" },
+        { uid: "prs_me", slug: "personal", personalMode: true },
+      ],
+      completedCompanySlugs: new Set(["acme", "deleted", "missing-directory", "personal"]),
+      getEntity: async (uid: string) => {
+        const entity = entities[uid];
+        if (!entity) throw new Error("entity not found");
+        return entity;
+      },
+    };
+
+    await reconcileCompanyManifest(options);
+    const firstWrite = fs.readFileSync(manifestPath, "utf8");
+    await reconcileCompanyManifest(options);
+
+    expect(fs.readFileSync(manifestPath, "utf8")).toBe(firstWrite);
+    expect(yaml.load(firstWrite)).toEqual({
+      version: 3,
+      settings: { keep: true },
+      companies: {
+        "local-only": {
+          name: "Local Only",
+          repos: ["repos/local"],
+        },
+        preserved: {
+          name: "Keep Me",
+          cloud_uid: "cmp_preserved",
+        },
+        acme: {
+          name: "Acme Inc",
+          cloud_uid: "cmp_acme",
+          bucket_name: "hq-vault-cmp-acme",
+        },
+      },
+    });
+  });
+});

package/src/manifest-reconcile.ts

@@ -0,0 +1,160 @@
+/**
+ * Reconcile successfully pulled cloud companies into the local manifest.
+ *
+ * The personal-vault leg also carries `companies/manifest.yaml`, so this runs
+ * only after the complete fanout has settled. That makes the locally
+ * materialized cloud companies authoritative for their own manifest entries
+ * without discarding entries the personal vault already had.
+ */
+
+import { randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import yaml from "js-yaml";
+
+import type { EntityInfo } from "./vault-client.js";
+
+export interface ManifestReconcileTarget {
+  uid: string;
+  slug: string;
+  personalMode?: boolean;
+}
+
+export interface ManifestReconcileOptions {
+  hqRoot: string;
+  targets: readonly ManifestReconcileTarget[];
+  completedCompanySlugs: ReadonlySet<string>;
+  getEntity: (uid: string) => Promise<EntityInfo>;
+}
+
+interface ManifestCompany {
+  name?: unknown;
+  cloud_uid?: unknown;
+  bucket_name?: unknown;
+  [key: string]: unknown;
+}
+
+interface ManifestDocument {
+  companies?: Record<string, ManifestCompany>;
+  [key: string]: unknown;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function isLiveCompany(entity: EntityInfo, uid: string): boolean {
+  return (
+    entity.uid === uid &&
+    entity.type === "company" &&
+    entity.status === "active" &&
+    typeof entity.name === "string" &&
+    entity.name.length > 0 &&
+    typeof entity.bucketName === "string" &&
+    entity.bucketName.length > 0
+  );
+}
+
+function hasCompanyDirectory(companiesDir: string, slug: string): boolean {
+  const resolvedCompaniesDir = path.resolve(companiesDir);
+  const companyDir = path.resolve(resolvedCompaniesDir, slug);
+  if (path.dirname(companyDir) !== resolvedCompaniesDir) return false;
+  try {
+    return fs.statSync(companyDir).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function loadManifest(manifestPath: string): ManifestDocument | null {
+  try {
+    const raw = fs.readFileSync(manifestPath, "utf8");
+    const parsed = yaml.load(raw);
+    return isRecord(parsed) ? (parsed as ManifestDocument) : null;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return {};
+    return null;
+  }
+}
+
+function writeManifestAtomically(manifestPath: string, document: ManifestDocument): void {
+  const temporaryPath = path.join(
+    path.dirname(manifestPath),
+    `.manifest-${process.pid}-${randomUUID()}.yaml`,
+  );
+  fs.writeFileSync(temporaryPath, yaml.dump(document, { lineWidth: -1 }), "utf8");
+  fs.renameSync(temporaryPath, manifestPath);
+}
+
+/**
+ * Atomically merge live, successfully pulled company targets into
+ * `companies/manifest.yaml`. Targets that are personal, incomplete, deleted,
+ * no longer resolvable, or not materialized on disk are intentionally ignored.
+ */
+export async function reconcileCompanyManifest(
+  options: ManifestReconcileOptions,
+): Promise<void> {
+  const companiesDir = path.join(options.hqRoot, "companies");
+  const candidates: Array<{ slug: string; entity: EntityInfo }> = [];
+
+  for (const target of options.targets) {
+    if (
+      target.personalMode === true ||
+      !options.completedCompanySlugs.has(target.slug) ||
+      !hasCompanyDirectory(companiesDir, target.slug)
+    ) {
+      continue;
+    }
+
+    try {
+      const entity = await options.getEntity(target.uid);
+      if (isLiveCompany(entity, target.uid)) {
+        candidates.push({ slug: target.slug, entity });
+      }
+    } catch {
+      // The target is not presently a live, resolvable company. Never mint a
+      // manifest entry from stale fanout data.
+    }
+  }
+
+  if (candidates.length === 0) return;
+
+  const manifestPath = path.join(companiesDir, "manifest.yaml");
+  const document = loadManifest(manifestPath);
+  if (!document) return;
+
+  let companies: Record<string, ManifestCompany>;
+  if (document.companies === undefined) {
+    companies = {};
+    document.companies = companies;
+  } else if (isRecord(document.companies)) {
+    companies = document.companies as Record<string, ManifestCompany>;
+  } else {
+    // A malformed `companies` key is user state; refusing to replace it is
+    // safer than losing a manifest that cannot be interpreted unambiguously.
+    return;
+  }
+
+  let changed = false;
+  for (const { slug, entity } of candidates) {
+    const existing = companies[slug];
+    if (existing !== undefined && !isRecord(existing)) continue;
+    const entry: ManifestCompany = existing ?? {};
+    if (
+      entry.name === entity.name &&
+      entry.cloud_uid === entity.uid &&
+      entry.bucket_name === entity.bucketName
+    ) {
+      continue;
+    }
+    companies[slug] = {
+      ...entry,
+      name: entity.name,
+      cloud_uid: entity.uid,
+      bucket_name: entity.bucketName,
+    };
+    changed = true;
+  }
+
+  if (changed) writeManifestAtomically(manifestPath, document);
+}

package/src/s3.test.ts

@@ -93,6 +93,7 @@ import {
   decodeSymlinkMetadataValue,
   FILE_MTIME_META_KEY,
   FILE_BTIME_META_KEY,
+  classifyVaultKey,
   validateVaultUploadKey,
 } from "./s3.js";
 import {
@@ -1450,6 +1451,23 @@ describe("validateVaultUploadKey — direct-S3 key poisoning guard (incident 202
     },
   );
 
+  it.each([
+    [
+      "companies/indigo/knowledge/x.md",
+      "company",
+      "INVALID_KEY_COMPANIES_SCOPED",
+    ],
+    ["bad\x00key.md", "personal", "INVALID_KEY_CONTROL_CHARS"],
+    ["docs/good.md", "personal", null],
+  ] as const)(
+    "classifyVaultKey reports the validator outcome without throwing: %j",
+    (key, scope, code) => {
+      expect(classifyVaultKey(key, scope)).toEqual(
+        code === null ? null : expect.objectContaining({ code }),
+      );
+    },
+  );
+
   it("uploadFile refuses to PUT a companies/-prefixed key into a company (cmp_) vault — no S3 command is sent", async () => {
     await expect(
       uploadFile(makeCtx(), tmpFile, "companies/indigo/knowledge/x.md"),

package/src/s3.ts

@@ -561,67 +561,101 @@ export function toPosixKey(key: string): string {
  * (`cmp_*` → "company", otherwise "personal").
  *
  * Errors carry the same `code` values the server throws so telemetry and
- * operator messaging line up across client and server.
+ * operator messaging line up across client and server. Pull planning also
+ * uses the non-throwing classifier below to keep poison keys away from the
+ * presign transport.
  */
 // Matching control chars is the whole point — block NUL/0x00–0x1F/0x7F in keys
 // (they can smuggle past path checks or corrupt HTTP headers on the signed URL).
 // eslint-disable-next-line no-control-regex
 const KEY_CONTROL_CHARS = /[\x00-\x1F\x7F]/;
 
-export function validateVaultUploadKey(
+export type VaultKeyScope = "company" | "personal";
+
+export interface VaultKeyValidationIssue {
+  code: string;
+  message: string;
+}
+
+/**
+ * Classify a vault key with the same rules as {@link validateVaultUploadKey}
+ * without throwing. Callers that enumerate remote objects can use this before
+ * presigning so permanently-invalid legacy keys remain benign skips.
+ */
+export function classifyVaultKey(
   key: string,
-  scope: "company" | "personal",
-): void {
-  const fail = (message: string, code: string): never => {
-    throw Object.assign(new Error(message), { code });
-  };
+  scope: VaultKeyScope,
+): VaultKeyValidationIssue | null {
   if (key.length === 0) {
-    fail("Invalid key: must not be empty", "INVALID_KEY_EMPTY");
+    return {
+      code: "INVALID_KEY_EMPTY",
+      message: "Invalid key: must not be empty",
+    };
   }
   if (key.startsWith("/")) {
-    fail("Invalid key: leading '/' not allowed", "INVALID_KEY_LEADING_SLASH");
+    return {
+      code: "INVALID_KEY_LEADING_SLASH",
+      message: "Invalid key: leading '/' not allowed",
+    };
   }
   if (key.endsWith("/")) {
-    fail(
-      "Invalid key: trailing '/' (folder marker, not an object)",
-      "INVALID_KEY_TRAILING_SLASH",
-    );
+    return {
+      code: "INVALID_KEY_TRAILING_SLASH",
+      message: "Invalid key: trailing '/' (folder marker, not an object)",
+    };
   }
   if (scope === "company" && key.startsWith("companies/")) {
-    fail(
-      "Invalid key: vault keys are bucket-relative and the vault is already " +
+    return {
+      code: "INVALID_KEY_COMPANIES_SCOPED",
+      message:
+        "Invalid key: vault keys are bucket-relative and the vault is already " +
         "company-scoped — do not prefix with 'companies/<slug>/'. This " +
         "usually means a stale doubled local tree " +
         "(companies/<slug>/companies/<slug>/…); remove the inner copy.",
-      "INVALID_KEY_COMPANIES_SCOPED",
-    );
+    };
   }
   if (KEY_CONTROL_CHARS.test(key)) {
-    fail("Invalid key: contains control characters", "INVALID_KEY_CONTROL_CHARS");
+    return {
+      code: "INVALID_KEY_CONTROL_CHARS",
+      message: "Invalid key: contains control characters",
+    };
   }
   if (key.includes("\\")) {
-    // Unreachable after toPosixKey normalization at the upload boundary, but
-    // kept so the validator mirrors the server rule set 1:1 for other callers.
-    fail(
-      "Invalid key: backslash separators are not allowed — vault keys are POSIX ('/')",
-      "INVALID_KEY_BACKSLASH",
-    );
+    return {
+      code: "INVALID_KEY_BACKSLASH",
+      message:
+        "Invalid key: backslash separators are not allowed — vault keys are POSIX ('/')",
+    };
   }
   if (key.includes("//")) {
-    fail("Invalid key: consecutive slashes not allowed", "INVALID_KEY_DOUBLE_SLASH");
+    return {
+      code: "INVALID_KEY_DOUBLE_SLASH",
+      message: "Invalid key: consecutive slashes not allowed",
+    };
   }
   for (const segment of key.split("/")) {
     if (segment === "." || segment === "..") {
-      fail(
-        "Invalid key: '.' and '..' path components are not allowed",
-        "INVALID_KEY_DOT_COMPONENT",
-      );
+      return {
+        code: "INVALID_KEY_DOT_COMPONENT",
+        message: "Invalid key: '.' and '..' path components are not allowed",
+      };
     }
   }
+  return null;
+}
+
+export function validateVaultUploadKey(
+  key: string,
+  scope: VaultKeyScope,
+): void {
+  const issue = classifyVaultKey(key, scope);
+  if (issue !== null) {
+    throw Object.assign(new Error(issue.message), { code: issue.code });
+  }
 }
 
 /** Scope for {@link validateVaultUploadKey}, derived from the entity uid. */
-function uploadScopeFor(ctx: EntityContext): "company" | "personal" {
+function uploadScopeFor(ctx: EntityContext): VaultKeyScope {
   return ctx.uid.startsWith("cmp_") ? "company" : "personal";
 }