@indigoai-us/hq-cloud 6.14.15 → 6.14.17

package/src/s3.test.ts

@@ -96,6 +96,7 @@ import {
   classifyVaultKey,
   validateVaultUploadKey,
   replaceStagedPath,
+  sweepStaleStagedFiles,
 } from "./s3.js";
 import {
   setObjectIOFactory,
@@ -1580,3 +1581,55 @@ describe("validateVaultUploadKey — direct-S3 key poisoning guard (incident 202
     expect(sentCommands.length).toBeGreaterThan(0);
   });
 });
+
+describe("sweepStaleStagedFiles", () => {
+  let tmpRoot: string;
+  const HOUR = 60 * 60 * 1000;
+
+  beforeEach(() => {
+    tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-sweep-test-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpRoot, { recursive: true, force: true });
+  });
+
+  function makeAged(name: string, ageMs: number): string {
+    const full = path.join(tmpRoot, name);
+    fs.writeFileSync(full, "x");
+    const stamp = new Date(Date.now() - ageMs);
+    fs.utimesSync(full, stamp, stamp);
+    return full;
+  }
+
+  it("removes stale .hq-tmp-* and .hq-backup-* files, keeps everything else", () => {
+    const staleTmp = makeAged(".hq-tmp-deadbeef01", 2 * HOUR);
+    const staleBackup = makeAged(".hq-backup-deadbeef02", 2 * HOUR);
+    const freshTmp = makeAged(".hq-tmp-deadbeef03", 0);
+    const ordinary = makeAged("notes.md", 2 * HOUR);
+    // A directory matching the pattern must never be touched.
+    const dirSentinel = path.join(tmpRoot, ".hq-tmp-a-directory");
+    fs.mkdirSync(dirSentinel);
+
+    const removed = sweepStaleStagedFiles(tmpRoot);
+
+    expect(removed.sort()).toEqual([staleTmp, staleBackup].sort());
+    expect(fs.existsSync(staleTmp)).toBe(false);
+    expect(fs.existsSync(staleBackup)).toBe(false);
+    expect(fs.existsSync(freshTmp)).toBe(true);
+    expect(fs.existsSync(ordinary)).toBe(true);
+    expect(fs.existsSync(dirSentinel)).toBe(true);
+  });
+
+  it("returns empty for a missing directory without throwing", () => {
+    expect(
+      sweepStaleStagedFiles(path.join(tmpRoot, "does-not-exist")),
+    ).toEqual([]);
+  });
+
+  it("respects a custom age threshold", () => {
+    const halfHourOld = makeAged(".hq-tmp-halfhour", HOUR / 2);
+    expect(sweepStaleStagedFiles(tmpRoot, HOUR / 4)).toEqual([halfHourOld]);
+    expect(fs.existsSync(halfHourOld)).toBe(false);
+  });
+});

package/src/s3.ts

@@ -239,8 +239,76 @@ export function encodeSymlinkBody(target: string): Buffer {
   return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
 }
 
+/**
+ * Staged-download (`.hq-tmp-*`) and replacement-backup (`.hq-backup-*`)
+ * sentinels are cleaned by try/finally in-process, but a hard kill
+ * mid-transfer (agent job `timeout -k`, OOM, systemd stop) never reaches the
+ * finally and nothing else removes them. On a fleet box that leaked 866M of
+ * orphans in two days and filled /tmp (2026-07-20 incident). Sweep stale
+ * sentinels in the destination dir before staging a new one there.
+ *
+ * The age threshold far exceeds any live transfer, so a concurrent download's
+ * fresh sentinel is never touched. Only non-directories matching the sentinel
+ * pattern are candidates. Per-dir memoization bounds the readdir cost to once
+ * per directory per process run.
+ */
+const STAGED_SENTINEL_PATTERN = /^\.hq-(tmp|backup)-/;
+const STAGED_SWEEP_MAX_AGE_MS = 60 * 60 * 1000;
+const sweptStagedDirs = new Set<string>();
+
+export function sweepStaleStagedFiles(
+  dir: string,
+  maxAgeMs: number = STAGED_SWEEP_MAX_AGE_MS,
+  nowMs: number = Date.now(),
+): string[] {
+  let entries: string[];
+  try {
+    entries = fs.readdirSync(dir);
+  } catch (err) {
+    // Destination dir may not exist yet (created later by downloadFile) —
+    // nothing to sweep. Anything else is unexpected but must not fail the
+    // download that triggered the sweep.
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      console.warn(`[hq-sync] staged-file sweep: cannot read ${dir}: ${err}`);
+    }
+    return [];
+  }
+  const removed: string[] = [];
+  for (const name of entries) {
+    if (!STAGED_SENTINEL_PATTERN.test(name)) continue;
+    const full = path.join(dir, name);
+    try {
+      const stat = fs.lstatSync(full);
+      if (stat.isDirectory()) continue;
+      if (nowMs - stat.mtimeMs < maxAgeMs) continue;
+      fs.unlinkSync(full);
+      removed.push(full);
+    } catch (err) {
+      // A concurrent process may have removed it first (ENOENT: fine), or the
+      // file is unremovable (EPERM/EROFS) — either way the download must
+      // proceed; surface non-ENOENT so the leak stays visible.
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+        console.warn(`[hq-sync] staged-file sweep: cannot remove ${full}: ${err}`);
+      }
+    }
+  }
+  if (removed.length > 0) {
+    console.warn(
+      `[hq-sync] removed ${removed.length} stale staged file(s) in ${dir}`,
+    );
+  }
+  return removed;
+}
+
+function sweepStaleStagedFilesOnce(dir: string): void {
+  if (sweptStagedDirs.has(dir)) return;
+  sweptStagedDirs.add(dir);
+  sweepStaleStagedFiles(dir);
+}
+
 function downloadTempPath(localPath: string): string {
   const dir = path.dirname(localPath);
+  sweepStaleStagedFilesOnce(dir);
   const random = crypto.randomBytes(10).toString("hex");
   return path.join(dir, `.hq-tmp-${random}`);
 }

package/src/vault-client.test.ts

@@ -365,7 +365,7 @@ describe("API surface", () => {
 
   it("postTelemetryEvents sends action events without personUid", async () => {
     fetchSpy.mockResolvedValueOnce(
-      jsonResponse(200, { ok: true, written: 1, skipped: [] }),
+      jsonResponse(200, { contractVersion: 1, ok: true, written: 1, skipped: [] }),
     );
     const batch: TelemetryEventsBatch = {
       events: [
@@ -397,6 +397,17 @@ describe("API surface", () => {
     expect(JSON.stringify(body)).not.toContain("personUid");
   });
 
+  it("postTelemetryEvents normalizes the compatible pre-v1 accepted response", async () => {
+    fetchSpy.mockResolvedValueOnce(jsonResponse(200, { accepted: 2, deduped: 1 }));
+    const batch: TelemetryEventsBatch = { events: [] };
+
+    await expect(client.postTelemetryEvents(batch)).resolves.toEqual({
+      ok: true,
+      written: 2,
+      skipped: [],
+    });
+  });
+
   it("updateRole sends correct payload", async () => {
     fetchSpy.mockResolvedValueOnce(
       jsonResponse(200, {

package/src/vault-client.ts

@@ -462,7 +462,14 @@ export interface TelemetryEventsBatch {
   events: RawTelemetryEventInput[];
 }
 
-/** Same wire shape as `UsageIngestResult`; aliased for call-site clarity. */
+/**
+ * Normalized response shape for `POST /v1/telemetry/events`.
+ *
+ * hq-pro response contract v1 uses the established ingest fields (`ok`,
+ * `written`, `skipped`). The decoder also accepts the briefly deployed
+ * `{ accepted, deduped }` shape so clients remain compatible while the server
+ * rollout catches up.
+ */
 export type TelemetryEventsIngestResult = UsageIngestResult;
 
 // ---------------------------------------------------------------------------
@@ -649,6 +656,41 @@ const usageIngestResultSchema: VaultResponseSchema<UsageIngestResult> = z
   })
   .strip();
 
+/**
+ * Accept the canonical v1 response and the previous raw-event aliases, then
+ * normalize at this boundary so telemetry remains best-effort during rollout.
+ */
+const telemetryEventsIngestResultSchema: VaultResponseSchema<TelemetryEventsIngestResult> = z
+  .object({
+    contractVersion: z.literal(1).optional(),
+    ok: z.boolean().optional(),
+    written: z.number().nonnegative().optional(),
+    skipped: z
+      .array(
+        z
+          .object({
+            index: z.number(),
+            code: z.string(),
+            error: z.string(),
+          })
+          .strip(),
+      )
+      .optional(),
+    // Compatibility aliases emitted by hq-pro before response contract v1.
+    accepted: z.number().nonnegative().optional(),
+    deduped: z.number().nonnegative().optional(),
+  })
+  .refine(
+    (value) =>
+      value.ok !== undefined || value.written !== undefined || value.accepted !== undefined,
+    { message: "telemetry ingest response has no recognized result fields" },
+  )
+  .transform((value) => ({
+    ok: value.ok ?? true,
+    written: value.written ?? value.accepted ?? 0,
+    skipped: value.skipped ?? [],
+  }));
+
 const createInviteResponseSchema: VaultResponseSchema<CreateInviteResult> = z
   .object({
     membership: membershipSchema,
@@ -1349,7 +1391,7 @@ export class VaultClient {
     batch: TelemetryEventsBatch,
     options: { timeoutMs?: number } = {},
   ): Promise<TelemetryEventsIngestResult> {
-    return this.post("/v1/telemetry/events", batch, usageIngestResultSchema, {
+    return this.post("/v1/telemetry/events", batch, telemetryEventsIngestResultSchema, {
       timeoutMs: options.timeoutMs ?? 1500,
       maxRetries: 0,
     });