@indigoai-us/hq-cloud 6.14.30 → 6.14.31

package/src/telemetry.ts

@@ -77,6 +77,10 @@ export interface CollectTelemetryOptions {
   claudeProjectsRoot?: string;
   /** Override `~/.codex` for tests. Both live and archived rollouts are scanned. */
   codexRoot?: string;
+  /** Override the per-runtime scan budget for deterministic tests. */
+  maxScanBytesPerSource?: number;
+  /** Override the upload batch cap for deterministic tests. */
+  maxBatchesPerRun?: number;
   /** Override `~/.hq/telemetry-cursor.json` for tests. */
   cursorPath?: string;
   /** Override `~/.hq/menubar.json` (the offline opt-in fallback) for tests. */
@@ -107,6 +111,7 @@ interface CodexUsageContext {
   session_model?: string;
   collaboration_model?: string;
   turn_model?: string;
+  discarding_partial_line?: boolean;
 }
 
 interface CursorEntry {
@@ -545,13 +550,32 @@ function codexUsageRow(
   return normalized;
 }
 
+async function sortFilesFreshestFirst(files: string[]): Promise<string[]> {
+  const ranked = await Promise.all(
+    files.map(async (filePath) => {
+      try {
+        return { filePath, mtime: (await fs.stat(filePath)).mtimeMs };
+      } catch {
+        return { filePath, mtime: 0 };
+      }
+    }),
+  );
+  ranked.sort(
+    (left, right) =>
+      right.mtime - left.mtime || left.filePath.localeCompare(right.filePath),
+  );
+  return ranked.map(({ filePath }) => filePath);
+}
+
 async function listCodexRolloutFiles(codexRoot: string): Promise<string[]> {
   const [live, archived] = await Promise.all([
     listJsonlFiles(path.join(codexRoot, "sessions")),
     listJsonlFiles(path.join(codexRoot, "archived_sessions")),
   ]);
-  return [...live, ...archived].filter((file) =>
-    path.basename(file).startsWith("rollout-"),
+  return sortFilesFreshestFirst(
+    [...live, ...archived].filter((file) =>
+      path.basename(file).startsWith("rollout-"),
+    ),
   );
 }
 
@@ -559,6 +583,9 @@ async function listCodexRolloutFiles(codexRoot: string): Promise<string[]> {
 
 const MAX_BATCH_EVENTS = 100;
 const MAX_BATCH_BYTES = 240 * 1024;
+const MAX_SCAN_BYTES_PER_SOURCE = 4 * 1024 * 1024;
+const MAX_BATCHES_PER_RUN = 4;
+const MAX_PARTIAL_LINE_BYTES = 64 * 1024;
 const ROW_TRUNCATION_SUFFIX = "...[truncated]";
 
 interface RowSource {
@@ -650,6 +677,9 @@ export async function collectAndSendTelemetry(
   const cursorPath = opts.cursorPath ?? path.join(home, ".hq", "telemetry-cursor.json");
   const menubarPath = opts.menubarPath ?? path.join(home, ".hq", "menubar.json");
   const log = opts.log ?? (() => {});
+  const maxScanBytesPerSource =
+    opts.maxScanBytesPerSource ?? MAX_SCAN_BYTES_PER_SOURCE;
+  const maxBatchesPerRun = opts.maxBatchesPerRun ?? MAX_BATCHES_PER_RUN;
 
   // Company attribution (US-002): parse the manifest ONCE per run and reuse the
   // repo-path→companyUid map for every event below. No per-event manifest read.
@@ -745,13 +775,16 @@ export async function collectAndSendTelemetry(
   const rotationResets: Record<string, CursorEntry> = {};
   const newlyCommitted: Record<string, CursorEntry> = {};
 
-  const [claudeFiles, codexFiles] = await Promise.all([
+  const [claudeFilesUnsorted, codexFiles] = await Promise.all([
     listJsonlFiles(claudeProjectsRoot),
     listCodexRolloutFiles(codexRoot),
   ]);
+  const claudeFiles = await sortFilesFreshestFirst(claudeFilesUnsorted);
+  // Codex goes first so a fresh Linux install cannot spend its entire bounded
+  // pass replaying an older Claude backlog before reporting current Codex use.
   const files: Array<{ filePath: string; kind: "claude" | "codex" }> = [
-    ...claudeFiles.map((filePath) => ({ filePath, kind: "claude" as const })),
     ...codexFiles.map((filePath) => ({ filePath, kind: "codex" as const })),
+    ...claudeFiles.map((filePath) => ({ filePath, kind: "claude" as const })),
   ];
 
   // 3. Walk each file, sanitize new rows, batch, flush at the server contract.
@@ -768,15 +801,45 @@ export async function collectAndSendTelemetry(
   let batchBytes = ENVELOPE_BYTES;
   let eventsSent = 0;
   let batchesSent = 0;
+  let uploadFailed = false;
+  let batchLimitReached = false;
+  const scannedBytes = { claude: 0, codex: 0 };
+
+  const commitSources = (sources: RowSource[]): void => {
+    const maxPerFile = new Map<
+      string,
+      { mtime: number; offset: number; context?: CodexUsageContext }
+    >();
+    for (const source of sources) {
+      const current = maxPerFile.get(source.filePath);
+      if (!current || source.endOffset > current.offset) {
+        maxPerFile.set(source.filePath, {
+          mtime: source.mtime,
+          offset: source.endOffset,
+          ...(source.context
+            ? { context: cloneCodexContext(source.context) }
+            : {}),
+        });
+      }
+    }
+    for (const [filePath, entry] of maxPerFile) {
+      const current = newlyCommitted[filePath];
+      if (!current || entry.offset > current.offset) {
+        newlyCommitted[filePath] = entry;
+      }
+    }
+  };
 
-  const flush = async (): Promise<void> => {
-    if (batchEvents.length === 0) return;
+  const flush = async (): Promise<boolean> => {
     const events = batchEvents;
     const sources = batchSources;
     batchEvents = [];
     batchSources = [];
     batchBytes = ENVELOPE_BYTES;
-
+    if (events.length === 0) {
+      commitSources(sources);
+      return true;
+    }
     try {
       await opts.client.postUsage({
         machineId: opts.machineId,
@@ -785,35 +848,26 @@ export async function collectAndSendTelemetry(
       });
       batchesSent++;
       eventsSent += events.length;
-      // Advance cursor to max(endOffset) per file in this batch.
-      const maxPerFile = new Map<
-        string,
-        { mtime: number; offset: number; context?: CodexUsageContext }
-      >();
-      for (const src of sources) {
-        const cur = maxPerFile.get(src.filePath);
-        if (!cur || src.endOffset > cur.offset) {
-          maxPerFile.set(src.filePath, {
-            mtime: src.mtime,
-            offset: src.endOffset,
-            ...(src.context ? { context: cloneCodexContext(src.context) } : {}),
-          });
-        }
-      }
-      for (const [fp, entry] of maxPerFile) {
-        newlyCommitted[fp] = {
-          offset: entry.offset,
-          mtime: entry.mtime,
-          ...(entry.context ? { context: entry.context } : {}),
-        };
-      }
+      commitSources(sources);
+      return true;
     } catch (err) {
-      log(`[telemetry] postUsage failed (${(err as Error).message ?? err}) — cursor not advanced for ${sources.length} rows`);
-      // Cursor intentionally left un-advanced — next sync retries.
+      log(
+        "[telemetry] postUsage failed (" +
+          ((err as Error).message ?? err) +
+          ") — cursor not advanced for " +
+          sources.length +
+          " rows",
+      );
+      return false;
     }
   };
 
-  for (const { filePath, kind } of files) {
+  filesLoop: for (const { filePath, kind } of files) {
+    if (uploadFailed || batchLimitReached) break;
+    const sourceBudgetRemaining =
+      maxScanBytesPerSource - scannedBytes[kind];
+    if (sourceBudgetRemaining <= 0) continue;
+
     let stat;
     try {
       stat = await fs.stat(filePath);
@@ -822,12 +876,10 @@ export async function collectAndSendTelemetry(
     }
     const currentSize = stat.size;
     const currentMtime = Math.floor(stat.mtimeMs / 1000);
-
     const stored = cursor.files[filePath] ?? { offset: 0, mtime: 0 };
     let offset = stored.offset;
     let codexContext: CodexUsageContext | undefined =
       kind === "codex" ? cloneCodexContext(stored.context ?? {}) : undefined;
-
     const rotated =
       currentSize < offset || (stored.mtime > 0 && currentMtime < stored.mtime);
     if (rotated) {
@@ -835,44 +887,117 @@ export async function collectAndSendTelemetry(
       if (kind === "codex") codexContext = {};
       rotationResets[filePath] = { offset: 0, mtime: currentMtime };
     }
-
     if (offset >= currentSize && !rotated) continue;
 
     let content: string;
+    let bytesRead = 0;
+    let readBuffer = Buffer.alloc(0);
     try {
-      const fh = await fs.open(filePath, "r");
+      const fileHandle = await fs.open(filePath, "r");
       try {
-        const length = Math.max(0, currentSize - offset);
-        const buf = Buffer.alloc(length);
-        await fh.read(buf, 0, length, offset);
-        content = buf.toString("utf-8");
+        const length = Math.max(
+          0,
+          Math.min(currentSize - offset, sourceBudgetRemaining),
+        );
+        readBuffer = Buffer.alloc(length);
+        ({ bytesRead } = await fileHandle.read(
+          readBuffer,
+          0,
+          length,
+          offset,
+        ));
+        content = readBuffer.toString("utf-8", 0, bytesRead);
       } finally {
-        await fh.close();
+        await fileHandle.close();
       }
     } catch {
       continue;
     }
-    if (content.length === 0) continue;
+    if (bytesRead === 0) continue;
+
+    // Resume a previously discarded oversized Codex row using raw byte
+    // positions. Decoding from the middle of a multibyte UTF-8 codepoint would
+    // otherwise skew byte offsets and could skip or replay later rows.
+    if (codexContext?.discarding_partial_line) {
+      const newlineIndex = readBuffer.subarray(0, bytesRead).indexOf(0x0a);
+      const discardedBytes = newlineIndex >= 0 ? newlineIndex + 1 : bytesRead;
+      codexContext.discarding_partial_line = newlineIndex < 0;
+      scannedBytes[kind] += discardedBytes;
+      batchSources.push({
+        filePath,
+        endOffset: offset + discardedBytes,
+        mtime: currentMtime,
+        context: cloneCodexContext(codexContext),
+      });
+      if (newlineIndex < 0) continue;
+      offset += discardedBytes;
+      bytesRead -= discardedBytes;
+      readBuffer = readBuffer.subarray(discardedBytes);
+      content = readBuffer.toString("utf-8", 0, bytesRead);
+      if (bytesRead === 0) continue;
+    }
 
+    const reachedPhysicalEof = offset + bytesRead >= currentSize;
     const segments = content.split("\n");
     const lineEndOffsets: number[] = [];
     let cumulative = 0;
     for (let i = 0; i < segments.length; i++) {
       cumulative += Buffer.byteLength(segments[i], "utf-8");
-      if (i < segments.length - 1) cumulative += 1; // newline byte
+      if (i < segments.length - 1) cumulative += 1;
       lineEndOffsets.push(offset + cumulative);
     }
 
     for (let i = 0; i < segments.length; i++) {
+      const startOffset = i === 0 ? offset : lineEndOffsets[i - 1];
+      const hasNewline = i < segments.length - 1;
+      const endOffset =
+        !hasNewline && !reachedPhysicalEof
+          ? offset + bytesRead
+          : lineEndOffsets[i];
+      const lineBytes = endOffset - startOffset;
+      if (!hasNewline && !reachedPhysicalEof) {
+        const discarding = codexContext?.discarding_partial_line === true;
+        if (discarding || lineBytes >= MAX_PARTIAL_LINE_BYTES) {
+          if (codexContext) codexContext.discarding_partial_line = true;
+          scannedBytes[kind] += lineBytes;
+          batchSources.push({
+            filePath,
+            endOffset,
+            mtime: currentMtime,
+            ...(codexContext
+              ? { context: cloneCodexContext(codexContext) }
+              : {}),
+          });
+        }
+        break;
+      }
+
+      scannedBytes[kind] += lineBytes;
+      const source = (): RowSource => ({
+        filePath,
+        endOffset,
+        mtime: currentMtime,
+        ...(codexContext
+          ? { context: cloneCodexContext(codexContext) }
+          : {}),
+      });
+      if (codexContext?.discarding_partial_line) {
+        codexContext.discarding_partial_line = false;
+        batchSources.push(source());
+        continue;
+      }
       const trimmed = segments[i].trim();
-      if (trimmed.length === 0) continue;
+      if (trimmed.length === 0) {
+        batchSources.push(source());
+        continue;
+      }
       let parsed: unknown;
       try {
         parsed = JSON.parse(trimmed);
       } catch {
+        batchSources.push(source());
         continue;
       }
-      const startOffset = i === 0 ? offset : lineEndOffsets[i - 1];
       const sourceRow =
         kind === "codex"
           ? codexUsageRow(
@@ -880,11 +1005,9 @@ export async function collectAndSendTelemetry(
               codexContext!,
               filePath,
               startOffset,
-              lineEndOffsets[i],
+              endOffset,
             )
           : parsed;
-      // Resolve this row's cwd → owning company (cmp_* uid) before sanitizing,
-      // using the per-run map. Unresolved → undefined → companyUid omitted.
       const rowCwd =
         sourceRow && typeof sourceRow === "object" && !Array.isArray(sourceRow)
           ? (sourceRow as Record<string, unknown>).cwd
@@ -894,52 +1017,62 @@ export async function collectAndSendTelemetry(
         repoCompanyMap,
       );
       const sanitized = sanitizeRow(sourceRow, companyUid);
-      if (!sanitized) continue;
+      if (!sanitized) {
+        batchSources.push(source());
+        continue;
+      }
+
       const maxRowBytes = MAX_BATCH_BYTES - ENVELOPE_BYTES;
       const wasOversized = jsonBytes(sanitized) > maxRowBytes;
       const bounded = boundRowForPost(sanitized, maxRowBytes);
       if (!bounded) {
         log(
-          `[telemetry] oversized row dropped before send (${filePath}:${i + 1})`,
+          "[telemetry] oversized row dropped before send (" +
+            filePath +
+            ":" +
+            (i + 1) +
+            ")",
         );
+        batchSources.push(source());
         continue;
       }
       if (wasOversized) {
         log(
-          `[telemetry] oversized row truncated before send (${filePath}:${i + 1})`,
+          "[telemetry] oversized row truncated before send (" +
+            filePath +
+            ":" +
+            (i + 1) +
+            ")",
         );
       }
 
-      // Cost of appending this row to the current batch: the row's JSON
-      // length plus 1 byte for the leading comma when there's already at
-      // least one row. (No comma when the batch is empty — the row sits
-      // alone inside the events array.)
-      const rowJsonBytes = Buffer.byteLength(JSON.stringify(bounded), "utf-8");
+      const rowJsonBytes = jsonBytes(bounded);
       const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
-
       if (
         batchEvents.length > 0 &&
         (batchEvents.length >= MAX_BATCH_EVENTS ||
           batchBytes + addCost > MAX_BATCH_BYTES)
       ) {
-        await flush();
-        // After flush, batchEvents is empty → no comma needed for the first row.
+        if (!(await flush())) {
+          uploadFailed = true;
+          break filesLoop;
+        }
+        if (batchesSent >= maxBatchesPerRun) {
+          batchLimitReached = true;
+          break filesLoop;
+        }
         batchBytes = ENVELOPE_BYTES + rowJsonBytes;
       } else {
         batchBytes += addCost;
       }
-
       batchEvents.push(bounded);
-      batchSources.push({
-        filePath,
-        endOffset: lineEndOffsets[i],
-        mtime: currentMtime,
-        ...(codexContext ? { context: cloneCodexContext(codexContext) } : {}),
-      });
+      batchSources.push(source());
     }
   }
 
-  await flush();
+  if (!uploadFailed && !batchLimitReached) {
+    uploadFailed = !(await flush());
+  }
 
   // 4. Persist cursor: loaded < rotation_resets < newly_committed.
   const finalFiles: Record<string, CursorEntry> = { ...loadedFiles };