@indigoai-us/hq-cloud 5.19.0 → 5.20.0

package/src/telemetry.ts

@@ -0,0 +1,436 @@
+/**
+ * Usage telemetry collector — TypeScript port of the Tauri Rust collector that
+ * used to live at `hq-workspace/apps/hq-sync/src-tauri/src/commands/telemetry.rs`.
+ *
+ * Why it moved: the Rust copy only ran inside the macOS menubar app. By moving
+ * the logic into `@indigoai-us/hq-cloud`, every consumer of the package
+ * (`hq-sync-runner`, `hq-cli`, mobile wrappers) emits telemetry uniformly.
+ *
+ * What it does: after each successful sync (`all-complete` arm of
+ * `bin/sync-runner.ts`), walks `~/.claude/projects/**\/*.jsonl`, diffs each
+ * file against a persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`,
+ * sanitizes new rows through a tight allowlist that matches the server's
+ * KEEP_FIELDS set in `apps/hq-pro/src/vault-service/handlers/usage.ts`,
+ * batches into ≤1 MiB POST bodies, and ships them to `/v1/usage`.
+ *
+ * Trust model: the caller's `personUid` is resolved on the server from the
+ * Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
+ * thinking content, tool inputs/outputs, and any nested `message` object so
+ * the wire payload contains only token-accounting fields.
+ *
+ * Errors are swallowed by design — telemetry must never abort or delay a
+ * sync. The cursor is only advanced for batches the server 2xx'd, so a
+ * transient outage retries automatically on the next sync.
+ */
+
+import { promises as fs } from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+
+import type {
+  TelemetryOptInResponse,
+  UsageBatch,
+  UsageIngestResult,
+} from "./vault-client.js";
+
+// ── Public surface ────────────────────────────────────────────────────────────
+
+/**
+ * Minimal subset of `VaultClient` the collector needs. Declared as an
+ * interface so tests can inject a stub without spinning up a fetch mock.
+ * The real `VaultClient` from `./vault-client.js` satisfies this structurally.
+ */
+export interface TelemetryClientSurface {
+  getTelemetryOptIn(): Promise<TelemetryOptInResponse>;
+  postUsage(batch: UsageBatch): Promise<UsageIngestResult>;
+}
+
+export interface CollectTelemetryOptions {
+  client: TelemetryClientSurface;
+  /** Stable per-machine id. The Tauri menubar reads this from `~/.hq/menubar.json`; the runner can pass it through or generate one once and cache. */
+  machineId: string;
+  /** Version of the wrapping caller (menubar app, CLI, etc.). Reaches CloudWatch metrics as the `installerVersion` dimension. */
+  installerVersion: string;
+  /** Override `~/.claude/projects` for tests. */
+  claudeProjectsRoot?: string;
+  /** Override `~/.hq/telemetry-cursor.json` for tests. */
+  cursorPath?: string;
+  /** Override `~/.hq/menubar.json` (the offline opt-in fallback) for tests. */
+  menubarPath?: string;
+  /** Diagnostic sink. No-op by default. */
+  log?: (msg: string) => void;
+}
+
+export interface CollectTelemetryResult {
+  /** Whether the opt-in check resolved to true (either server-side or via the menubar fallback). When false, nothing else ran. */
+  enabled: boolean;
+  /** Source for the `enabled` decision — useful for diagnosing missing-events reports. */
+  optInSource: "server" | "menubar-fallback" | "skipped";
+  /** How many `.jsonl` files we considered (before the cursor diff). */
+  filesScanned: number;
+  /** Total events successfully POSTed across all batches. */
+  eventsSent: number;
+  /** Number of `POST /v1/usage` requests made. */
+  batchesSent: number;
+}
+
+// ── Cursor schema ─────────────────────────────────────────────────────────────
+
+interface CursorEntry {
+  offset: number;
+  mtime: number;
+}
+
+interface TelemetryCursor {
+  version: string;
+  files: Record<string, CursorEntry>;
+}
+
+function emptyCursor(): TelemetryCursor {
+  return { version: "1", files: {} };
+}
+
+async function loadCursor(cursorPath: string): Promise<TelemetryCursor> {
+  try {
+    const raw = await fs.readFile(cursorPath, "utf-8");
+    const parsed = JSON.parse(raw) as Partial<TelemetryCursor>;
+    if (parsed && typeof parsed === "object" && parsed.files && typeof parsed.files === "object") {
+      return { version: parsed.version ?? "1", files: parsed.files as Record<string, CursorEntry> };
+    }
+  } catch {
+    // Missing / unparseable — start fresh.
+  }
+  return emptyCursor();
+}
+
+async function saveCursor(cursorPath: string, cursor: TelemetryCursor): Promise<void> {
+  // Atomic write: tmp + rename. The Rust impl uses the same .tmp suffix; we
+  // keep it for cross-implementation grep-ability.
+  await fs.mkdir(path.dirname(cursorPath), { recursive: true });
+  const tmp = `${cursorPath}.tmp`;
+  await fs.writeFile(tmp, JSON.stringify(cursor, null, 2), "utf-8");
+  await fs.rename(tmp, cursorPath);
+}
+
+// ── Local opt-in fallback ─────────────────────────────────────────────────────
+
+async function readLocalTelemetryEnabled(menubarPath: string): Promise<boolean> {
+  try {
+    const raw = await fs.readFile(menubarPath, "utf-8");
+    const parsed = JSON.parse(raw) as { telemetryEnabled?: unknown };
+    return parsed.telemetryEnabled === true;
+  } catch {
+    return false;
+  }
+}
+
+// ── Sanitizer ─────────────────────────────────────────────────────────────────
+
+/** Top-level fields the server accepts. Keep aligned with `KEEP_FIELDS` in
+ *  `apps/hq-pro/src/vault-service/handlers/usage.ts` — any drift will surface
+ *  as an `unexpected-event-field` rejection in `UsageIngestResult.skipped`. */
+const KEEP_TOP_LEVEL = [
+  "sessionId",
+  "timestamp",
+  "uuid",
+  "cwd",
+  "gitBranch",
+  "userType",
+] as const;
+
+/**
+ * Build an outgoing event row matching the server's KEEP allowlist.
+ *
+ * Two transforms:
+ *   1. Top-level fields are copied straight through (string identity).
+ *   2. `message.model` and `message.usage.{input_tokens, output_tokens,
+ *      cache_creation_input_tokens, cache_read_input_tokens}` are promoted to
+ *      camelCase top-level fields. The original `message` object — which
+ *      carries prompt/response text, thinking, and tool data — is dropped.
+ *
+ * Returns `null` when the input isn't an object. Empty results (e.g. a row
+ * with no recognised fields) are still returned as `{}` and emitted; the
+ * server accepts empty rows and they're useful as a "Claude Code was run at
+ * this time" heartbeat.
+ */
+export function sanitizeRow(row: unknown): Record<string, unknown> | null {
+  if (!row || typeof row !== "object" || Array.isArray(row)) return null;
+  const obj = row as Record<string, unknown>;
+  const out: Record<string, unknown> = {};
+
+  for (const key of KEEP_TOP_LEVEL) {
+    if (key in obj) {
+      out[key] = obj[key];
+    }
+  }
+
+  const message = obj.message;
+  if (message && typeof message === "object" && !Array.isArray(message)) {
+    const m = message as Record<string, unknown>;
+    if ("model" in m) out.model = m.model;
+    const usage = m.usage;
+    if (usage && typeof usage === "object" && !Array.isArray(usage)) {
+      const u = usage as Record<string, unknown>;
+      if ("input_tokens" in u) out.inputTokens = u.input_tokens;
+      if ("output_tokens" in u) out.outputTokens = u.output_tokens;
+      if ("cache_creation_input_tokens" in u) {
+        out.cacheCreationInputTokens = u.cache_creation_input_tokens;
+      }
+      if ("cache_read_input_tokens" in u) {
+        out.cacheReadInputTokens = u.cache_read_input_tokens;
+      }
+    }
+  }
+
+  return out;
+}
+
+// ── File walker ───────────────────────────────────────────────────────────────
+
+/** Recursively collect every `.jsonl` file under `root`. Skips errors silently
+ *  (missing dir, EACCES on a stray subdir) — anything we can't enter is
+ *  treated as absent rather than fatal, matching the Rust glob behavior. */
+async function listJsonlFiles(root: string): Promise<string[]> {
+  const out: string[] = [];
+  async function walk(dir: string): Promise<void> {
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const ent of entries) {
+      const full = path.join(dir, ent.name);
+      if (ent.isDirectory()) {
+        await walk(full);
+      } else if (ent.isFile() && ent.name.endsWith(".jsonl")) {
+        out.push(full);
+      }
+    }
+  }
+  await walk(root);
+  return out;
+}
+
+// ── Batching primitives ───────────────────────────────────────────────────────
+
+const MAX_BATCH_BYTES = 1_000_000;
+
+interface RowSource {
+  filePath: string;
+  endOffset: number;
+  mtime: number;
+}
+
+/**
+ * Byte cost of the fixed wire-payload skeleton:
+ *   {"machineId":"…","installerVersion":"…","events":[]}
+ *
+ * The Rust implementation re-serializes the entire growing batch on every
+ * row to check size — O(n²) bytes of JSON.stringify work per batch, which
+ * for 60K events takes ~4 minutes wall-clock in V8. This computes the same
+ * payload size incrementally instead: skeleton + Σ(per-row JSON length) +
+ * commas between rows. Same threshold semantics, O(n) total cost.
+ */
+function envelopeBytes(machineId: string, installerVersion: string): number {
+  // Serialize the empty-events envelope once and measure it. Captures the
+  // exact JSON whitespace / escaping V8 produces so we match what would
+  // actually go over the wire.
+  return Buffer.byteLength(
+    JSON.stringify({ machineId, installerVersion, events: [] }),
+    "utf-8",
+  );
+}
+
+// ── Main entry point ──────────────────────────────────────────────────────────
+
+/**
+ * Scan, sanitize, and POST any new Claude Code session rows.
+ *
+ * Fire-and-forget from the caller's perspective: errors are caught internally
+ * and surfaced only via `log`. The returned summary lets observers (e.g.
+ * sync-runner) decide whether to record a "telemetry attempted" breadcrumb,
+ * but no consumer is expected to react to it.
+ */
+export async function collectAndSendTelemetry(
+  opts: CollectTelemetryOptions,
+): Promise<CollectTelemetryResult> {
+  const home = os.homedir();
+  const claudeProjectsRoot = opts.claudeProjectsRoot ?? path.join(home, ".claude", "projects");
+  const cursorPath = opts.cursorPath ?? path.join(home, ".hq", "telemetry-cursor.json");
+  const menubarPath = opts.menubarPath ?? path.join(home, ".hq", "menubar.json");
+  const log = opts.log ?? (() => {});
+
+  // 1. Opt-in check (server-authoritative, with local fallback).
+  let enabled: boolean;
+  let optInSource: CollectTelemetryResult["optInSource"];
+  try {
+    const resp = await opts.client.getTelemetryOptIn();
+    enabled = resp.enabled === true;
+    optInSource = "server";
+  } catch (err) {
+    log(`[telemetry] opt-in check failed (${(err as Error).message ?? err}) — falling back to local menubar.json`);
+    enabled = await readLocalTelemetryEnabled(menubarPath);
+    optInSource = "menubar-fallback";
+  }
+
+  if (!enabled) {
+    return { enabled: false, optInSource, filesScanned: 0, eventsSent: 0, batchesSent: 0 };
+  }
+
+  // 2. Cursor + file enumeration.
+  const cursor = await loadCursor(cursorPath);
+  const loadedFiles = { ...cursor.files };
+  const rotationResets: Record<string, CursorEntry> = {};
+  const newlyCommitted: Record<string, CursorEntry> = {};
+
+  const files = await listJsonlFiles(claudeProjectsRoot);
+
+  // 3. Walk each file, sanitize new rows, batch, flush at 1 MiB.
+  //
+  // Byte accounting is incremental: we track `batchBytes` as the projected
+  // serialized size of the current batch (envelope + per-row JSON + commas).
+  // Each row contributes its own JSON.stringify length once; we never
+  // re-serialize the growing batch. This is the O(n) replacement for the
+  // O(n²) projected-payload check the Rust impl uses (which spent ~4 min
+  // on a 60K-event first-run in the E2E smoke against hq-prod).
+  const ENVELOPE_BYTES = envelopeBytes(opts.machineId, opts.installerVersion);
+  let batchEvents: Array<Record<string, unknown>> = [];
+  let batchSources: RowSource[] = [];
+  let batchBytes = ENVELOPE_BYTES;
+  let eventsSent = 0;
+  let batchesSent = 0;
+
+  const flush = async (): Promise<void> => {
+    if (batchEvents.length === 0) return;
+    const events = batchEvents;
+    const sources = batchSources;
+    batchEvents = [];
+    batchSources = [];
+    batchBytes = ENVELOPE_BYTES;
+
+    try {
+      await opts.client.postUsage({
+        machineId: opts.machineId,
+        installerVersion: opts.installerVersion,
+        events,
+      });
+      batchesSent++;
+      eventsSent += events.length;
+      // Advance cursor to max(endOffset) per file in this batch.
+      const maxPerFile = new Map<string, { mtime: number; offset: number }>();
+      for (const src of sources) {
+        const cur = maxPerFile.get(src.filePath);
+        if (!cur || src.endOffset > cur.offset) {
+          maxPerFile.set(src.filePath, { mtime: src.mtime, offset: src.endOffset });
+        }
+      }
+      for (const [fp, entry] of maxPerFile) {
+        newlyCommitted[fp] = { offset: entry.offset, mtime: entry.mtime };
+      }
+    } catch (err) {
+      log(`[telemetry] postUsage failed (${(err as Error).message ?? err}) — cursor not advanced for ${sources.length} rows`);
+      // Cursor intentionally left un-advanced — next sync retries.
+    }
+  };
+
+  for (const filePath of files) {
+    let stat;
+    try {
+      stat = await fs.stat(filePath);
+    } catch {
+      continue;
+    }
+    const currentSize = stat.size;
+    const currentMtime = Math.floor(stat.mtimeMs / 1000);
+
+    const stored = cursor.files[filePath] ?? { offset: 0, mtime: 0 };
+    let offset = stored.offset;
+
+    const rotated =
+      currentSize < offset || (stored.mtime > 0 && currentMtime < stored.mtime);
+    if (rotated) {
+      offset = 0;
+      rotationResets[filePath] = { offset: 0, mtime: currentMtime };
+    }
+
+    if (offset >= currentSize && !rotated) continue;
+
+    let content: string;
+    try {
+      const fh = await fs.open(filePath, "r");
+      try {
+        const length = Math.max(0, currentSize - offset);
+        const buf = Buffer.alloc(length);
+        await fh.read(buf, 0, length, offset);
+        content = buf.toString("utf-8");
+      } finally {
+        await fh.close();
+      }
+    } catch {
+      continue;
+    }
+    if (content.length === 0) continue;
+
+    const segments = content.split("\n");
+    const lineEndOffsets: number[] = [];
+    let cumulative = 0;
+    for (let i = 0; i < segments.length; i++) {
+      cumulative += Buffer.byteLength(segments[i], "utf-8");
+      if (i < segments.length - 1) cumulative += 1; // newline byte
+      lineEndOffsets.push(offset + cumulative);
+    }
+
+    for (let i = 0; i < segments.length; i++) {
+      const trimmed = segments[i].trim();
+      if (trimmed.length === 0) continue;
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch {
+        continue;
+      }
+      const sanitized = sanitizeRow(parsed);
+      if (!sanitized) continue;
+
+      // Cost of appending this row to the current batch: the row's JSON
+      // length plus 1 byte for the leading comma when there's already at
+      // least one row. (No comma when the batch is empty — the row sits
+      // alone inside the events array.)
+      const rowJsonBytes = Buffer.byteLength(JSON.stringify(sanitized), "utf-8");
+      const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
+
+      if (batchEvents.length > 0 && batchBytes + addCost > MAX_BATCH_BYTES) {
+        await flush();
+        // After flush, batchEvents is empty → no comma needed for the first row.
+        batchBytes = ENVELOPE_BYTES + rowJsonBytes;
+      } else {
+        batchBytes += addCost;
+      }
+
+      batchEvents.push(sanitized);
+      batchSources.push({
+        filePath,
+        endOffset: lineEndOffsets[i],
+        mtime: currentMtime,
+      });
+    }
+  }
+
+  await flush();
+
+  // 4. Persist cursor: loaded < rotation_resets < newly_committed.
+  const finalFiles: Record<string, CursorEntry> = { ...loadedFiles };
+  for (const [fp, entry] of Object.entries(rotationResets)) finalFiles[fp] = entry;
+  for (const [fp, entry] of Object.entries(newlyCommitted)) finalFiles[fp] = entry;
+
+  await saveCursor(cursorPath, { version: "1", files: finalFiles });
+
+  return {
+    enabled: true,
+    optInSource,
+    filesScanned: files.length,
+    eventsSent,
+    batchesSent,
+  };
+}

package/src/types.ts

@@ -91,6 +91,24 @@ export interface VaultCredentials {
   sessionToken: string;
 }
 
+/**
+ * Caller identification stamped on every request to hq-cloud-api so the server
+ * can attribute traffic and gate on minimum versions.
+ */
+export interface ClientInfo {
+  /** Package name, e.g. "@indigoai-us/hq-cli" */
+  name: string;
+  /** Package version, e.g. "5.15.0" */
+  version: string;
+  /**
+   * `hqVersion` from `core/core.yaml` — set when the caller is running inside
+   * an hq-core checkout. Lets the server see scaffold-generation skew.
+   */
+  hqCoreVersion?: string;
+  /** Arbitrary extra key/value pairs forwarded as `x-hq-client-<key>` headers. */
+  extra?: Record<string, string>;
+}
+
 /**
  * Configuration for connecting to the vault-service API.
  */
@@ -110,6 +128,11 @@ export interface VaultServiceConfig {
   authToken: string | (() => string | Promise<string>);
   /** AWS region for S3 client (defaults to entity region or us-east-1) */
   region?: string;
+  /**
+   * Identifies the calling package + version on every outbound request.
+   * Optional for back-compat, but all first-party clients should pass it.
+   */
+  clientInfo?: ClientInfo;
 }
 
 // ── Conflict index (consumed by /resolve-conflicts) ─────────────────────────

package/src/vault-client.ts

@@ -6,7 +6,8 @@
  * share one client instead of each rolling its own HTTP layer.
  */
 
-import type { VaultServiceConfig } from "./types.js";
+import type { ClientInfo, VaultServiceConfig } from "./types.js";
+import { buildClientHeaders } from "./client-info.js";
 
 // ---------------------------------------------------------------------------
 // Error classes
@@ -150,6 +151,19 @@ export interface CreateEntityResult {
   entity: EntityInfo;
 }
 
+// -- Entity STS vending (sources/signals entity resolver) ------------------
+
+export interface EntityStsResult {
+  credentials: {
+    accessKeyId: string;
+    secretAccessKey: string;
+    sessionToken: string;
+  };
+  expiresAt: string;
+  bucketName: string;
+  region: string;
+}
+
 // -- STS child vending (VLT-8) --------------------------------------------
 
 export type TaskAction = "read" | "write";
@@ -191,6 +205,35 @@ export interface VendChildResult {
   expiresAt: string;
 }
 
+// ---------------------------------------------------------------------------
+// Usage telemetry (hq-pro `/v1/usage` + `/v1/usage/opt-in`)
+// ---------------------------------------------------------------------------
+
+export interface TelemetryOptInResponse {
+  enabled: boolean;
+  updatedAt: string | null;
+}
+
+export interface UsageBatch {
+  machineId: string;
+  installerVersion: string;
+  /**
+   * Sanitized event rows. Each row is a plain object containing only the
+   * fields in the server's KEEP allowlist (sessionId, timestamp, uuid, cwd,
+   * gitBranch, userType, model, inputTokens, outputTokens,
+   * cacheCreationInputTokens, cacheReadInputTokens). Any extra field is
+   * rejected by hq-pro with `unexpected-event-field`, so the sanitizer in
+   * `./telemetry.ts` is the only thing allowed to produce these.
+   */
+  events: Array<Record<string, unknown>>;
+}
+
+export interface UsageIngestResult {
+  ok: boolean;
+  written: number;
+  skipped: Array<{ index: number; code: string; error: string }>;
+}
+
 // ---------------------------------------------------------------------------
 // Retry config
 // ---------------------------------------------------------------------------
@@ -213,6 +256,7 @@ async function sleep(ms: number): Promise<void> {
 export class VaultClient {
   private readonly apiUrl: string;
   private readonly getAuthToken: () => Promise<string>;
+  private readonly clientInfo: ClientInfo | undefined;
 
   constructor(config: VaultServiceConfig) {
     this.apiUrl = config.apiUrl.replace(/\/+$/, "");
@@ -227,6 +271,7 @@ export class VaultClient {
       typeof tok === "function"
         ? async () => tok()
         : async () => tok;
+    this.clientInfo = config.clientInfo;
   }
 
   // -- Membership operations ------------------------------------------------
@@ -461,8 +506,52 @@ export class VaultClient {
     }> => {
       return this.post("/sts/vend-self", input);
     },
+
+    /**
+     * Vend STS-scoped credentials for an entity's S3 bucket.
+     * Backed by `POST /entities/{uid}/sts` — simpler than vend-child,
+     * returns full-bucket read credentials for the entity.
+     */
+    vendForEntity: async (entityUid: string): Promise<EntityStsResult> => {
+      const data = await this.post<EntityStsResult>(
+        `/entities/${encodeURIComponent(entityUid)}/sts`,
+        {},
+      );
+      return data;
+    },
   };
 
+  // -- Usage telemetry ------------------------------------------------------
+  //
+  // The server resolves `personUid` from the JWT (`extractCallerSub` →
+  // `resolveCallerPersonUid` in hq-pro `src/vault-service/handlers/_shared.ts`)
+  // and explicitly rejects any request body that carries a top-level
+  // `personUid`. So clients only send `{ machineId, installerVersion, events }`
+  // — tenant isolation is preserved no matter how this client is wrapped.
+
+  /**
+   * `GET /v1/usage/opt-in` — read whether the authenticated caller has opted
+   * in to per-event usage telemetry. Defaults to `false` server-side when the
+   * person row carries no `telemetryOptIn` field. Callers should treat any
+   * thrown error as "unknown — fall back to the local gate" rather than
+   * either yes or no; see `./telemetry.ts::collectAndSendTelemetry`.
+   */
+  async getTelemetryOptIn(): Promise<TelemetryOptInResponse> {
+    return this.get<TelemetryOptInResponse>("/v1/usage/opt-in");
+  }
+
+  /**
+   * `POST /v1/usage` — upload a batch of sanitized telemetry events.
+   *
+   * `personUid` MUST NOT appear in the batch — server-side resolution from
+   * the JWT is the only path. The server caps the body at 256 KiB and the
+   * event list at 100 rows; the collector in `./telemetry.ts` enforces a
+   * 1 MiB pre-flush cap which is the binding limit in practice.
+   */
+  async postUsage(batch: UsageBatch): Promise<UsageIngestResult> {
+    return this.post<UsageIngestResult>("/v1/usage", batch);
+  }
+
   // -- HTTP primitives with retry -------------------------------------------
 
   private async get<T>(path: string): Promise<T> {
@@ -485,6 +574,7 @@ export class VaultClient {
       const headers: Record<string, string> = {
         Authorization: `Bearer ${await this.getAuthToken()}`,
         Accept: "application/json",
+        ...buildClientHeaders(this.clientInfo),
       };
 
       const init: RequestInit = { method, headers };