@traice/codex-collector 0.1.0 → 0.1.3

package/collector.mjs

@@ -1,85 +1,109 @@
 #!/usr/bin/env node
-import { createHash } from 'node:crypto';
-import { execFileSync } from 'node:child_process';
-import { createServer } from 'node:http';
-import {
-  appendFileSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  readdirSync,
-  statSync,
-  writeFileSync,
-} from 'node:fs';
-import { homedir } from 'node:os';
-import { basename, dirname, join, resolve } from 'node:path';
-import { gunzipSync } from 'node:zlib';
-
-const DEFAULT_OUT_DIR = '~/.traice/internal-spend-codex';
-const LEGACY_OUT_DIR = '.traice-internal-spend';
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { createServer } from "node:http";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import { gunzipSync } from "node:zlib";
+import { readApiKeyCredential, storeApiKey } from "./credentials.mjs";
+
+const DEFAULT_OUT_DIR = "~/.traice/internal-spend-codex";
+const LEGACY_OUT_DIR = ".traice-internal-spend";
 const DEFAULT_INSTALL_CONFIG = `${DEFAULT_OUT_DIR}/install.json`;
 const LEGACY_INSTALL_CONFIG = `${LEGACY_OUT_DIR}/install.json`;
 const DEFAULT_MAX_BATCH_SIZE = 50;
 const MAX_SENT_IDS = 20_000;
+const collectorStartedAtMs = Date.now();
+const pendingTurnLatency = new Map();
 
 const args = parseArgs(process.argv.slice(2));
-loadEnvFile(resolve('apps/web/.env'));
+const inheritedApiKey = process.env.TRAICE_API_KEY;
+loadEnvFile(resolve("apps/web/.env"));
 
-const configPath = args.get('config') ? resolveHome(args.get('config')) : findDefaultInstallConfigPath();
-const outDir = resolveHome(args.get('out') ?? dirname(configPath));
+const configPath = args.get("config") ? resolveHome(args.get("config")) : findDefaultInstallConfigPath();
+const outDir = resolveHome(args.get("out") ?? dirname(configPath));
 const installConfig = readJsonIfExists(configPath) ?? {};
+const apiKey = await resolveCollectorApiKey();
 const config = {
-  serverUrl: normalizeUrl(args.get('server-url') ?? installConfig.serverUrl ?? process.env.TRAICE_API_URL ?? 'http://127.0.0.1:3003'),
-  apiKey: readApiKey() ?? installConfig.apiKey ?? process.env.TRAICE_API_KEY,
-  workspaceSlug: args.get('workspace-slug') ?? installConfig.workspaceSlug ?? 'internal-spend-poc',
-  sourceKey: args.get('source-key') ?? installConfig.sourceKey ?? 'codex-local',
-  sourceName: args.get('source-name') ?? installConfig.sourceName ?? 'Codex local collector',
-  sourceKind: args.get('source-kind') ?? installConfig.sourceKind ?? 'codex_otel',
-  tool: args.get('tool') ?? installConfig.tool ?? 'codex',
-  category: args.get('category') ?? installConfig.category ?? 'coding_agent',
-  employeeEmail: args.get('employee-email') ?? installConfig.employeeEmail ?? process.env.TRAICE_EMPLOYEE_EMAIL,
-  employeeName: args.get('employee-name') ?? installConfig.employeeName ?? process.env.TRAICE_EMPLOYEE_NAME,
-  employeeExternalId: args.get('employee-external-id') ?? installConfig.employeeExternalId,
-  teamName: args.get('team-name') ?? installConfig.teamName ?? process.env.TRAICE_TEAM_NAME,
-  teamExternalId: args.get('team-external-id') ?? installConfig.teamExternalId,
-  sourcePrincipal: args.get('source-principal') ?? installConfig.sourcePrincipal,
+  serverUrl: normalizeUrl(
+    args.get("server-url") ?? installConfig.serverUrl ?? process.env.TRAICE_API_URL ?? "http://127.0.0.1:3003",
+  ),
+  apiKey,
+  workspaceSlug: args.get("workspace-slug") ?? installConfig.workspaceSlug ?? "internal-spend-poc",
+  sourceKey: args.get("source-key") ?? installConfig.sourceKey ?? "codex-local",
+  sourceName: args.get("source-name") ?? installConfig.sourceName ?? "Codex local collector",
+  sourceKind: args.get("source-kind") ?? installConfig.sourceKind ?? "codex_otel",
+  tool: args.get("tool") ?? installConfig.tool ?? "codex",
+  category: args.get("category") ?? installConfig.category ?? "coding_agent",
+  employeeEmail: args.get("employee-email") ?? installConfig.employeeEmail ?? process.env.TRAICE_EMPLOYEE_EMAIL,
+  employeeName: args.get("employee-name") ?? installConfig.employeeName ?? process.env.TRAICE_EMPLOYEE_NAME,
+  employeeExternalId: args.get("employee-external-id") ?? installConfig.employeeExternalId,
+  teamName: args.get("team-name") ?? installConfig.teamName ?? process.env.TRAICE_TEAM_NAME,
+  teamExternalId: args.get("team-external-id") ?? installConfig.teamExternalId,
+  sourcePrincipal: args.get("source-principal") ?? installConfig.sourcePrincipal,
   seatMonthlyUsd: parseOptionalMoney(
-    args.get('seat-monthly-usd') ?? installConfig.seatMonthlyUsd ?? process.env.TRAICE_SEAT_MONTHLY_USD,
+    args.get("seat-monthly-usd") ?? installConfig.seatMonthlyUsd ?? process.env.TRAICE_SEAT_MONTHLY_USD,
+  ),
+  codexHome: resolveHome(args.get("codex-home") ?? installConfig.codexHome ?? process.env.CODEX_HOME ?? "~/.codex"),
+  listenHost: args.get("listen-host") ?? installConfig.listenHost ?? "127.0.0.1",
+  listenPort: parsePort(args.get("listen-port") ?? installConfig.listenPort ?? "4318"),
+  pollIntervalMs: parsePositiveInt(args.get("poll-interval-ms") ?? installConfig.pollIntervalMs ?? "3000"),
+  backfillHours: parseBackfillHours(
+    args.get("backfill-hours") ??
+      (Number(installConfig.version ?? 1) >= 2 ? installConfig.backfillHours : undefined) ??
+      "all",
   ),
-  codexHome: resolveHome(args.get('codex-home') ?? installConfig.codexHome ?? process.env.CODEX_HOME ?? '~/.codex'),
-  listenHost: args.get('listen-host') ?? installConfig.listenHost ?? '127.0.0.1',
-  listenPort: parsePort(args.get('listen-port') ?? installConfig.listenPort ?? '4318'),
-  pollIntervalMs: parsePositiveInt(args.get('poll-interval-ms') ?? installConfig.pollIntervalMs ?? '3000'),
-  backfillHours: Number(args.get('backfill-hours') ?? installConfig.backfillHours ?? 6),
-  includeZeroOutput: boolArg(args.get('include-zero-output') ?? installConfig.includeZeroOutput, false),
-  maxBatchSize: parsePositiveInt(args.get('max-batch-size') ?? installConfig.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE),
+  includeZeroOutput: boolArg(args.get("include-zero-output") ?? installConfig.includeZeroOutput, false),
+  maxBatchSize: parsePositiveInt(args.get("max-batch-size") ?? installConfig.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE),
 };
 
+async function resolveCollectorApiKey() {
+  const explicitApiKey = readApiKey() ?? inheritedApiKey;
+  if (explicitApiKey) return explicitApiKey;
+  if (installConfig.apiKeyCredential) return readApiKeyCredential(installConfig.apiKeyCredential);
+  if (installConfig.apiKey) {
+    const stored = await storeApiKey(configPath, installConfig.apiKey, "auto");
+    const migrated = { ...installConfig, apiKeyCredential: stored.credential };
+    delete migrated.apiKey;
+    writeFileSync(configPath, `${JSON.stringify(migrated, null, 2)}\n`, { mode: 0o600 });
+    if (stored.warning) console.warn(`[codex-otel] ${stored.warning}`);
+    return readApiKeyCredential(stored.credential);
+  }
+  return undefined;
+}
+
 if (!config.apiKey) {
   console.error(
     [
-      'Missing collector API key.',
-      'Run: npx @traice/codex-collector@latest install',
-      'Provide the key with TRAICE_API_KEY or --api-key-stdin.',
-    ].join('\n'),
+      "Missing collector API key.",
+      "Run: npx @traice/codex-collector@latest install",
+      "Provide the key with TRAICE_API_KEY or --api-key-stdin.",
+    ].join("\n"),
   );
   process.exit(2);
 }
 
 mkdirSync(outDir, { recursive: true });
-const rawFile = resolve(outDir, 'requests.ndjson');
-const eventsFile = resolve(outDir, 'events.ndjson');
-const statePath = resolve(outDir, 'collector-state.json');
+const rawFile = resolve(outDir, "requests.ndjson");
+const eventsFile = resolve(outDir, "events.ndjson");
+const statePath = resolve(outDir, "collector-state.json");
 const state = loadState(statePath);
+const destinationKey = hashObject({ serverUrl: config.serverUrl, apiKeyPrefix: config.apiKey.slice(0, 12) });
+if (state.destinationKey !== destinationKey) {
+  state.files = {};
+  state.sent = {};
+  state.destinationKey = destinationKey;
+}
 
-const shouldRunOnce = args.get('once') === 'true';
-const sessionTailEnabled = args.get('session-tail') !== 'false';
-const otelListenerEnabled = args.get('otel-listener') !== 'false' && !shouldRunOnce;
+const shouldRunOnce = args.get("once") === "true";
+const sessionTailEnabled = args.get("session-tail") !== "false";
+const otelListenerEnabled = args.get("otel-listener") !== "false" && !shouldRunOnce;
 
 if (shouldRunOnce) {
   const result = sessionTailEnabled ? await scanCodexSessions() : { candidates: 0, sent: 0, files: 0 };
   saveState(statePath, state);
-  console.log(JSON.stringify({ ok: true, mode: 'once', ...result }, null, 2));
+  console.log(JSON.stringify({ ok: true, mode: "once", ...result }, null, 2));
   process.exit(0);
 }
 
@@ -97,39 +121,39 @@ if (sessionTailEnabled) {
 if (otelListenerEnabled) {
   startOtelServer();
 } else if (!sessionTailEnabled) {
-  console.error('Nothing to run: both OTel listener and session tail are disabled.');
+  console.error("Nothing to run: both OTel listener and session tail are disabled.");
   process.exit(2);
 }
 
-process.on('SIGINT', () => {
+process.on("SIGINT", () => {
   saveState(statePath, state);
   process.exit(0);
 });
-process.on('SIGTERM', () => {
+process.on("SIGTERM", () => {
   saveState(statePath, state);
   process.exit(0);
 });
 
 function startOtelServer() {
   const server = createServer(async (req, res) => {
-    if (req.method === 'GET') {
-      res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8' });
+    if (req.method === "GET") {
+      res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
       res.end(
         [
-          'trAIce Internal Spend Codex collector is running.',
-          '',
+          "trAIce Internal Spend Codex collector is running.",
+          "",
           `OTLP endpoint: http://${config.listenHost}:${config.listenPort}/v1/logs`,
           `Internal usage ingest: ${config.serverUrl}/api/v1/internal-usage`,
           `Internal Spend view: ${config.serverUrl}/dashboard?tab=internal`,
-          '',
-          'New Codex agents send OTel here when user-level [otel] config is active.',
-          'Already-open Codex agents are covered by tailing ~/.codex/sessions token_count events.',
-        ].join('\n'),
+          "",
+          "New Codex agents send OTel here when user-level [otel] config is active.",
+          "Already-open Codex agents are covered by tailing ~/.codex/sessions token_count events.",
+        ].join("\n"),
       );
       return;
     }
 
-    if (req.method !== 'POST') {
+    if (req.method !== "POST") {
       res.writeHead(404).end();
       return;
     }
@@ -143,25 +167,25 @@ function startOtelServer() {
       return;
     }
 
-    const contentType = req.headers['content-type'] ?? '';
+    const contentType = req.headers["content-type"] ?? "";
     const rawEnvelope = {
       received_at: receivedAt,
       method: req.method,
       url: req.url,
       content_type: contentType,
-      content_encoding: req.headers['content-encoding'],
-      body_base64: contentType.includes('json') ? undefined : body.toString('base64'),
+      content_encoding: req.headers["content-encoding"],
+      body_base64: contentType.includes("json") ? undefined : body.toString("base64"),
     };
 
     let normalized = [];
-    if (contentType.includes('json')) {
+    if (contentType.includes("json")) {
       try {
-        const payload = JSON.parse(body.toString('utf8'));
+        const payload = JSON.parse(body.toString("utf8"));
         rawEnvelope.body = redactSensitive(payload);
         normalized = normalizeOtlpLogs(payload);
       } catch (error) {
         rawEnvelope.parse_error = error.message;
-        rawEnvelope.body_text = redactSensitive(body.toString('utf8'));
+        rawEnvelope.body_text = redactSensitive(body.toString("utf8"));
       }
     }
 
@@ -175,7 +199,7 @@ function startOtelServer() {
         console.log(
           JSON.stringify({
             received_at: receivedAt,
-            source: 'otel',
+            source: "otel",
             records: normalized.length,
             ingest_candidates: ingestEvents.length,
             sent,
@@ -186,12 +210,14 @@ function startOtelServer() {
       console.error(`[codex-otel] internal usage ingest failed: ${error.message}`);
     }
 
-    res.writeHead(200, { 'content-type': 'application/json' });
-    res.end('{}');
+    res.writeHead(200, { "content-type": "application/json" });
+    res.end("{}");
   });
 
   server.listen(config.listenPort, config.listenHost, () => {
-    console.log(`trAIce Internal Spend Codex collector listening on http://${config.listenHost}:${config.listenPort}/v1/logs`);
+    console.log(
+      `trAIce Internal Spend Codex collector listening on http://${config.listenHost}:${config.listenPort}/v1/logs`,
+    );
     console.log(`Forwarding Codex internal usage to ${config.serverUrl}/api/v1/internal-usage`);
     if (config.seatMonthlyUsd > 0) {
       console.log(`Recording seat subscription commitment: $${config.seatMonthlyUsd}/mo`);
@@ -201,10 +227,13 @@ function startOtelServer() {
 }
 
 async function scanCodexSessions() {
-  const sessionsDir = resolve(config.codexHome, 'sessions');
+  const sessionsDir = resolve(config.codexHome, "sessions");
   if (!existsSync(sessionsDir)) return { candidates: 0, sent: 0, files: 0 };
 
-  const cutoffMs = Date.now() - Math.max(0, config.backfillHours) * 60 * 60 * 1000;
+  const cutoffMs =
+    config.backfillHours === Infinity
+      ? Number.NEGATIVE_INFINITY
+      : collectorStartedAtMs - config.backfillHours * 60 * 60 * 1000;
   const threadMetadata = loadThreadMetadata(config.codexHome);
   const files = findSessionFiles(sessionsDir)
     .map((file) => ({ file, stat: statSync(file) }))
@@ -237,7 +266,7 @@ async function scanCodexSessions() {
   }
 
   if (sent > 0) {
-    console.log(JSON.stringify({ source: 'codex_session_jsonl', candidates, sent, files: touchedFiles }));
+    console.log(JSON.stringify({ source: "codex_session_jsonl", candidates, sent, files: touchedFiles }));
   }
 
   return { candidates, sent, files: touchedFiles };
@@ -255,20 +284,19 @@ function readNewSessionTokenEvents(file, stat, cutoffMs, threadMetadata) {
 
   const buffer = readFileSync(file);
   const chunk = buffer.subarray(startOffset);
-  const text = chunk.toString('utf8');
+  const text = chunk.toString("utf8");
   if (!text) return { events: [], nextOffset: startOffset };
 
-  const completeChars = text.endsWith('\n') ? text.length : text.lastIndexOf('\n') + 1;
+  const completeChars = text.endsWith("\n") ? text.length : text.lastIndexOf("\n") + 1;
   if (completeChars <= 0) return { events: [], nextOffset: startOffset };
 
   const completeText = text.slice(0, completeChars);
   const nextOffset = startOffset + Buffer.byteLength(completeText);
   const threadId = extractThreadId(file);
   const metadata = (threadId && threadMetadata.get(threadId)) || threadMetadata.get(file) || null;
-  const isInitialBackfill = !prior;
   const events = [];
 
-  for (const line of completeText.split('\n')) {
+  for (const line of completeText.split("\n")) {
     if (!line) continue;
     let entry;
     try {
@@ -276,10 +304,12 @@ function readNewSessionTokenEvents(file, stat, cutoffMs, threadMetadata) {
     } catch {
       continue;
     }
-    if (entry?.type !== 'event_msg' || entry?.payload?.type !== 'token_count') continue;
+    if (entry?.type !== "event_msg" || entry?.payload?.type !== "token_count") continue;
 
     const tsMs = new Date(entry.timestamp).getTime();
-    if (isInitialBackfill && Number.isFinite(tsMs) && tsMs < cutoffMs) continue;
+    // A finite startup boundary intentionally limits history. The default is
+    // full history; destination-scoped state and source event IDs dedupe restarts.
+    if (Number.isFinite(tsMs) && tsMs < cutoffMs) continue;
 
     const event = sessionTokenCountToIngestEvent({ entry, file, threadId, metadata });
     if (event) events.push(event);
@@ -290,21 +320,21 @@ function readNewSessionTokenEvents(file, stat, cutoffMs, threadMetadata) {
 
 function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
   const usage = entry?.payload?.info?.last_token_usage;
-  if (!usage || typeof usage !== 'object') return null;
+  if (!usage || typeof usage !== "object") return null;
 
-  const inputTokens = getNumber(usage, 'input_tokens');
-  const cachedInputTokens = getNumber(usage, 'cached_input_tokens');
-  const outputTokens = getNumber(usage, 'output_tokens');
-  const reasoningOutputTokens = getNumber(usage, 'reasoning_output_tokens');
+  const inputTokens = getNumber(usage, "input_tokens");
+  const cachedInputTokens = getNumber(usage, "cached_input_tokens");
+  const outputTokens = getNumber(usage, "output_tokens");
+  const reasoningOutputTokens = getNumber(usage, "reasoning_output_tokens");
   const nonCachedInputTokens = Math.max(inputTokens - cachedInputTokens, 0);
   const billableTokenEstimate = nonCachedInputTokens + outputTokens;
   if (!config.includeZeroOutput && outputTokens === 0 && reasoningOutputTokens === 0) return null;
 
   const timestamp = normalizeTimestamp(entry.timestamp);
-  const model = metadata?.model ?? 'unknown';
+  const model = metadata?.model ?? "unknown";
   const eventId = hashObject({
-    source: 'codex_session_jsonl',
-    file,
+    sourceAgent: "codex",
+    conversationId: threadId ?? basename(file, ".jsonl").slice(-36),
     timestamp,
     inputTokens,
     cachedInputTokens,
@@ -319,7 +349,7 @@ function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
     sourceKind: config.sourceKind,
     tool: config.tool,
     category: config.category,
-    provider: metadata?.modelProvider ?? 'openai',
+    provider: metadata?.modelProvider ?? "openai",
     model,
     employeeEmail: config.employeeEmail,
     employeeName: config.employeeName,
@@ -327,8 +357,8 @@ function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
     teamName: config.teamName,
     teamExternalId: config.teamExternalId,
     sourcePrincipal: config.sourcePrincipal,
-    sourcePrincipalType: 'device_user',
-    runId: threadId ?? basename(file, '.jsonl').slice(-36),
+    sourcePrincipalType: "device_user",
+    runId: threadId ?? basename(file, ".jsonl").slice(-36),
     stepId: eventId.slice(0, 32),
     sourceEventId: eventId,
     inputTokens,
@@ -336,13 +366,13 @@ function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
     outputTokens,
     totalTokens: inputTokens + outputTokens,
     costUsd: 0,
-    costBasis: config.seatMonthlyUsd > 0 ? 'seat_subscription' : 'usage_only',
-    status: 'ok',
+    costBasis: config.seatMonthlyUsd > 0 ? "seat_subscription" : "usage_only",
+    status: "ok",
     metadata: compactMetadata({
-      usageDomain: 'developer_agent',
-      sourceAgent: 'codex',
-      sourceTransport: 'codex_session_jsonl',
-      sourceEventName: 'codex.session_token_count',
+      usageDomain: "developer_agent",
+      sourceAgent: "codex",
+      sourceTransport: "codex_session_jsonl",
+      sourceEventName: "codex.session_token_count",
       workspaceSlug: config.workspaceSlug,
       tokenAccounting: {
         rawInputTokens: inputTokens,
@@ -351,20 +381,19 @@ function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
         outputTokens,
         reasoningOutputTokens,
         billableTokenEstimate,
-        rawTotalTokens: getNumber(usage, 'total_tokens'),
+        rawTotalTokens: getNumber(usage, "total_tokens"),
         cumulative: entry?.payload?.info?.total_token_usage ?? null,
       },
       subscription: subscriptionMetadata(),
       codex: {
         conversationId: threadId,
         sessionFile: compactPath(file),
-        eventKind: 'token_count',
-        modelContextWindow: getNumber(entry?.payload?.info, 'model_context_window'),
+        eventKind: "token_count",
+        modelContextWindow: getNumber(entry?.payload?.info, "model_context_window"),
         rateLimits: rateLimitSummary(entry?.payload?.rate_limits),
       },
       thread: metadata
         ? {
-            title: metadata.title,
             cwd: metadata.cwd,
             source: metadata.source,
             sandboxPolicy: metadata.sandboxPolicy,
@@ -379,26 +408,46 @@ function sessionTokenCountToIngestEvent({ entry, file, threadId, metadata }) {
 
 function otelRecordToIngestEvent(record) {
   const attributes = record.attributes ?? {};
-  if (record.event_name !== 'codex.sse_event') return [];
-  if (getString(attributes, 'event.kind') !== 'response.completed') return [];
-  if (getString(attributes, 'error.message')) return [];
-
-  const inputTokens = getNumber(attributes, 'input_token_count');
-  const cachedInputTokens = getNumber(attributes, 'cached_token_count');
-  const outputTokens = getNumber(attributes, 'output_token_count');
-  const reasoningOutputTokens = getNumber(attributes, 'reasoning_token_count');
+  if (record.event_name === "codex.turn_ttft") {
+    const conversationId = getString(attributes, "conversation.id");
+    const durationMs = getNullableNumber(attributes, "duration_ms");
+    if (conversationId && durationMs != null) {
+      pendingTurnLatency.set(conversationId, {
+        durationMs,
+        recordedAtMs: new Date(getString(attributes, "event.timestamp") ?? record.received_at).getTime(),
+      });
+      if (pendingTurnLatency.size > 1_000) pendingTurnLatency.delete(pendingTurnLatency.keys().next().value);
+    }
+    return [];
+  }
+  if (record.event_name !== "codex.sse_event") return [];
+  if (getString(attributes, "event.kind") !== "response.completed") return [];
+  if (getString(attributes, "error.message")) return [];
+
+  const inputTokens = getNumber(attributes, "input_token_count");
+  const cachedInputTokens = getNumber(attributes, "cached_token_count");
+  const outputTokens = getNumber(attributes, "output_token_count");
+  const reasoningOutputTokens = getNumber(attributes, "reasoning_token_count");
   const nonCachedInputTokens = Math.max(inputTokens - cachedInputTokens, 0);
   const zeroOutputRecord = outputTokens === 0 && reasoningOutputTokens === 0;
   if (!config.includeZeroOutput && zeroOutputRecord) return [];
 
-  const timestamp = normalizeTimestamp(getString(attributes, 'event.timestamp') ?? record.received_at);
-  const conversationId = getString(attributes, 'conversation.id') ?? 'unknown';
-  const model = getString(attributes, 'model') ?? 'unknown';
+  const timestamp = normalizeTimestamp(getString(attributes, "event.timestamp") ?? record.received_at);
+  const conversationId = getString(attributes, "conversation.id") ?? "unknown";
+  const model = getString(attributes, "model") ?? "unknown";
+  const turnLatency = pendingTurnLatency.get(conversationId);
+  const responseTimestampMs = new Date(timestamp).getTime();
+  const latencyMs =
+    turnLatency &&
+    responseTimestampMs - turnLatency.recordedAtMs >= 0 &&
+    responseTimestampMs - turnLatency.recordedAtMs < 300_000
+      ? turnLatency.durationMs
+      : getNullableNumber(attributes, "duration_ms");
+  if (turnLatency) pendingTurnLatency.delete(conversationId);
   const eventId = hashObject({
-    source: 'otel',
+    sourceAgent: "codex",
     conversationId,
     timestamp,
-    model,
     inputTokens,
     cachedInputTokens,
     outputTokens,
@@ -413,7 +462,7 @@ function otelRecordToIngestEvent(record) {
       sourceKind: config.sourceKind,
       tool: config.tool,
       category: config.category,
-      provider: 'openai',
+      provider: "openai",
       model,
       employeeEmail: config.employeeEmail,
       employeeName: config.employeeName,
@@ -421,7 +470,7 @@ function otelRecordToIngestEvent(record) {
       teamName: config.teamName,
       teamExternalId: config.teamExternalId,
       sourcePrincipal: config.sourcePrincipal,
-      sourcePrincipalType: 'device_user',
+      sourcePrincipalType: "device_user",
       runId: conversationId,
       stepId: eventId.slice(0, 32),
       sourceEventId: eventId,
@@ -430,14 +479,15 @@ function otelRecordToIngestEvent(record) {
       outputTokens,
       totalTokens: inputTokens + outputTokens,
       costUsd: 0,
-      costBasis: config.seatMonthlyUsd > 0 ? 'seat_subscription' : 'usage_only',
-      latencyMs: getNullableNumber(attributes, 'duration_ms') ?? undefined,
-      status: 'ok',
+      costBasis: config.seatMonthlyUsd > 0 ? "seat_subscription" : "usage_only",
+      latencyMs: latencyMs ?? undefined,
+      status: "ok",
       metadata: compactMetadata({
-        usageDomain: 'developer_agent',
-        sourceAgent: 'codex',
-        sourceTransport: 'otel',
+        usageDomain: "developer_agent",
+        sourceAgent: "codex",
+        sourceTransport: "otel",
         sourceEventName: record.event_name,
+        latencyKind: latencyMs != null ? (turnLatency ? "time_to_first_token" : "duration") : null,
         workspaceSlug: config.workspaceSlug,
         tokenAccounting: {
           rawInputTokens: inputTokens,
@@ -450,16 +500,16 @@ function otelRecordToIngestEvent(record) {
         },
         subscription: subscriptionMetadata(),
         service: {
-          name: getString(record.resource, 'service.name'),
-          version: getString(record.resource, 'service.version'),
-          environment: getString(record.resource, 'env'),
+          name: getString(record.resource, "service.name"),
+          version: getString(record.resource, "service.version"),
+          environment: getString(record.resource, "env"),
         },
         codex: {
           conversationId,
-          eventKind: getString(attributes, 'event.kind'),
-          authMode: getString(attributes, 'auth_mode'),
-          sandboxPolicy: getString(attributes, 'sandbox_policy'),
-          approvalPolicy: getString(attributes, 'approval_policy'),
+          eventKind: getString(attributes, "event.kind"),
+          authMode: getString(attributes, "auth_mode"),
+          sandboxPolicy: getString(attributes, "sandbox_policy"),
+          approvalPolicy: getString(attributes, "approval_policy"),
         },
       }),
     },
@@ -477,15 +527,15 @@ async function forwardEvents(events) {
   for (let i = 0; i < pending.length; i += config.maxBatchSize) {
     const batch = pending.slice(i, i + config.maxBatchSize);
     const response = await fetch(`${config.serverUrl}/api/v1/internal-usage`, {
-      method: 'POST',
+      method: "POST",
       headers: {
         authorization: `Bearer ${config.apiKey}`,
-        'content-type': 'application/json',
+        "content-type": "application/json",
       },
       body: JSON.stringify({ events: batch }),
     });
     if (!response.ok) {
-      const text = await response.text().catch(() => '');
+      const text = await response.text().catch(() => "");
       throw new Error(`POST /api/v1/internal-usage returned ${response.status}: ${text.slice(0, 500)}`);
     }
 
@@ -546,21 +596,21 @@ function appendNormalizedEvents(records) {
 
 function loadThreadMetadata(codexHome) {
   const byId = new Map();
-  const dbPath = resolve(codexHome, 'state_5.sqlite');
+  const dbPath = resolve(codexHome, "state_5.sqlite");
   if (!existsSync(dbPath)) return byId;
 
   try {
     const query = [
-      'select id, rollout_path, source, model_provider, cwd, title, sandbox_policy,',
-      'approval_mode, tokens_used, model, reasoning_effort, created_at, updated_at',
-      'from threads',
-    ].join(' ');
-    const output = execFileSync('sqlite3', ['-json', dbPath, query], {
-      encoding: 'utf8',
+      "select id, rollout_path, source, model_provider, cwd, sandbox_policy,",
+      "approval_mode, tokens_used, model, reasoning_effort, created_at, updated_at",
+      "from threads",
+    ].join(" ");
+    const output = execFileSync("sqlite3", ["-json", dbPath, query], {
+      encoding: "utf8",
       timeout: 3000,
       maxBuffer: 8 * 1024 * 1024,
     });
-    const rows = JSON.parse(output || '[]');
+    const rows = JSON.parse(output || "[]");
     for (const row of rows) {
       const metadata = {
         id: row.id,
@@ -568,7 +618,6 @@ function loadThreadMetadata(codexHome) {
         source: row.source,
         modelProvider: row.model_provider,
         cwd: row.cwd,
-        title: row.title,
         sandboxPolicy: row.sandbox_policy,
         approvalMode: row.approval_mode,
         tokensUsed: row.tokens_used,
@@ -594,7 +643,7 @@ function findSessionFiles(dir) {
       const fullPath = join(current, entry.name);
       if (entry.isDirectory()) {
         visit(fullPath);
-      } else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
+      } else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
         files.push(fullPath);
       }
     }
@@ -606,10 +655,10 @@ function findSessionFiles(dir) {
 function readRequestBody(req) {
   return new Promise((resolveBody, rejectBody) => {
     const chunks = [];
-    req.on('data', (chunk) => chunks.push(chunk));
-    req.on('end', () => {
+    req.on("data", (chunk) => chunks.push(chunk));
+    req.on("end", () => {
       const body = Buffer.concat(chunks);
-      if (req.headers['content-encoding'] === 'gzip') {
+      if (req.headers["content-encoding"] === "gzip") {
         try {
           resolveBody(gunzipSync(body));
         } catch (error) {
@@ -619,19 +668,19 @@ function readRequestBody(req) {
       }
       resolveBody(body);
     });
-    req.on('error', rejectBody);
+    req.on("error", rejectBody);
   });
 }
 
 function otlpValueToJs(value) {
-  if (!value || typeof value !== 'object') return value;
-  if ('stringValue' in value) return value.stringValue;
-  if ('intValue' in value) return Number(value.intValue);
-  if ('doubleValue' in value) return Number(value.doubleValue);
-  if ('boolValue' in value) return Boolean(value.boolValue);
-  if ('bytesValue' in value) return value.bytesValue;
-  if ('arrayValue' in value) return (value.arrayValue?.values ?? []).map(otlpValueToJs);
-  if ('kvlistValue' in value) return keyValuesToObject(value.kvlistValue?.values ?? []);
+  if (!value || typeof value !== "object") return value;
+  if ("stringValue" in value) return value.stringValue;
+  if ("intValue" in value) return Number(value.intValue);
+  if ("doubleValue" in value) return Number(value.doubleValue);
+  if ("boolValue" in value) return Boolean(value.boolValue);
+  if ("bytesValue" in value) return value.bytesValue;
+  if ("arrayValue" in value) return (value.arrayValue?.values ?? []).map(otlpValueToJs);
+  if ("kvlistValue" in value) return keyValuesToObject(value.kvlistValue?.values ?? []);
   return value;
 }
 
@@ -645,9 +694,9 @@ function keyValuesToObject(values) {
 }
 
 function maybeParseJsonString(value) {
-  if (typeof value !== 'string') return value;
+  if (typeof value !== "string") return value;
   const trimmed = value.trim();
-  if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) return value;
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return value;
   try {
     return JSON.parse(trimmed);
   } catch {
@@ -656,8 +705,8 @@ function maybeParseJsonString(value) {
 }
 
 function extractEventName(body, attributes) {
-  if (typeof body === 'string') return body;
-  if (body && typeof body === 'object') {
+  if (typeof body === "string") return body;
+  if (body && typeof body === "object") {
     return (
       body.event_name ??
       body.eventName ??
@@ -665,27 +714,27 @@ function extractEventName(body, attributes) {
       body.name ??
       body.type ??
       body.message ??
-      attributes['event.name'] ??
+      attributes["event.name"] ??
       attributes.event_name ??
       attributes.name ??
-      'unknown'
+      "unknown"
     );
   }
-  return attributes['event.name'] ?? attributes.event_name ?? attributes.name ?? 'unknown';
+  return attributes["event.name"] ?? attributes.event_name ?? attributes.name ?? "unknown";
 }
 
-function redactSensitive(value, path = '') {
+function redactSensitive(value, path = "") {
   if (value === null || value === undefined) return value;
   if (Array.isArray(value)) return value.map((item, index) => redactSensitive(item, `${path}.${index}`));
-  if (typeof value === 'object') {
+  if (typeof value === "object") {
     const out = {};
     for (const [key, child] of Object.entries(value)) {
       const childPath = path ? `${path}.${key}` : key;
-      out[key] = isSensitiveKey(childPath) || isSensitiveKey(key) ? '[REDACTED]' : redactSensitive(child, childPath);
+      out[key] = isSensitiveKey(childPath) || isSensitiveKey(key) ? "[REDACTED]" : redactSensitive(child, childPath);
     }
     return out;
   }
-  if (typeof value === 'string' && /\b[^@\s]+@[^@\s]+\.[^@\s]+\b/.test(value)) return '[REDACTED]';
+  if (typeof value === "string" && /\b[^@\s]+@[^@\s]+\.[^@\s]+\b/.test(value)) return "[REDACTED]";
   return value;
 }
 
@@ -703,8 +752,8 @@ function subscriptionMetadata() {
   if (config.seatMonthlyUsd <= 0) return null;
   return {
     monthlySeatUsd: config.seatMonthlyUsd,
-    allocation: 'monthly_seat_commitment',
-    note: 'Subscription commitment is shown separately from per-request usage cost.',
+    allocation: "monthly_seat_commitment",
+    note: "Subscription commitment is shown separately from per-request usage cost.",
   };
 }
 
@@ -712,7 +761,7 @@ function pruneNullish(value) {
   if (Array.isArray(value)) {
     return value.map(pruneNullish).filter((item) => item !== undefined);
   }
-  if (value && typeof value === 'object') {
+  if (value && typeof value === "object") {
     const out = {};
     for (const [key, child] of Object.entries(value)) {
       const pruned = pruneNullish(child);
@@ -720,11 +769,11 @@ function pruneNullish(value) {
     }
     return Object.keys(out).length > 0 ? out : undefined;
   }
-  return value === null || value === undefined || value === '' ? undefined : value;
+  return value === null || value === undefined || value === "" ? undefined : value;
 }
 
 function rateLimitSummary(rateLimits) {
-  if (!rateLimits || typeof rateLimits !== 'object') return null;
+  if (!rateLimits || typeof rateLimits !== "object") return null;
   return {
     limitId: rateLimits.limit_id,
     planType: rateLimits.plan_type,
@@ -747,14 +796,15 @@ function rateLimitSummary(rateLimits) {
 }
 
 function loadState(path) {
-  const fallback = { version: 1, files: {}, sent: {} };
+  const fallback = { version: 2, files: {}, sent: {}, destinationKey: undefined };
   if (!existsSync(path)) return fallback;
   try {
-    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    const parsed = JSON.parse(readFileSync(path, "utf8"));
     return {
-      version: 1,
-      files: parsed.files && typeof parsed.files === 'object' ? parsed.files : {},
-      sent: parsed.sent && typeof parsed.sent === 'object' ? parsed.sent : {},
+      version: 2,
+      files: parsed.files && typeof parsed.files === "object" ? parsed.files : {},
+      sent: parsed.sent && typeof parsed.sent === "object" ? parsed.sent : {},
+      destinationKey: typeof parsed.destinationKey === "string" ? parsed.destinationKey : undefined,
       lastSessionScanAt: parsed.lastSessionScanAt,
     };
   } catch {
@@ -781,14 +831,14 @@ function parseArgs(argv) {
   const parsed = new Map();
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
-    if (!arg.startsWith('--')) continue;
+    if (!arg.startsWith("--")) continue;
     const key = arg.slice(2);
     const next = argv[i + 1];
-    if (next && !next.startsWith('--')) {
+    if (next && !next.startsWith("--")) {
       parsed.set(key, next);
       i += 1;
     } else {
-      parsed.set(key, 'true');
+      parsed.set(key, "true");
     }
   }
   return parsed;
@@ -796,9 +846,9 @@ function parseArgs(argv) {
 
 function loadEnvFile(path) {
   if (!existsSync(path)) return;
-  for (const line of readFileSync(path, 'utf8').split('\n')) {
+  for (const line of readFileSync(path, "utf8").split("\n")) {
     const trimmed = line.trim();
-    if (!trimmed || trimmed.startsWith('#')) continue;
+    if (!trimmed || trimmed.startsWith("#")) continue;
     const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
     if (!match) continue;
     const [, key, rawValue] = match;
@@ -808,10 +858,7 @@ function loadEnvFile(path) {
 }
 
 function unquoteEnvValue(value) {
-  if (
-    (value.startsWith('"') && value.endsWith('"')) ||
-    (value.startsWith("'") && value.endsWith("'"))
-  ) {
+  if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
     return value.slice(1, -1);
   }
   return value;
@@ -820,14 +867,14 @@ function unquoteEnvValue(value) {
 function readJsonIfExists(path) {
   if (!existsSync(path)) return null;
   try {
-    return JSON.parse(readFileSync(path, 'utf8'));
+    return JSON.parse(readFileSync(path, "utf8"));
   } catch {
     return null;
   }
 }
 
 function normalizeUrl(value) {
-  return String(value).replace(/\/+$/, '');
+  return String(value).replace(/\/+$/, "");
 }
 
 function findDefaultInstallConfigPath() {
@@ -841,8 +888,8 @@ function findDefaultInstallConfigPath() {
 }
 
 function resolveHome(value) {
-  if (value === '~') return homedir();
-  if (String(value).startsWith('~/')) return resolve(homedir(), String(value).slice(2));
+  if (value === "~") return homedir();
+  if (String(value).startsWith("~/")) return resolve(homedir(), String(value).slice(2));
   return resolve(String(value));
 }
 
@@ -864,9 +911,19 @@ function parsePositiveInt(value) {
   return parsed;
 }
 
+function parseBackfillHours(value) {
+  if (["all", "full", "infinity"].includes(String(value).toLowerCase())) return Infinity;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    console.error(`Invalid backfill hours: ${value}. Use a non-negative number or "all".`);
+    process.exit(2);
+  }
+  return parsed;
+}
+
 function parseOptionalMoney(value) {
-  if (value === undefined || value === null || value === '') return 0;
-  const parsed = Number(String(value).replace(/[$,]/g, ''));
+  if (value === undefined || value === null || value === "") return 0;
+  const parsed = Number(String(value).replace(/[$,]/g, ""));
   if (!Number.isFinite(parsed) || parsed < 0) {
     console.error(`Invalid USD amount: ${value}`);
     process.exit(2);
@@ -875,19 +932,19 @@ function parseOptionalMoney(value) {
 }
 
 function boolArg(value, fallback) {
-  if (value === undefined || value === null || value === '') return fallback;
-  if (typeof value === 'boolean') return value;
-  return String(value) === 'true';
+  if (value === undefined || value === null || value === "") return fallback;
+  if (typeof value === "boolean") return value;
+  return String(value) === "true";
 }
 
 function readApiKey() {
-  if (args.get('api-key-stdin') !== 'true') return args.get('api-key');
-  return readFileSync(0, 'utf8').trim();
+  if (args.get("api-key-stdin") !== "true") return args.get("api-key");
+  return readFileSync(0, "utf8").trim();
 }
 
 function getString(source, key) {
   const value = source?.[key];
-  return typeof value === 'string' && value.length > 0 ? value : null;
+  return typeof value === "string" && value.length > 0 ? value : null;
 }
 
 function getNumber(source, key) {
@@ -896,8 +953,8 @@ function getNumber(source, key) {
 
 function getNullableNumber(source, key) {
   const value = source?.[key];
-  if (typeof value === 'number' && Number.isFinite(value)) return value;
-  if (typeof value === 'string') {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string") {
     const parsed = Number(value);
     return Number.isFinite(parsed) ? parsed : null;
   }
@@ -928,5 +985,5 @@ function compactPath(file) {
 }
 
 function hashObject(value) {
-  return createHash('sha256').update(JSON.stringify(value)).digest('hex');
+  return createHash("sha256").update(JSON.stringify(value)).digest("hex");
 }