@botbuddy/cli 1.33.11 → 1.34.1

package/src/usage/codex.mjs

@@ -0,0 +1,236 @@
+// BOT-803 — Codex rollout adapter (`~/.codex/sessions/**/*.jsonl`, or
+// `$CODEX_HOME/sessions`).
+//
+// VERIFIED SHAPE (parser_version 'codex-2'): checked on 2026-09-28 against real
+// Codex Desktop rollouts (0.117 → 0.155.0-alpha.9.2, ~/.codex/sessions). The
+// fixtures in cli/test-fixtures/usage/ are synthetic copies of that shape.
+// Every record is {"timestamp","type","ordinal"?,"payload"}:
+//   session_meta     {id, session_id, cli_version, thread_source, parent_thread_id?, cwd,
+//                     git:{commit_hash, branch, repository_url}, …}
+//                    A subagent rollout is its own file: id = the subagent thread,
+//                    session_id = the ROOT session, thread_source 'subagent'.
+//   turn_context     {model, cwd, turn_id, …}
+//   response_item    {type:'function_call'|'custom_tool_call'|'local_shell_call'|…} —
+//                    written BEFORE the usage record of the response that made it
+//   token_usage_record (≥0.151) {response_id, session_id, thread_id, turn_id,
+//                    usage:{input_tokens, cached_input_tokens, cache_write_input_tokens,
+//                    output_tokens, reasoning_output_tokens, total_tokens},
+//                    turn_token_usage, thread_token_usage} — ONE per model response,
+//                    including a compaction's own response (which has no token_count)
+//   event_msg/token_count {info:{total_token_usage, last_token_usage, model_context_window}|null,
+//                    rate_limits} — follows each usage record with the same numbers,
+//                    is re-emitted with unchanged totals, and is null-info for a
+//                    rate-limit-only update
+// Identity: a usage record keys on its provider `response_id`. Rollouts written
+// before token_usage_record existed (and the pre-upgrade head of a session
+// resumed on a newer Codex) fall back to token_count keyed on the cumulative
+// total; once a file shows a usage record, token_count is ignored (it would
+// double count). `cache_write_input_tokens` has no BOT-1019 column and was 0 in
+// every real rollout checked; it is not sent.
+// Codex classes stay provider-faithful (codex_* columns); Claude's four classes
+// are null on Codex rows. Codex `input_tokens` INCLUDES `cached_input_tokens`.
+import {
+  commitSha, finalizeEnvelope, inferTicketKey, modelLabel, normalizeRepo, safeLabel, str, tok, usageQuality, uuid, withRoute,
+} from "./envelope.mjs";
+
+export const CODEX_PROVIDER = "codex";
+export const CODEX_PARSER_VERSION = "codex-2";
+
+// Mirrors the server's PROVIDER_ID_RE (usageEnvelope.ts): resp_… ids, no '/' or '@'.
+const RESPONSE_ID_RE = /^[A-Za-z0-9_.:-]{1,200}$/;
+const responseId = (value) => (typeof value === "string" && RESPONSE_ID_RE.test(value) ? value : null);
+
+function totalOf(usage) {
+  if (!usage || typeof usage !== "object") return null;
+  const explicit = tok(usage.total_tokens);
+  if (explicit !== null) return explicit;
+  const input = tok(usage.input_tokens);
+  const output = tok(usage.output_tokens);
+  return input === null && output === null ? null : (input ?? 0) + (output ?? 0);
+}
+
+function deltaUsage(total, previous) {
+  const keys = ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens"];
+  const out = {};
+  for (const key of keys) {
+    const now = tok(total?.[key]);
+    const before = tok(previous?.[key]) ?? 0;
+    out[key] = now === null ? null : Math.max(0, now - before);
+  }
+  return out;
+}
+
+// Only derived values are kept in ctx (it is persisted in the cursor file):
+// the cwd's HMAC and the ticket key its last segment names — never the path.
+function setCwd(ctx, cwd, hmac, tenantFor, repoFor = () => null) {
+  const value = str(cwd);
+  if (!value) return;
+  const cwdHmac = hmac(value);
+  // A later turn in ANOTHER checkout: the session_meta git metadata described
+  // the old one. Drop it (the ticket then falls back to the new cwd) and
+  // re-derive the repo from the new checkout.
+  if (ctx.cwd_hmac && ctx.cwd_hmac !== cwdHmac) {
+    ctx.git_branch = null;
+    ctx.commit_sha = null;
+    ctx.repo = repoFor(value) ?? null;
+  }
+  ctx.cwd_hmac = cwdHmac;
+  ctx.cwd_ticket = inferTicketKey(null, value);
+  ctx.cwd_tenant = tenantFor(value); // a tenant slug (or null), never the path
+}
+
+const NONE = Object.freeze({ envelopes: [], refused: 0 });
+
+/**
+ * @param {object} opts
+ * @param {(value: string) => string} opts.hmac
+ * @param {string} opts.environment
+ */
+export function createCodexAdapter({ hmac, environment, tenantFor = () => null, repoFor = () => null }) {
+  function parseCodexRecord(record, { offset, sourceHmac, ctx }) {
+    if (!record || typeof record !== "object") return NONE;
+    const payload = record.payload && typeof record.payload === "object" ? record.payload : null;
+    if (!payload) return NONE;
+
+    if (record.type === "session_meta") {
+      // The harness session is the ROOT thread (`session_id`); a subagent's own
+      // thread id (`id`) only keeps its legacy token_count keys apart.
+      const root = uuid(payload.session_id) ?? uuid(payload.id);
+      if (root) ctx.session_id = root;
+      const thread = uuid(payload.id);
+      ctx.subagent_thread = thread && root && thread !== root ? thread
+        : payload.thread_source === "subagent" && thread ? thread : null;
+      if (safeLabel(payload.cli_version)) ctx.cli_version = safeLabel(payload.cli_version);
+      setCwd(ctx, payload.cwd, hmac, tenantFor, repoFor);
+      const git = payload.git && typeof payload.git === "object" ? payload.git : {};
+      ctx.git_branch = safeLabel(git.branch) ?? ctx.git_branch ?? null;
+      ctx.commit_sha = commitSha(git.commit_hash) ?? ctx.commit_sha ?? null;
+      ctx.repo = normalizeRepo(git.repository_url) ?? ctx.repo ?? null;
+      return NONE;
+    }
+    if (record.type === "turn_context") {
+      if (modelLabel(payload.model)) ctx.model = modelLabel(payload.model);
+      setCwd(ctx, payload.cwd, hmac, tenantFor, repoFor);
+      return NONE;
+    }
+    if (record.type === "response_item") {
+      if (["function_call", "local_shell_call", "custom_tool_call"].includes(payload.type)) {
+        ctx.pending_tool_calls = (ctx.pending_tool_calls ?? 0) + 1;
+      }
+      return NONE;
+    }
+    if (record.type === "token_usage_record") {
+      ctx.has_usage_records = true;
+      const id = responseId(payload.response_id);
+      const session = uuid(payload.session_id) ?? ctx.session_id ?? null;
+      const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null;
+      if (!id || !session || !usage) return { envelopes: [], refused: 1 };
+      const toolCalls = ctx.pending_tool_calls ?? 0;
+      ctx.pending_tool_calls = 0;
+      return { envelopes: [envelopeFor({ record, usage, usageFinal: true, messageId: id, session, toolCalls, offset, sourceHmac, ctx })], refused: 0 };
+    }
+    if (record.type !== "event_msg" || payload.type !== "token_count") return NONE;
+    // Each usage record is followed by a token_count carrying the same numbers.
+    if (ctx.has_usage_records) return NONE;
+
+    const info = payload.info && typeof payload.info === "object" ? payload.info : null;
+    const total = info?.total_token_usage;
+    const cumulative = totalOf(total);
+    // A token_count without cumulative usage (e.g. a rate-limit-only update)
+    // has no stable identity: refuse it rather than invent one.
+    if (!info || cumulative === null || !ctx.session_id) return info ? { envelopes: [], refused: 1 } : NONE;
+    // Codex re-emits token_count with unchanged totals; within one accounting
+    // epoch the cumulative total strictly increases per real response, so it is
+    // the message identity. Older rollouts reset the total (after compaction):
+    // a drop starts a new epoch so a later equal total is never read as a repeat.
+    if (ctx.last_total_tokens === cumulative) return NONE;
+    if (typeof ctx.last_total_tokens === "number" && cumulative < ctx.last_total_tokens) {
+      ctx.total_epoch = (ctx.total_epoch ?? 0) + 1;
+      ctx.last_total_usage = null; // a delta across the reset would be meaningless
+    }
+
+    let usage = info.last_token_usage && typeof info.last_token_usage === "object" ? info.last_token_usage : null;
+    let usageFinal = true;
+    if (!usage) {
+      // No per-response usage: derive it from the previous cumulative total;
+      // without one the row cannot be split from earlier turns → not final.
+      usage = deltaUsage(total, ctx.last_total_usage);
+      usageFinal = Boolean(ctx.last_total_usage);
+    }
+    ctx.last_total_tokens = cumulative;
+    ctx.last_total_usage = {
+      input_tokens: tok(total.input_tokens),
+      cached_input_tokens: tok(total.cached_input_tokens),
+      output_tokens: tok(total.output_tokens),
+      reasoning_output_tokens: tok(total.reasoning_output_tokens),
+    };
+    const toolCalls = ctx.pending_tool_calls ?? 0;
+    ctx.pending_tool_calls = 0;
+
+    // A legacy subagent's cumulative totals restart at zero: keep its keys
+    // apart from the parent's under the shared root session.
+    const epoch = ctx.total_epoch ? `r${ctx.total_epoch}:` : "";
+    const messageId = ctx.subagent_thread
+      ? `token_count:${ctx.subagent_thread}:${epoch}${cumulative}`
+      : `token_count:${epoch}${cumulative}`;
+    return {
+      envelopes: [envelopeFor({ record, usage, usageFinal, messageId, session: ctx.session_id, toolCalls, offset, sourceHmac, ctx })],
+      refused: 0,
+    };
+  }
+
+  function envelopeFor({ record, usage, usageFinal, messageId, session, toolCalls, offset, sourceHmac, ctx }) {
+    const classes = {
+      codex_input_tokens: tok(usage.input_tokens),
+      codex_cached_input_tokens: tok(usage.cached_input_tokens),
+      codex_output_tokens: tok(usage.output_tokens),
+      codex_reasoning_output_tokens: tok(usage.reasoning_output_tokens),
+    };
+    const ticket = inferTicketKey(ctx.git_branch, null) ?? ctx.cwd_ticket ?? null;
+    return withRoute(finalizeEnvelope({
+      provider: CODEX_PROVIDER,
+      provider_schema_version: ctx.cli_version ?? null,
+      harness_session_id: session,
+      message_id: messageId,
+      occurred_at: str(record.timestamp),
+      model: ctx.model ?? null,
+      ...classes,
+      tool_call_count: toolCalls,
+      is_sidechain: Boolean(ctx.subagent_thread),
+      usage_final: usageFinal,
+      // The server's rule: Codex is 'measured' when input/cached/output are present.
+      usage_quality: usageQuality({
+        usageFinal, classes: [classes.codex_input_tokens, classes.codex_cached_input_tokens, classes.codex_output_tokens],
+      }),
+      git_branch: ctx.git_branch ?? null,
+      repo: ctx.repo ?? null,
+      commit_sha: ctx.commit_sha ?? null,
+      ticket_key_inferred: ticket,
+      attribution_confidence: ticket ? "inferred" : null,
+      environment,
+      cwd_hmac: ctx.cwd_hmac ?? null,
+      source_hmac: sourceHmac,
+      source_offset: offset,
+      parser_version: CODEX_PARSER_VERSION,
+    }), ctx.cwd_tenant ?? null);
+  }
+
+  return {
+    provider: CODEX_PROVIDER,
+    parserVersion: CODEX_PARSER_VERSION,
+    /** One file's parser; token_count rows are complete when written, so nothing is ever held back. */
+    beginFile({ sourceHmac, ctx }) {
+      const envelopes = [];
+      const stats = { refused: 0, withheld: 0 };
+      return {
+        stats,
+        parseRecord(record, offset) {
+          const result = parseCodexRecord(record, { offset, sourceHmac, ctx });
+          stats.refused += result.refused;
+          envelopes.push(...result.envelopes);
+        },
+        finish() { return { envelopes, holdOffset: null }; },
+      };
+    },
+  };
+}

package/src/usage/cursor.mjs

@@ -0,0 +1,151 @@
+// BOT-803 — per-file byte-offset cursors for usage sync.
+//
+// `~/.botbuddy/usage-cursors.json` (0600) maps each transcript's source HMAC
+// (never its path) to the byte offset just past the last COMPLETE line that was
+// successfully delivered, plus the adapter's derived per-file context. A live
+// writer's half-written last line is never consumed, so the next run resumes on
+// it. Writes are tmp-file + rename and merge with whatever is on disk at write
+// time — no lock, so a pre-push run can never wait on the daily sweep (BOT-1023
+// AC4). A lost race only means a later run re-reads a few lines; the server's
+// idempotency key makes that a no-op.
+import { open, readFile, rename, mkdir, writeFile, unlink } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { randomBytes } from "node:crypto";
+
+export const CURSOR_FILE = "usage-cursors.json";
+export const MAX_RUNS_KEPT = 20;
+const CHUNK = 1 << 20;
+const NEWLINE = 0x0a;
+
+export function cursorPath(home = homedir()) {
+  return join(home, ".botbuddy", CURSOR_FILE);
+}
+
+const emptyState = () => ({ schema_version: 1, files: {}, runs: [] });
+
+/**
+ * Load the cursor state. A MISSING file is an empty state; a corrupt or
+ * unknown-schema file fails closed with `cursor_state_invalid` (Codex #982 R7):
+ * reading it as empty would drop the offsets that keep partly read files in
+ * scope and then overwrite them. The file is left untouched for the user —
+ * delete it to start over (every transcript in --since is re-read once and the
+ * server dedupes).
+ */
+export async function loadCursorState(path) {
+  let text;
+  try {
+    text = await readFile(path, "utf8");
+  } catch (error) {
+    if (error?.code === "ENOENT") return emptyState();
+    throw Object.assign(new Error("usage cursor state is unreadable"), { code: "cursor_state_invalid" });
+  }
+  let parsed;
+  try { parsed = JSON.parse(text); } catch { parsed = null; }
+  if (!parsed || typeof parsed !== "object" || parsed.schema_version !== 1
+    || (parsed.files !== undefined && (typeof parsed.files !== "object" || parsed.files === null || Array.isArray(parsed.files)))) {
+    throw Object.assign(new Error("usage cursor state is corrupt or from another version"), { code: "cursor_state_invalid" });
+  }
+  return {
+    schema_version: 1,
+    files: parsed.files ?? {},
+    runs: Array.isArray(parsed.runs) ? parsed.runs : [],
+  };
+}
+
+/**
+ * Merge `updates` (source_hmac → entry) and `runs` into the on-disk state and
+ * write it atomically. For a file both sides know, the entry with the larger
+ * offset wins — two overlapping triggers can only move a cursor forward.
+ */
+export async function saveCursorState(path, { files = {}, runs = [] } = {}) {
+  const current = await loadCursorState(path);
+  for (const [key, entry] of Object.entries(files)) {
+    const existing = current.files[key];
+    if (!existing || !(existing.offset > entry.offset) || entry.reset === true) {
+      const { reset: _reset, ...clean } = entry;
+      current.files[key] = clean;
+    }
+  }
+  current.runs = [...current.runs, ...runs].slice(-MAX_RUNS_KEPT);
+  await mkdir(join(path, ".."), { recursive: true, mode: 0o700 });
+  const tmp = `${path}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
+  try {
+    await writeFile(tmp, `${JSON.stringify(current)}\n`, { mode: 0o600 });
+    await rename(tmp, path);
+  } catch (error) {
+    await unlink(tmp).catch(() => {});
+    throw error;
+  }
+  return current;
+}
+
+/** A transcript line longer than this is hostile or corrupt: skipped, never buffered. */
+export const MAX_LINE_BYTES = 16 * 1024 * 1024;
+const STOP_CHECK_EVERY = 256;
+
+/**
+ * Stream the complete lines of `path` starting at byte `fromOffset`.
+ * `onLine(text, lineStartOffset)` is called per newline-terminated line (a
+ * trailing CR is stripped). Returns the offset just past the last line handed
+ * out, whether an unterminated tail (a live writer mid-line) was left unread,
+ * whether the budget stopped the read, and how many oversized lines were
+ * skipped. `shouldStop()` is polled between chunks AND every 256 lines, so one
+ * 1 MiB chunk of tiny lines cannot outlive the budget; a line longer than
+ * `maxLineBytes` is dropped (counted) instead of growing the buffer unbounded.
+ */
+export async function readCompleteLines(path, fromOffset, onLine, { shouldStop = () => false, maxLineBytes = MAX_LINE_BYTES } = {}) {
+  const handle = await open(path, "r");
+  try {
+    let position = fromOffset;
+    let consumed = fromOffset;
+    let carry = Buffer.alloc(0);
+    const buffer = Buffer.alloc(CHUNK);
+    let stopped = false;
+    let skipping = false;
+    let oversized = 0;
+    let lines = 0;
+    outer: for (;;) {
+      if (shouldStop()) { stopped = true; break; }
+      const { bytesRead } = await handle.read(buffer, 0, CHUNK, position);
+      if (bytesRead === 0) break;
+      position += bytesRead;
+      let data = carry.length ? Buffer.concat([carry, buffer.subarray(0, bytesRead)]) : Buffer.from(buffer.subarray(0, bytesRead));
+      let start = 0;
+      if (skipping) {
+        const nl = data.indexOf(NEWLINE);
+        if (nl === -1) { consumed += data.length; carry = Buffer.alloc(0); continue; }
+        start = nl + 1;
+        skipping = false;
+      }
+      for (;;) {
+        const nl = data.indexOf(NEWLINE, start);
+        if (nl === -1) break;
+        let end = nl;
+        if (end > start && data[end - 1] === 0x0d) end -= 1;
+        if (nl - start > maxLineBytes) oversized += 1;
+        else onLine(data.toString("utf8", start, end), consumed + start);
+        start = nl + 1;
+        lines += 1;
+        if (lines % STOP_CHECK_EVERY === 0 && shouldStop()) {
+          stopped = true;
+          consumed += start;
+          carry = Buffer.alloc(0);
+          break outer;
+        }
+      }
+      consumed += start;
+      carry = data.subarray(start);
+      if (carry.length > maxLineBytes) {
+        oversized += 1;
+        consumed += carry.length;
+        carry = Buffer.alloc(0);
+        skipping = true;
+      }
+      data = null;
+    }
+    return { endOffset: consumed, partialTail: carry.length > 0, stopped, oversized };
+  } finally {
+    await handle.close();
+  }
+}