@threadcrumbs/common 0.0.0-stage → 0.1.0

package/telemetry/codex.mjs

@@ -0,0 +1,556 @@
+import fsp from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { exists, jsonLines, listDirs, listFiles } from '../lib/fsutil.mjs'
+import { jsonlFollower } from '../lib/jsonl-follower.mjs'
+import { columnsOf, tablesOf, withReadOnly } from '../lib/sqlite.mjs'
+import {
+  DETAIL_LIMITS,
+  RECENT_MS,
+  activityDetail,
+  addFileChange,
+  clip,
+  clipBlock,
+  diffHunk,
+  emptyActivity,
+  endCall,
+  fileList,
+  keepNewest,
+  nextPlan,
+  number,
+  planItems,
+  planSummary,
+  relativePath,
+  settleCalls,
+  startCall,
+  text,
+  titleFromPrompt,
+  toolInput,
+  toolTarget,
+} from '../lib/telemetry-fields.mjs'
+
+const ACTIVE_WINDOW_MS = 30 * 60 * 1000
+const SUBAGENT_WINDOW_MS = 10 * 60 * 1000
+const ROLLOUT_FILE = /^rollout-.{19}-([0-9a-f-]{36})(?:_[0-9a-f-]{36})?\.jsonl$/i
+const STATE_DB = /^state_(\d+)\.sqlite$/
+const CALLS = new Set(['function_call', 'custom_tool_call', 'local_shell_call', 'tool_search_call'])
+const OUTPUTS = new Set(['function_call_output', 'custom_tool_call_output', 'tool_search_output'])
+const AGENT_TOOLS = new Set(['spawn_agent', 'wait_agent', 'send_input'])
+const QUESTION_TOOLS = new Set(['request_user_input', 'request_permissions'])
+const INTERNAL_SOURCES = new Set(['guardian_review', 'memory_consolidation'])
+const PATCH_APPLIED = /Success\. Updated the following files/
+const PATCH_FILE = /^\*\*\* (?:Update|Add|Delete) File: (.+)$/
+const NOISE = /<([a-z][\w-]*)(?:\s[^>]*)?>[\s\S]*?<\/\1>/g
+
+function contentText(content) {
+  if (typeof content === 'string') return content
+  if (!Array.isArray(content)) return ''
+  return content.map((p) => (typeof p === 'string' ? p : text(p?.text) || text(p?.input_text))).filter(Boolean).join('\n')
+}
+
+function promptText(raw) {
+  const prompt = text(raw).replace(NOISE, ' ').trim()
+  return prompt && !prompt.startsWith('<') && !prompt.startsWith('# AGENTS.md') ? prompt : ''
+}
+
+function parseArgs(raw) {
+  if (raw && typeof raw === 'object') return raw
+  try {
+    return JSON.parse(text(raw) || '{}')
+  } catch {
+    return null
+  }
+}
+
+function outputText(output) {
+  if (Array.isArray(output)) return contentText(output)
+  const raw = text(output)
+  try {
+    const parsed = JSON.parse(raw)
+    return typeof parsed?.output === 'string' ? parsed.output : raw
+  } catch {
+    return raw
+  }
+}
+
+function exitFailed(output) {
+  if (typeof output !== 'string') return null
+  try {
+    const code = JSON.parse(output)?.metadata?.exit_code
+    return Number.isFinite(code) ? code !== 0 : null
+  } catch {
+    return null
+  }
+}
+
+const patchFile = (patch) => patch.split('\n').map((line) => PATCH_FILE.exec(line)?.[1]).find(Boolean)?.trim() || ''
+
+function callTarget(item, args, patch, cwd) {
+  if (patch) {
+    const file = patchFile(patch)
+    return file ? toolTarget({ path: file }, cwd) : ''
+  }
+  if (item.type === 'local_shell_call') return toolTarget({ command: item.action?.command }, cwd)
+  if (item.type === 'custom_tool_call') return toolTarget(text(item.input), cwd)
+  return toolTarget(args ?? parseArgs(item.arguments), cwd)
+}
+
+function callInput(item, args, patch) {
+  if (patch) return null
+  if (item.type === 'local_shell_call') return toolInput({ command: item.action?.command })
+  if (item.type === 'custom_tool_call') return toolInput(text(item.input))
+  return toolInput(args ?? parseArgs(item.arguments))
+}
+
+function latestCall(calls) {
+  let latest = null
+  for (const call of calls.values()) {
+    if (!latest || call.startedAt >= latest.startedAt) latest = { name: call.name, startedAt: call.startedAt }
+  }
+  return latest
+}
+
+function emptyRollout() {
+  return {
+    threadId: '',
+    parentId: '',
+    subagent: false,
+    internal: false,
+    cwd: '',
+    gitBranch: '',
+    model: '',
+    effort: '',
+    startedAt: 0,
+    lastAt: 0,
+    firstPrompt: '',
+    prompts: 0,
+    typed: false,
+    fallbackPrompt: '',
+    fallbackPrompts: 0,
+    open: false,
+    failed: false,
+    calls: new Map(),
+    patches: new Map(),
+    lines: { added: 0, removed: 0 },
+    tokens: null,
+    context: null,
+    plan: null,
+    history: [],
+    contextParts: null,
+    lastTotal: -1,
+    modelTokens: new Map(),
+    activity: emptyActivity(),
+    files: new Map(),
+    promptList: [],
+    fallbackPromptList: [],
+    errors: [],
+  }
+}
+
+function applyRecord(state, record) {
+  const at = Date.parse(record.timestamp)
+  if (at) {
+    if (!state.startedAt) state.startedAt = at
+    if (at > state.lastAt) state.lastAt = at
+  }
+  const payload = record.payload
+  if (!payload || typeof payload !== 'object') return
+  if (record.type === 'session_meta') applyMeta(state, payload)
+  else if (record.type === 'turn_context') applySettings(state, payload.model, payload.effort, payload.cwd)
+  else if (record.type === 'response_item') applyItem(state, payload, at || state.lastAt)
+  else if (record.type === 'event_msg') applyEvent(state, payload, at || state.lastAt)
+}
+
+function applyMeta(state, meta) {
+  const subagent = meta.source?.subagent
+  const spawn = subagent?.thread_spawn
+  state.threadId ||= text(meta.id).toLowerCase()
+  state.parentId ||= (text(meta.parent_thread_id) || text(spawn?.parent_thread_id)).toLowerCase()
+  state.subagent ||= meta.thread_source === 'subagent' || Boolean(spawn)
+  state.internal ||= INTERNAL_SOURCES.has(meta.thread_source) || (subagent != null && !spawn)
+  state.cwd ||= text(meta.cwd)
+  state.gitBranch ||= text(meta.git?.branch)
+}
+
+function applySettings(state, model, effort, cwd) {
+  if (model) state.model = text(model)
+  if (effort) state.effort = text(effort)
+  if (cwd) state.cwd = text(cwd)
+}
+
+function countPrompt(state, raw, at) {
+  const prompt = promptText(raw)
+  if (!prompt) return
+  state.prompts++
+  state.firstPrompt ||= prompt
+  keepNewest(state.promptList, { at, text: clipBlock(prompt, DETAIL_LIMITS.promptText) }, DETAIL_LIMITS.prompts)
+}
+
+function applyItem(state, item, at) {
+  if (item.type === 'message' && item.role === 'user') {
+    const prompt = promptText(contentText(item.content))
+    if (prompt) {
+      state.fallbackPrompts++
+      state.fallbackPrompt ||= prompt
+      keepNewest(state.fallbackPromptList, { at, text: clipBlock(prompt, DETAIL_LIMITS.promptText) }, DETAIL_LIMITS.prompts)
+    }
+    return
+  }
+  const id = text(item.call_id)
+  if (!id) return
+  if (CALLS.has(item.type)) {
+    let name = `${text(item.namespace)}${text(item.name)}`
+    if (item.type === 'local_shell_call') name = 'shell'
+    if (item.type === 'tool_search_call') name = 'tool_search'
+    state.calls.set(id, { name, startedAt: at })
+    const args = item.type === 'function_call' ? parseArgs(item.arguments) : null
+    if (item.type === 'function_call' && !item.namespace && item.name === 'update_plan') state.plan = nextPlan(state.plan, 'update_plan', args)
+    const patch = item.name === 'apply_patch' ? (item.type === 'custom_tool_call' ? text(item.input) : text(args?.input)) : ''
+    if (patch) state.patches.set(id, patch)
+    const file = patch ? patchFile(patch) : ''
+    startCall(state.activity, {
+      id,
+      name,
+      target: callTarget(item, args, patch, state.cwd),
+      input: callInput(item, args, patch),
+      path: file ? relativePath(file, state.cwd) : null,
+      startedAt: at,
+    })
+    return
+  }
+  if (!OUTPUTS.has(item.type)) return
+  state.calls.delete(id)
+  const patch = state.patches.get(id)
+  if (patch === undefined) {
+    const failed = exitFailed(item.output)
+    endCall(state.activity, id, at, failed, failed ? outputText(item.output) : null)
+    return
+  }
+  state.patches.delete(id)
+  const output = outputText(item.output)
+  const applied = PATCH_APPLIED.test(output)
+  endCall(state.activity, id, at, !applied, applied ? null : output)
+  if (applied) addLines(state, patch)
+}
+
+function addLines(state, patch) {
+  const counts = new Map()
+  let entry = null
+  let hunk = null
+  for (const line of patch.split('\n')) {
+    const header = PATCH_FILE.exec(line)
+    if (header) {
+      const file = header[1].trim()
+      entry = counts.get(file) || { added: 0, removed: 0, hunks: [] }
+      counts.set(file, entry)
+      hunk = null
+      continue
+    }
+    if (line.startsWith('@@')) {
+      hunk = null
+      continue
+    }
+    if (line.startsWith('***')) continue
+    if (line.startsWith('+')) {
+      state.lines.added++
+      if (entry) entry.added++
+    } else if (line.startsWith('-')) {
+      state.lines.removed++
+      if (entry) entry.removed++
+    } else if (!line.startsWith(' ')) continue
+    if (!entry) continue
+    if (!hunk) {
+      hunk = []
+      entry.hunks.push(hunk)
+    }
+    hunk.push(line)
+  }
+  for (const [name, count] of counts) {
+    addFileChange(state.files, name, count.added, count.removed, count.hunks.map((lines) => diffHunk({ lines })))
+  }
+}
+
+function applyEvent(state, event, at) {
+  switch (event.type) {
+    case 'task_started':
+    case 'turn_started':
+      state.open = true
+      state.failed = false
+      state.calls.clear()
+      settleCalls(state.activity)
+      break
+    case 'task_complete':
+    case 'turn_complete':
+      state.open = false
+      state.failed = event.error != null
+      state.calls.clear()
+      settleCalls(state.activity)
+      if (event.error != null) {
+        const message = text(event.error?.message) || text(event.error) || 'The turn failed'
+        keepNewest(state.errors, { at, message: clip(message, DETAIL_LIMITS.errorText) }, DETAIL_LIMITS.errors)
+      }
+      break
+    case 'turn_aborted':
+      state.open = false
+      state.failed = false
+      state.calls.clear()
+      settleCalls(state.activity)
+      break
+    case 'token_count':
+      applyTokens(state, event.info, at)
+      break
+    case 'user_message':
+      state.typed = true
+      countPrompt(state, event.message, at)
+      break
+    case 'item_completed':
+      if (event.item?.type !== 'UserMessage') break
+      state.typed = true
+      countPrompt(state, contentText(event.item.content), at)
+      break
+    case 'thread_settings_applied': {
+      const settings = event.thread_settings
+      if (settings) applySettings(state, settings.model, settings.reasoning_effort, settings.cwd)
+      break
+    }
+  }
+}
+
+function applyTokens(state, info, at) {
+  if (!info) return
+  const total = info.total_token_usage
+  if (total) {
+    const cached = number(total.cached_input_tokens)
+    const next = {
+      input: Math.max(0, number(total.input_tokens) - cached),
+      output: number(total.output_tokens),
+      cacheRead: cached,
+      cacheWrite: number(total.cache_write_input_tokens),
+    }
+    if (state.model) {
+      const sums = state.modelTokens.get(state.model) || { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
+      for (const key of Object.keys(next)) sums[key] += Math.max(0, next[key] - (state.tokens?.[key] || 0))
+      state.modelTokens.set(state.model, sums)
+    }
+    state.tokens = next
+  }
+  const last = info.last_token_usage
+  if (!last) return
+  state.context = { used: number(last.total_tokens), limit: number(info.model_context_window) }
+  if (last.input_tokens != null) {
+    const cached = number(last.cached_input_tokens)
+    state.contextParts = { input: Math.max(0, number(last.input_tokens) - cached), cacheRead: cached, cacheWrite: 0, output: number(last.output_tokens) }
+  }
+  const sum = number(total?.total_tokens)
+  if (sum === state.lastTotal) return
+  state.lastTotal = sum
+  keepNewest(state.history, { at, context: number(last.total_tokens), output: number(last.output_tokens) }, DETAIL_LIMITS.history)
+}
+
+export function codex({
+  home = process.env.CODEX_HOME || path.join(os.homedir(), '.codex'),
+  sqliteHome = process.env.CODEX_SQLITE_HOME || home,
+  now = Date.now,
+} = {}) {
+  const sessionsDir = path.join(home, 'sessions')
+  const locksDir = path.join(home, 'thread-writer-locks')
+  const indexFile = path.join(home, 'session_index.jsonl')
+  const rollouts = jsonlFollower({ empty: emptyRollout, apply: applyRecord })
+  let index = { mtime: -1, names: new Map() }
+  let known = new Map()
+  let childrenOf = new Map()
+
+  async function findRollouts(locks) {
+    const cutoff = now() - RECENT_MS
+    const byId = new Map()
+    for (const year of await listDirs(sessionsDir)) {
+      for (const month of await listDirs(year)) {
+        for (const day of await listDirs(month)) {
+          for (const file of await listFiles(day, (n) => ROLLOUT_FILE.test(n))) {
+            const id = ROLLOUT_FILE.exec(path.basename(file))[1].toLowerCase()
+            const stat = await fsp.stat(file).catch(() => null)
+            if (!stat || (stat.mtimeMs < cutoff && !locks.has(id))) continue
+            const existing = byId.get(id)
+            if (existing && existing.stat.mtimeMs >= stat.mtimeMs) continue
+            byId.set(id, { id, file, stat })
+          }
+        }
+      }
+    }
+    return [...byId.values()]
+  }
+
+  async function latestStateDb() {
+    const files = await listFiles(sqliteHome, (n) => STATE_DB.test(n))
+    return files.sort((a, b) => Number(STATE_DB.exec(path.basename(b))[1]) - Number(STATE_DB.exec(path.basename(a))[1]))[0] || ''
+  }
+
+  async function threadRecords() {
+    const stat = await fsp.stat(indexFile).catch(() => null)
+    if (stat && stat.mtimeMs !== index.mtime) {
+      const names = new Map()
+      for (const row of jsonLines(await fsp.readFile(indexFile, 'utf8').catch(() => ''))) {
+        if (row?.id && row.thread_name) names.set(String(row.id).toLowerCase(), text(row.thread_name))
+      }
+      index = { mtime: stat.mtimeMs, names }
+    }
+    const names = new Map(index.names)
+    const archived = new Set()
+    const read = new Map()
+    await withReadOnly(await latestStateDb(), (db) => {
+      const tables = tablesOf(db)
+      if (!tables.has('threads')) return
+      const columns = columnsOf(db, 'threads')
+      if (columns.has('name')) {
+        for (const row of db.prepare("SELECT id, name FROM threads WHERE name IS NOT NULL AND name != ''").all()) {
+          names.set(String(row.id).toLowerCase(), text(row.name))
+        }
+      }
+      if (columns.has('archived')) {
+        for (const row of db.prepare('SELECT id FROM threads WHERE archived = 1').all()) archived.add(String(row.id).toLowerCase())
+      }
+      if (tables.has('thread_read_receipts') && ['thread_id', 'first_unread_turn'].every((c) => columnsOf(db, 'thread_read_receipts').has(c))) {
+        for (const row of db.prepare('SELECT thread_id, first_unread_turn FROM thread_read_receipts').all()) {
+          read.set(String(row.thread_id).toLowerCase(), row.first_unread_turn != null)
+        }
+      }
+    })
+    return { names, archived, read }
+  }
+
+  async function scanSessions() {
+    const at = now()
+    const locking = await exists(locksDir)
+    const locks = new Set((await listFiles(locksDir, (n) => n.endsWith('.lock'))).map((f) => path.basename(f, '.lock').toLowerCase()))
+    const [found, { names, archived, read: unread }] = await Promise.all([findRollouts(locks), threadRecords()])
+    const read = []
+    for (const rollout of found) {
+      try {
+        read.push({ ...rollout, state: await rollouts.read(rollout.file, rollout.stat) })
+      } catch {}
+    }
+    rollouts.sweep()
+    known = new Map(read.map((r) => [r.id, r.file]))
+    childrenOf = new Map()
+    for (const { id, state } of read) {
+      if (state.subagent && state.parentId) childrenOf.set(state.parentId, [...(childrenOf.get(state.parentId) || []), id])
+    }
+
+    const running = new Map()
+    for (const { id, stat, state } of read) {
+      if (!state.subagent || !state.parentId || !state.open) continue
+      if (!locks.has(id) && at - stat.mtimeMs > SUBAGENT_WINDOW_MS) continue
+      running.set(state.parentId, (running.get(state.parentId) || 0) + 1)
+    }
+
+    return read
+      .filter(({ id, state }) => !state.subagent && !state.internal && !archived.has(id))
+      .map(({ id, stat, state }) => {
+        const lastActivityAt = Math.max(state.lastAt, Math.floor(stat.mtimeMs))
+        const fresh = at - lastActivityAt < ACTIVE_WINDOW_MS
+        const live = locking ? locks.has(id) : state.open && fresh
+        return toSession({ id, state, names, live, turnLive: state.open && (locking ? live : fresh), fresh, locking, unread: unread.get(id), subagents: running.get(id) || 0, lastActivityAt })
+      })
+  }
+
+  async function readKnown(id) {
+    const file = known.get(id)
+    const stat = file ? await fsp.stat(file).catch(() => null) : null
+    if (!stat) return null
+    const state = await rollouts.read(file, stat).catch(() => null)
+    return state ? { file, stat, state } : null
+  }
+
+  async function sessionDetail(key) {
+    const id = String(key).replace(/^codex:/, '').toLowerCase()
+    const found = await readKnown(id)
+    if (!found || found.state.subagent || found.state.internal) return null
+    const { file, state } = found
+    const subagents = []
+    for (const childId of childrenOf.get(id) || []) {
+      const child = await readKnown(childId)
+      if (!child) continue
+      const c = child.state
+      const lastActivityAt = Math.max(c.lastAt, Math.floor(child.stat.mtimeMs))
+      const status = c.open && now() - lastActivityAt < ACTIVE_WINDOW_MS ? 'working' : c.failed ? 'error' : 'done'
+      subagents.push({
+        id: childId,
+        name: titleFromPrompt(c.typed ? c.firstPrompt : c.fallbackPrompt) || 'Subagent',
+        type: '',
+        status,
+        model: c.model,
+        tokens: c.tokens ? { ...c.tokens } : null,
+        tool: status === 'working' ? latestCall(c.calls) : null,
+        startedAt: c.startedAt,
+        lastActivityAt,
+      })
+    }
+    return {
+      id: `codex:${id}`,
+      sessionId: id,
+      source: file,
+      models: [...state.modelTokens].map(([model, tokens]) => ({ model, tokens: { ...tokens }, costUsd: null })),
+      contextParts: state.contextParts ? { ...state.contextParts } : null,
+      history: state.history.map((p) => ({ ...p })),
+      compactions: null,
+      ...activityDetail(state.activity),
+      plan: planItems(state.plan),
+      subagents: subagents.sort((a, b) => a.startedAt - b.startedAt),
+      files: fileList(state.files, state.cwd),
+      prompts: (state.typed ? state.promptList : state.fallbackPromptList).map((p) => ({ ...p })),
+      errors: state.errors.map((e) => ({ ...e })),
+    }
+  }
+
+  return {
+    id: 'codex',
+    name: 'Codex',
+    detect: () => exists(sessionsDir),
+    watchRoots: () => [sessionsDir, locksDir],
+    scanSessions,
+    sessionDetail,
+  }
+}
+
+function toSession({ id, state, names, live, turnLive, fresh, locking, unread, subagents, lastActivityAt }) {
+  const calls = [...state.calls.values()]
+  let status
+  if (turnLive) {
+    if (calls.some((c) => QUESTION_TOOLS.has(c.name))) status = 'asking'
+    else if (calls.length ? calls.every((c) => AGENT_TOOLS.has(c.name)) : subagents > 0) status = 'delegating'
+    else status = 'working'
+  } else if (subagents) status = 'delegating'
+  else if (state.failed) status = 'error'
+  else if (unread !== undefined) status = unread ? 'done' : 'idle'
+  else status = (locking ? live : fresh) ? 'done' : 'idle'
+
+  const tool = turnLive && (status === 'working' || status === 'delegating') ? latestCall(state.calls) : null
+  const prompt = state.typed ? state.firstPrompt : state.fallbackPrompt
+  const cwd = state.cwd
+  return {
+    id: `codex:${id}`,
+    harness: 'codex',
+    title: names.get(id) || (prompt ? titleFromPrompt(prompt) : '') || 'Untitled session',
+    project: path.basename(cwd.replace(/\\/g, '/')) || cwd || 'unknown',
+    projectPath: cwd,
+    worktree: '',
+    cwd,
+    gitBranch: state.gitBranch,
+    model: state.model,
+    effort: state.effort,
+    status,
+    live: live || turnLive,
+    startedAt: state.startedAt,
+    lastActivityAt,
+    tool,
+    tokens: state.tokens ? { ...state.tokens } : null,
+    context: state.context && state.context.used ? { ...state.context } : null,
+    costUsd: null,
+    lines: { ...state.lines },
+    plan: planSummary(state.plan),
+    subagents,
+    prompts: state.typed ? state.prompts : state.fallbackPrompts,
+  }
+}
+
+export default codex()