@threadcrumbs/common 0.1.2 → 0.1.4

package/telemetry/codex.mjs

@@ -2,35 +2,49 @@ import fsp from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
 import { exists, jsonLines, listDirs, listFiles } from '../lib/fsutil.mjs'
-import { jsonlFollower } from '../lib/jsonl-follower.mjs'
+import { contextBreakdown, emptyFixed, emptySizes, setFixed, sizeCall, sizeCompaction, sizePrompt, sizeReply, sizeRequest, sizeResult } from '../lib/context-breakdown.mjs'
+import { jsonlFollower, recentReaders } from '../lib/jsonl-follower.mjs'
 import { columnsOf, tablesOf, withReadOnly } from '../lib/sqlite.mjs'
 import {
   DETAIL_LIMITS,
   RECENT_MS,
   activityDetail,
   addFileChange,
+  askOf,
+  askQuestions,
   clip,
   clipBlock,
   diffHunk,
   emptyActivity,
+  emptySample,
   endCall,
   fileList,
+  inputKind,
+  joinedText,
   keepNewest,
+  keepText,
   nextPlan,
   number,
   planItems,
   planSummary,
   relativePath,
+  sampleList,
+  samplePoint,
   settleCalls,
   startCall,
   text,
+  textList,
   titleFromPrompt,
   toolInput,
+  toolNow,
   toolTarget,
 } from '../lib/telemetry-fields.mjs'
 
 const ACTIVE_WINDOW_MS = 30 * 60 * 1000
 const SUBAGENT_WINDOW_MS = 10 * 60 * 1000
+const OPENED_KEPT = 3
+const COMPACT_SHARE = 0.9
+const REPORTED_SHARE = 0.95
 const ROLLOUT_FILE = /^rollout-.{19}-([0-9a-f-]{36})(?:_[0-9a-f-]{36})?\.jsonl$/i
 const STATE_DB = /^state_(\d+)\.sqlite$/
 const CALLS = new Set(['function_call', 'custom_tool_call', 'local_shell_call', 'tool_search_call'])
@@ -105,9 +119,9 @@ function callInput(item, args, patch) {
 function latestCall(calls) {
   let latest = null
   for (const call of calls.values()) {
-    if (!latest || call.startedAt >= latest.startedAt) latest = { name: call.name, startedAt: call.startedAt }
+    if (!latest || call.startedAt >= latest.startedAt) latest = call
   }
-  return latest
+  return toolNow(latest)
 }
 
 function emptyRollout() {
@@ -128,12 +142,15 @@ function emptyRollout() {
     fallbackPrompt: '',
     fallbackPrompts: 0,
     open: false,
+    turnStartedAt: 0,
+    usageLimits: null,
     failed: false,
     calls: new Map(),
     patches: new Map(),
     lines: { added: 0, removed: 0 },
     tokens: null,
     context: null,
+    contextSample: emptySample(),
     plan: null,
     history: [],
     contextParts: null,
@@ -147,6 +164,87 @@ function emptyRollout() {
   }
 }
 
+const AGENTS_FILE = /^# AGENTS\.md instructions for ([^\n]+)/
+
+const emptyOpened = () => ({
+  lastAt: 0,
+  thoughts: [],
+  replies: [],
+  sizes: emptySizes(),
+  sent: { base: 0, developer: [], other: 0, memory: new Map(), tools: new Map() },
+})
+
+function callName(item) {
+  if (item.type === 'local_shell_call') return 'shell'
+  if (item.type === 'tool_search_call') return 'tool_search'
+  return `${text(item.namespace)}${text(item.name)}`
+}
+
+function codexFixed(sent) {
+  const fixed = emptyFixed()
+  const developer = sent.developer.reduce((n, chars) => n + chars, 0)
+  fixed.system = developer + (sent.developer.includes(sent.base) ? 0 : sent.base) + sent.other
+  fixed.mcp = new Map(sent.tools)
+  fixed.memory = new Map(sent.memory)
+  return fixed
+}
+
+function applyOpened(state, record) {
+  const at = Date.parse(record.timestamp) || state.lastAt
+  state.lastAt = at
+  const item = record.payload
+  if (!item || typeof item !== 'object') return
+  const { sizes, sent } = state
+  const before = sizes.first == null
+  if (record.type === 'session_meta') {
+    sent.base = text(item.base_instructions?.text).length
+  } else if (record.type === 'compacted') {
+    sizeCompaction(sizes)
+    return
+  } else if (record.type === 'event_msg') {
+    if (item.type === 'token_count' && item.info?.last_token_usage) sizeRequest(sizes, number(item.info.last_token_usage.input_tokens), at)
+    return
+  } else if (record.type !== 'response_item') {
+    return
+  } else if (item.type === 'reasoning') {
+    const said = joinedText(item.summary, 'summary_text', 'text')
+    keepText(state.thoughts, 'thoughts', { id: text(item.id) || null, at, text: said })
+    sizeReply(sizes, said.length)
+    return
+  } else if (item.type === 'message' && item.role === 'assistant') {
+    const said = joinedText(item.content, 'output_text', 'text')
+    keepText(state.replies, 'replies', { id: text(item.id) || null, at, text: said })
+    sizeReply(sizes, said.length)
+    return
+  } else if (CALLS.has(item.type)) {
+    sizeCall(sizes, text(item.call_id), callName(item), (text(item.arguments) || text(item.input) || JSON.stringify(item.action ?? '')).length)
+    return
+  } else if (OUTPUTS.has(item.type)) {
+    sizeResult(sizes, text(item.call_id), outputText(item.output).length)
+    return
+  } else if (item.type === 'additional_tools') {
+    for (const group of Array.isArray(item.tools) ? item.tools : []) {
+      const tools = Array.isArray(group?.tools) ? group.tools.length : 0
+      sent.tools.set(text(group?.name) || 'tools', { tools: tools || null, chars: JSON.stringify(group ?? null).length })
+    }
+  } else if (item.type === 'message') {
+    const raw = contentText(item.content)
+    const agents = item.role === 'user' ? AGENTS_FILE.exec(raw) : null
+    if (agents) sent.memory.set(`${agents[1].trim().replace(/[\/]+$/, '')}/AGENTS.md`, raw.length)
+    else if (item.role === 'user' && promptText(raw)) {
+      sizePrompt(sizes, raw.length)
+      return
+    } else if (!before) {
+      sizePrompt(sizes, raw.length)
+      return
+    } else if (item.role === 'developer' || item.role === 'system') sent.developer.push(raw.length)
+    else sent.other += raw.length
+  } else {
+    return
+  }
+  setFixed(sizes, codexFixed(sent))
+}
+
 function applyRecord(state, record) {
   const at = Date.parse(record.timestamp)
   if (at) {
@@ -199,23 +297,19 @@ function applyItem(state, item, at) {
   const id = text(item.call_id)
   if (!id) return
   if (CALLS.has(item.type)) {
-    let name = `${text(item.namespace)}${text(item.name)}`
-    if (item.type === 'local_shell_call') name = 'shell'
-    if (item.type === 'tool_search_call') name = 'tool_search'
-    state.calls.set(id, { name, startedAt: at })
+    const name = callName(item)
     const args = item.type === 'function_call' ? parseArgs(item.arguments) : null
     if (item.type === 'function_call' && !item.namespace && item.name === 'update_plan') state.plan = nextPlan(state.plan, 'update_plan', args)
     const patch = item.name === 'apply_patch' ? (item.type === 'custom_tool_call' ? text(item.input) : text(args?.input)) : ''
     if (patch) state.patches.set(id, patch)
     const file = patch ? patchFile(patch) : ''
-    startCall(state.activity, {
-      id,
-      name,
-      target: callTarget(item, args, patch, state.cwd),
-      input: callInput(item, args, patch),
-      path: file ? relativePath(file, state.cwd) : null,
-      startedAt: at,
-    })
+    const target = callTarget(item, args, patch, state.cwd)
+    const input = callInput(item, args, patch)
+    const entry = { name, startedAt: at, target, kind: inputKind(name, input).kind }
+    const questions = QUESTION_TOOLS.has(name) ? askQuestions(args) : null
+    if (questions) entry.questions = questions
+    state.calls.set(id, entry)
+    startCall(state.activity, { id, name, target, input, path: file ? relativePath(file, state.cwd) : null, startedAt: at })
     return
   }
   if (!OUTPUTS.has(item.type)) return
@@ -274,6 +368,7 @@ function applyEvent(state, event, at) {
   switch (event.type) {
     case 'task_started':
     case 'turn_started':
+      if (!state.open) state.turnStartedAt = at
       state.open = true
       state.failed = false
       state.calls.clear()
@@ -298,6 +393,7 @@ function applyEvent(state, event, at) {
       break
     case 'token_count':
       applyTokens(state, event.info, at)
+      applyLimits(state, event.rate_limits, at)
       break
     case 'user_message':
       state.typed = true
@@ -316,6 +412,21 @@ function applyEvent(state, event, at) {
   }
 }
 
+const LIMIT_WINDOWS = ['primary', 'secondary']
+
+function applyLimits(state, limits, at) {
+  if (!limits || typeof limits !== 'object') return
+  const windows = []
+  for (const id of LIMIT_WINDOWS) {
+    const w = limits[id]
+    if (!w || typeof w !== 'object' || w.used_percent == null) continue
+    const resetsAt = w.resets_at != null ? number(w.resets_at) * 1000 : w.resets_in_seconds != null ? at + number(w.resets_in_seconds) * 1000 : null
+    windows.push({ id, usedPercent: number(w.used_percent), windowMinutes: w.window_minutes == null ? null : number(w.window_minutes), resetsAt: resetsAt || null })
+  }
+  if (!windows.length) return
+  state.usageLimits = { at, subscription: text(limits.plan_type) || null, windows }
+}
+
 function applyTokens(state, info, at) {
   if (!info) return
   const total = info.total_token_usage
@@ -344,6 +455,7 @@ function applyTokens(state, info, at) {
   const sum = number(total?.total_tokens)
   if (sum === state.lastTotal) return
   state.lastTotal = sum
+  samplePoint(state.contextSample, last.total_tokens)
   keepNewest(state.history, { at, context: number(last.total_tokens), output: number(last.output_tokens) }, DETAIL_LIMITS.history)
 }
 
@@ -356,6 +468,7 @@ export function codex({
   const locksDir = path.join(home, 'thread-writer-locks')
   const indexFile = path.join(home, 'session_index.jsonl')
   const rollouts = jsonlFollower({ empty: emptyRollout, apply: applyRecord })
+  const openedFor = recentReaders(() => jsonlFollower({ empty: emptyOpened, apply: applyOpened }), OPENED_KEPT)
   let index = { mtime: -1, names: new Map() }
   let known = new Map()
   let childrenOf = new Map()
@@ -440,8 +553,9 @@ export function codex({
     for (const { id, stat, state } of read) {
       if (!state.subagent || !state.parentId || !state.open) continue
       if (!locks.has(id) && at - stat.mtimeMs > SUBAGENT_WINDOW_MS) continue
-      running.set(state.parentId, (running.get(state.parentId) || 0) + 1)
+      running.set(state.parentId, [...(running.get(state.parentId) || []), { id, name: childName(state), startedAt: state.startedAt }])
     }
+    const workingOf = (id) => (running.get(id) || []).sort((a, b) => a.startedAt - b.startedAt).map(({ id: childId, name }) => ({ id: childId, name }))
 
     return read
       .filter(({ id, state }) => !state.subagent && !state.internal && !archived.has(id))
@@ -449,7 +563,8 @@ export function codex({
         const lastActivityAt = Math.max(state.lastAt, Math.floor(stat.mtimeMs))
         const fresh = at - lastActivityAt < ACTIVE_WINDOW_MS
         const live = locking ? locks.has(id) : state.open && fresh
-        return toSession({ id, state, names, live, turnLive: state.open && (locking ? live : fresh), fresh, locking, unread: unread.get(id), subagents: running.get(id) || 0, lastActivityAt })
+        const workingSubagents = workingOf(id)
+        return toSession({ id, state, names, live, turnLive: state.open && (locking ? live : fresh), fresh, locking, unread: unread.get(id), subagents: workingSubagents.length, workingSubagents, lastActivityAt })
       })
   }
 
@@ -465,7 +580,9 @@ export function codex({
     const id = String(key).replace(/^codex:/, '').toLowerCase()
     const found = await readKnown(id)
     if (!found || found.state.subagent || found.state.internal) return null
-    const { file, state } = found
+    const { file, stat, state } = found
+    const said = await openedFor(file).read(file, stat).catch(() => null)
+    const limit = state.context?.limit || null
     const subagents = []
     for (const childId of childrenOf.get(id) || []) {
       const child = await readKnown(childId)
@@ -475,7 +592,7 @@ export function codex({
       const status = c.open && now() - lastActivityAt < ACTIVE_WINDOW_MS ? 'working' : c.failed ? 'error' : 'done'
       subagents.push({
         id: childId,
-        name: titleFromPrompt(c.typed ? c.firstPrompt : c.fallbackPrompt) || 'Subagent',
+        name: childName(c),
         type: '',
         status,
         model: c.model,
@@ -498,6 +615,13 @@ export function codex({
       subagents: subagents.sort((a, b) => a.startedAt - b.startedAt),
       files: fileList(state.files, state.cwd),
       prompts: (state.typed ? state.promptList : state.fallbackPromptList).map((p) => ({ ...p })),
+      thoughts: textList(said?.thoughts),
+      replies: textList(said?.replies),
+      contextBreakdown: contextBreakdown(said?.sizes, {
+        total: state.context?.used,
+        limit,
+        reserved: limit ? Math.round(limit * (1 - COMPACT_SHARE / REPORTED_SHARE)) : null,
+      }),
       errors: state.errors.map((e) => ({ ...e })),
     }
   }
@@ -512,17 +636,21 @@ export function codex({
   }
 }
 
-function toSession({ id, state, names, live, turnLive, fresh, locking, unread, subagents, lastActivityAt }) {
-  const calls = [...state.calls.values()]
+const childName = (state) => titleFromPrompt(state.typed ? state.firstPrompt : state.fallbackPrompt) || 'Subagent'
+
+function toSession({ id, state, names, live, turnLive, fresh, locking, unread, subagents, workingSubagents, lastActivityAt }) {
+  const calls = [...state.calls.values()].sort((a, b) => a.startedAt - b.startedAt)
+  const asked = calls.filter((c) => QUESTION_TOOLS.has(c.name))
   let status
+  let statusReason
   if (turnLive) {
-    if (calls.some((c) => QUESTION_TOOLS.has(c.name))) status = 'asking'
-    else if (calls.length ? calls.every((c) => AGENT_TOOLS.has(c.name)) : subagents > 0) status = 'delegating'
-    else status = 'working'
-  } else if (subagents) status = 'delegating'
-  else if (state.failed) status = 'error'
-  else if (unread !== undefined) status = unread ? 'done' : 'idle'
-  else status = (locking ? live : fresh) ? 'done' : 'idle'
+    if (asked.length) [status, statusReason] = ['asking', asked.some((c) => c.name === 'request_user_input') ? 'question' : 'approval']
+    else if (calls.length ? calls.every((c) => AGENT_TOOLS.has(c.name)) : subagents > 0) [status, statusReason] = ['delegating', 'subagents']
+    else [status, statusReason] = ['working', 'turn-open']
+  } else if (subagents) [status, statusReason] = ['delegating', 'subagents']
+  else if (state.failed) [status, statusReason] = ['error', 'turn-failed']
+  else if (unread !== undefined) [status, statusReason] = unread ? ['done', 'unread'] : ['idle', 'read']
+  else [status, statusReason] = (locking ? live : fresh) ? ['done', 'recent'] : ['idle', 'quiet']
 
   const tool = turnLive && (status === 'working' || status === 'delegating') ? latestCall(state.calls) : null
   const prompt = state.typed ? state.firstPrompt : state.fallbackPrompt
@@ -545,11 +673,17 @@ function toSession({ id, state, names, live, turnLive, fresh, locking, unread, s
     tool,
     tokens: state.tokens ? { ...state.tokens } : null,
     context: state.context && state.context.used ? { ...state.context } : null,
+    contextHistory: sampleList(state.contextSample, state.context?.used),
     costUsd: null,
     lines: { ...state.lines },
     plan: planSummary(state.plan),
     subagents,
+    workingSubagents,
     prompts: state.typed ? state.prompts : state.fallbackPrompts,
+    turnStartedAt: turnLive ? state.turnStartedAt || null : null,
+    statusReason,
+    ask: askOf(status, { calls: asked, errorText: state.errors.at(-1)?.message }),
+    usageLimits: state.usageLimits ? { ...state.usageLimits, windows: state.usageLimits.windows.map((w) => ({ ...w })) } : null,
   }
 }
 

package/telemetry/cursor.mjs

@@ -3,24 +3,30 @@ import os from 'node:os'
 import path from 'node:path'
 import { decodeProjectDir } from '../lib/cursor-folder.mjs'
 import { exists, listDirs, listFiles } from '../lib/fsutil.mjs'
-import { jsonlFollower } from '../lib/jsonl-follower.mjs'
+import { jsonlFollower, recentReaders } from '../lib/jsonl-follower.mjs'
 import {
   DETAIL_LIMITS,
   RECENT_MS,
   activityDetail,
+  askOf,
   clip,
   clipBlock,
   emptyActivity,
+  joinedText,
   keepNewest,
+  keepText,
   nextPlan,
+  openTool,
   planItems,
   planSummary,
   relativePath,
   startCall,
   text,
+  textList,
   titleFromPrompt,
   toolFile,
   toolInput,
+  toolNow,
   toolTarget,
 } from '../lib/telemetry-fields.mjs'
 
@@ -57,6 +63,7 @@ function emptyTranscript() {
     closed: false,
     failed: false,
     modelsTurn: false,
+    turnStartedAt: 0,
     tool: null,
     plan: null,
     activity: emptyActivity(),
@@ -80,6 +87,7 @@ function applyRecord(state, record) {
   state.closed = false
   if (record.role === 'user') {
     const prompt = userText(record)
+    if (!state.modelsTurn) state.turnStartedAt = stamp(record)
     if (prompt) {
       state.prompts++
       state.firstPrompt ||= prompt
@@ -98,7 +106,16 @@ function applyRecord(state, record) {
     startCall(state.activity, { name: text(call.name), target: toolTarget(call.input, '', Infinity), input: toolInput(call.input), ...toolFile(call.input), startedAt: 0, running: false })
   }
   state.modelsTurn = calls.length > 0
-  state.tool = calls.length ? { name: text(calls.at(-1).name), startedAt: 0 } : null
+  state.tool = calls.length ? openTool(text(calls.at(-1).name), 0, calls.at(-1).input) : null
+}
+
+const WORDS_KEPT = 3
+
+const emptyWords = () => ({ turnAt: 0, replies: [] })
+
+function applyWords(state, record) {
+  if (record.role === 'user') state.turnAt = stamp(record) || state.turnAt
+  else if (record.role === 'assistant') keepText(state.replies, 'replies', { at: state.turnAt, text: joinedText(record.message?.content, 'text') })
 }
 
 export function cursor({
@@ -107,6 +124,7 @@ export function cursor({
 } = {}) {
   const transcripts = jsonlFollower({ empty: emptyTranscript, apply: applyRecord })
   const subagentFiles = jsonlFollower({ empty: emptyTranscript, apply: applyRecord })
+  const wordsFor = recentReaders(() => jsonlFollower({ empty: emptyWords, apply: applyWords }), WORDS_KEPT)
   let subagentsSweptAt = 0
   let known = new Map()
 
@@ -142,15 +160,17 @@ export function cursor({
   }
 
   async function runningSubagents(sessionDir) {
-    if (!sessionDir) return 0
-    let count = 0
+    if (!sessionDir) return []
+    const working = []
     for (const file of await listFiles(path.join(sessionDir, 'subagents'), (n) => n.endsWith('.jsonl'))) {
       const stat = await fsp.stat(file).catch(() => null)
       if (!stat || now() - stat.mtimeMs > SUBAGENT_WINDOW_MS) continue
       const state = await follow(file, stat, subagentFiles).catch(() => null)
-      if (state && isOpen(state, true)) count++
+      if (!state || !isOpen(state, true)) continue
+      const id = path.basename(file, '.jsonl')
+      working.push({ id, name: titleFromPrompt(state.firstPrompt) || id, startedAt: state.startedAt || Math.floor(stat.birthtimeMs) || Math.floor(stat.mtimeMs) })
     }
-    return count
+    return working.sort((a, b) => a.startedAt - b.startedAt).map(({ id, name }) => ({ id, name }))
   }
 
   async function subagentList(sessionDir) {
@@ -170,7 +190,7 @@ export function cursor({
         status: open ? 'working' : state.failed ? 'error' : 'done',
         model: '',
         tokens: null,
-        tool: open && state.tool ? { ...state.tool } : null,
+        tool: open && state.tool ? toolNow(state.tool) : null,
         startedAt: state.startedAt || Math.floor(stat.birthtimeMs) || Math.floor(stat.mtimeMs),
         lastActivityAt: Math.floor(stat.mtimeMs),
       })
@@ -184,6 +204,7 @@ export function cursor({
     const stat = entry ? await fsp.stat(entry.file).catch(() => null) : null
     if (!stat) return null
     const state = await follow(entry.file, stat)
+    const said = await wordsFor(entry.file).read(entry.file, stat).catch(() => null)
     const open = isOpen(state, now() - stat.mtimeMs < ACTIVE_WINDOW_MS)
     const { activity, tools } = activityDetail(state.activity)
     for (const call of activity) {
@@ -205,6 +226,9 @@ export function cursor({
       subagents: await subagentList(entry.sessionDir),
       files: null,
       prompts: state.promptList.map((p) => ({ ...p })),
+      thoughts: null,
+      replies: textList(said?.replies),
+      contextBreakdown: null,
       errors: state.errors.map((e) => ({ ...e })),
     }
   }
@@ -221,14 +245,16 @@ export function cursor({
       }
       const fresh = now() - entry.stat.mtimeMs < ACTIVE_WINDOW_MS
       const open = isOpen(state, fresh)
-      const subagents = await runningSubagents(entry.sessionDir)
+      const workingSubagents = await runningSubagents(entry.sessionDir)
+      const subagents = workingSubagents.length
       const projectPath = await decodeProjectDir(path.basename(entry.projectDir))
       found.set(entry.id, { file: entry.file, sessionDir: entry.sessionDir, projectPath })
       let status
-      if (subagents) status = 'delegating'
-      else if (open) status = 'working'
-      else if (state.failed) status = 'error'
-      else status = fresh ? 'done' : 'idle'
+      let statusReason
+      if (subagents) [status, statusReason] = ['delegating', 'subagents']
+      else if (open) [status, statusReason] = ['working', 'turn-open']
+      else if (state.failed) [status, statusReason] = ['error', 'turn-failed']
+      else [status, statusReason] = fresh ? ['done', 'recent'] : ['idle', 'quiet']
       sessions.push({
         id: `cursor:${entry.id}`,
         harness: 'cursor',
@@ -244,14 +270,20 @@ export function cursor({
         live: open,
         startedAt: state.startedAt || Math.floor(entry.stat.birthtimeMs) || Math.floor(entry.stat.mtimeMs),
         lastActivityAt: Math.floor(entry.stat.mtimeMs),
-        tool: open && state.tool ? { ...state.tool } : null,
+        tool: open && state.tool ? { ...toolNow(state.tool), target: clip(relativePath(state.tool.target, projectPath)) } : null,
         tokens: null,
         context: null,
+        contextHistory: null,
         costUsd: null,
         lines: null,
         plan: planSummary(state.plan),
         subagents,
+        workingSubagents,
         prompts: state.prompts,
+        turnStartedAt: open ? state.turnStartedAt || null : null,
+        statusReason,
+        ask: askOf(status, { errorText: state.errors.at(-1)?.message }),
+        usageLimits: null,
       })
     }
     transcripts.sweep()