@xortex/xcode 3.0.0

package/utils/tokenBudget.ts

@@ -0,0 +1,73 @@
+// Shorthand (+500k) anchored to start/end to avoid false positives in natural language.
+// Verbose (use/spend 2M tokens) matches anywhere.
+const SHORTHAND_START_RE = /^\s*\+(\d+(?:\.\d+)?)\s*(k|m|b)\b/i
+// Lookbehind (?<=\s) is avoided — it defeats YARR JIT in JSC, and the
+// interpreter scans O(n) even with the $ anchor. Capture the whitespace
+// instead; callers offset match.index by 1 where position matters.
+const SHORTHAND_END_RE = /\s\+(\d+(?:\.\d+)?)\s*(k|m|b)\s*[.!?]?\s*$/i
+const VERBOSE_RE = /\b(?:use|spend)\s+(\d+(?:\.\d+)?)\s*(k|m|b)\s*tokens?\b/i
+const VERBOSE_RE_G = new RegExp(VERBOSE_RE.source, 'gi')
+
+const MULTIPLIERS: Record<string, number> = {
+  k: 1_000,
+  m: 1_000_000,
+  b: 1_000_000_000,
+}
+
+function parseBudgetMatch(value: string, suffix: string): number {
+  return parseFloat(value) * MULTIPLIERS[suffix.toLowerCase()]!
+}
+
+export function parseTokenBudget(text: string): number | null {
+  const startMatch = text.match(SHORTHAND_START_RE)
+  if (startMatch) return parseBudgetMatch(startMatch[1]!, startMatch[2]!)
+  const endMatch = text.match(SHORTHAND_END_RE)
+  if (endMatch) return parseBudgetMatch(endMatch[1]!, endMatch[2]!)
+  const verboseMatch = text.match(VERBOSE_RE)
+  if (verboseMatch) return parseBudgetMatch(verboseMatch[1]!, verboseMatch[2]!)
+  return null
+}
+
+export function findTokenBudgetPositions(
+  text: string,
+): Array<{ start: number; end: number }> {
+  const positions: Array<{ start: number; end: number }> = []
+  const startMatch = text.match(SHORTHAND_START_RE)
+  if (startMatch) {
+    const offset =
+      startMatch.index! +
+      startMatch[0].length -
+      startMatch[0].trimStart().length
+    positions.push({
+      start: offset,
+      end: startMatch.index! + startMatch[0].length,
+    })
+  }
+  const endMatch = text.match(SHORTHAND_END_RE)
+  if (endMatch) {
+    // Avoid double-counting when input is just "+500k"
+    const endStart = endMatch.index! + 1 // +1: regex includes leading \s
+    const alreadyCovered = positions.some(
+      p => endStart >= p.start && endStart < p.end,
+    )
+    if (!alreadyCovered) {
+      positions.push({
+        start: endStart,
+        end: endMatch.index! + endMatch[0].length,
+      })
+    }
+  }
+  for (const match of text.matchAll(VERBOSE_RE_G)) {
+    positions.push({ start: match.index, end: match.index + match[0].length })
+  }
+  return positions
+}
+
+export function getBudgetContinuationMessage(
+  pct: number,
+  turnTokens: number,
+  budget: number,
+): string {
+  const fmt = (n: number): string => new Intl.NumberFormat('en-US').format(n)
+  return `Stopped at ${pct}% of token target (${fmt(turnTokens)} / ${fmt(budget)}). Keep working \u2014 do not summarize.`
+}

package/utils/tokens.ts

@@ -0,0 +1,261 @@
+import type { BetaUsage as Usage } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
+import { roughTokenCountEstimationForMessages } from '../services/tokenEstimation.js'
+import type { AssistantMessage, Message } from '../types/message.js'
+import { SYNTHETIC_MESSAGES, SYNTHETIC_MODEL } from './messages.js'
+import { jsonStringify } from './slowOperations.js'
+
+export function getTokenUsage(message: Message): Usage | undefined {
+  if (
+    message?.type === 'assistant' &&
+    'usage' in message.message &&
+    !(
+      message.message.content[0]?.type === 'text' &&
+      SYNTHETIC_MESSAGES.has(message.message.content[0].text)
+    ) &&
+    message.message.model !== SYNTHETIC_MODEL
+  ) {
+    return message.message.usage
+  }
+  return undefined
+}
+
+/**
+ * Get the API response id for an assistant message with real (non-synthetic) usage.
+ * Used to identify split assistant records that came from the same API response —
+ * when parallel tool calls are streamed, each content block becomes a separate
+ * AssistantMessage record, but they all share the same message.id.
+ */
+function getAssistantMessageId(message: Message): string | undefined {
+  if (
+    message?.type === 'assistant' &&
+    'id' in message.message &&
+    message.message.model !== SYNTHETIC_MODEL
+  ) {
+    return message.message.id
+  }
+  return undefined
+}
+
+/**
+ * Calculate total context window tokens from an API response's usage data.
+ * Includes input_tokens + cache tokens + output_tokens.
+ *
+ * This represents the full context size at the time of that API call.
+ * Use tokenCountWithEstimation() when you need context size from messages.
+ */
+export function getTokenCountFromUsage(usage: Usage): number {
+  return (
+    usage.input_tokens +
+    (usage.cache_creation_input_tokens ?? 0) +
+    (usage.cache_read_input_tokens ?? 0) +
+    usage.output_tokens
+  )
+}
+
+export function tokenCountFromLastAPIResponse(messages: Message[]): number {
+  let i = messages.length - 1
+  while (i >= 0) {
+    const message = messages[i]
+    const usage = message ? getTokenUsage(message) : undefined
+    if (usage) {
+      return getTokenCountFromUsage(usage)
+    }
+    i--
+  }
+  return 0
+}
+
+/**
+ * Final context window size from the last API response's usage.iterations[-1].
+ * Used for task_budget.remaining computation across compaction boundaries —
+ * the server's budget countdown is context-based, so remaining decrements by
+ * the pre-compact final window, not billing spend. See monorepo
+ * api/api/sampling/prompt/renderer.py:292 for the server-side computation.
+ *
+ * Falls back to top-level input_tokens + output_tokens when iterations is
+ * absent (no server-side tool loops, so top-level usage IS the final window).
+ * Both paths exclude cache tokens to match #304930's formula.
+ */
+export function finalContextTokensFromLastResponse(
+  messages: Message[],
+): number {
+  let i = messages.length - 1
+  while (i >= 0) {
+    const message = messages[i]
+    const usage = message ? getTokenUsage(message) : undefined
+    if (usage) {
+      // Stainless types don't include iterations yet — cast like advisor.ts:43
+      const iterations = (
+        usage as {
+          iterations?: Array<{
+            input_tokens: number
+            output_tokens: number
+          }> | null
+        }
+      ).iterations
+      if (iterations && iterations.length > 0) {
+        const last = iterations.at(-1)!
+        return last.input_tokens + last.output_tokens
+      }
+      // No iterations → no server tool loop → top-level usage IS the final
+      // window. Match the iterations path's formula (input + output, no cache)
+      // rather than getTokenCountFromUsage — #304930 defines final window as
+      // non-cache input + output. Whether the server's budget countdown
+      // (renderer.py:292 calculate_context_tokens) counts cache the same way
+      // is an open question; aligning with the iterations path keeps the two
+      // branches consistent until that's resolved.
+      return usage.input_tokens + usage.output_tokens
+    }
+    i--
+  }
+  return 0
+}
+
+/**
+ * Get only the output_tokens from the last API response.
+ * This excludes input context (system prompt, tools, prior messages).
+ *
+ * WARNING: Do NOT use this for threshold comparisons (autocompact, session memory).
+ * Use tokenCountWithEstimation() instead, which measures full context size.
+ * This function is only useful for measuring how many tokens Claude generated
+ * in a single response, not how full the context window is.
+ */
+export function messageTokenCountFromLastAPIResponse(
+  messages: Message[],
+): number {
+  let i = messages.length - 1
+  while (i >= 0) {
+    const message = messages[i]
+    const usage = message ? getTokenUsage(message) : undefined
+    if (usage) {
+      return usage.output_tokens
+    }
+    i--
+  }
+  return 0
+}
+
+export function getCurrentUsage(messages: Message[]): {
+  input_tokens: number
+  output_tokens: number
+  cache_creation_input_tokens: number
+  cache_read_input_tokens: number
+} | null {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i]
+    const usage = message ? getTokenUsage(message) : undefined
+    if (usage) {
+      return {
+        input_tokens: usage.input_tokens,
+        output_tokens: usage.output_tokens,
+        cache_creation_input_tokens: usage.cache_creation_input_tokens ?? 0,
+        cache_read_input_tokens: usage.cache_read_input_tokens ?? 0,
+      }
+    }
+  }
+  return null
+}
+
+export function doesMostRecentAssistantMessageExceed200k(
+  messages: Message[],
+): boolean {
+  const THRESHOLD = 200_000
+
+  const lastAsst = messages.findLast(m => m.type === 'assistant')
+  if (!lastAsst) return false
+  const usage = getTokenUsage(lastAsst)
+  return usage ? getTokenCountFromUsage(usage) > THRESHOLD : false
+}
+
+/**
+ * Calculate the character content length of an assistant message.
+ * Used for spinner token estimation (characters / 4 ≈ tokens).
+ * This is used when subagent streaming events are filtered out and we
+ * need to count content from completed messages instead.
+ *
+ * Counts the same content that handleMessageFromStream would count via deltas:
+ * - text (text_delta)
+ * - thinking (thinking_delta)
+ * - redacted_thinking data
+ * - tool_use input (input_json_delta)
+ * Note: signature_delta is excluded from streaming counts (not model output).
+ */
+export function getAssistantMessageContentLength(
+  message: AssistantMessage,
+): number {
+  let contentLength = 0
+  for (const block of message.message.content) {
+    if (block.type === 'text') {
+      contentLength += block.text.length
+    } else if (block.type === 'thinking') {
+      contentLength += block.thinking.length
+    } else if (block.type === 'redacted_thinking') {
+      contentLength += block.data.length
+    } else if (block.type === 'tool_use') {
+      contentLength += jsonStringify(block.input).length
+    }
+  }
+  return contentLength
+}
+
+/**
+ * Get the current context window size in tokens.
+ *
+ * This is the CANONICAL function for measuring context size when checking
+ * thresholds (autocompact, session memory init, etc.). Uses the last API
+ * response's token count (input + output + cache) plus estimates for any
+ * messages added since.
+ *
+ * Always use this instead of:
+ * - Cumulative token counting (which double-counts as context grows)
+ * - messageTokenCountFromLastAPIResponse (which only counts output_tokens)
+ * - tokenCountFromLastAPIResponse (which doesn't estimate new messages)
+ *
+ * Implementation note on parallel tool calls: when the model makes multiple
+ * tool calls in one response, the streaming code emits a SEPARATE assistant
+ * record per content block (all sharing the same message.id and usage), and
+ * the query loop interleaves each tool_result immediately after its tool_use.
+ * So the messages array looks like:
+ *   [..., assistant(id=A), user(result), assistant(id=A), user(result), ...]
+ * If we stop at the LAST assistant record, we only estimate the one tool_result
+ * after it and miss all the earlier interleaved tool_results — which will ALL
+ * be in the next API request. To avoid undercounting, after finding a usage-
+ * bearing record we walk back to the FIRST sibling with the same message.id
+ * so every interleaved tool_result is included in the rough estimate.
+ */
+export function tokenCountWithEstimation(messages: readonly Message[]): number {
+  let i = messages.length - 1
+  while (i >= 0) {
+    const message = messages[i]
+    const usage = message ? getTokenUsage(message) : undefined
+    if (message && usage) {
+      // Walk back past any earlier sibling records split from the same API
+      // response (same message.id) so interleaved tool_results between them
+      // are included in the estimation slice.
+      const responseId = getAssistantMessageId(message)
+      if (responseId) {
+        let j = i - 1
+        while (j >= 0) {
+          const prior = messages[j]
+          const priorId = prior ? getAssistantMessageId(prior) : undefined
+          if (priorId === responseId) {
+            // Earlier split of the same API response — anchor here instead.
+            i = j
+          } else if (priorId !== undefined) {
+            // Hit a different API response — stop walking.
+            break
+          }
+          // priorId === undefined: a user/tool_result/attachment message,
+          // possibly interleaved between splits — keep walking.
+          j--
+        }
+      }
+      return (
+        getTokenCountFromUsage(usage) +
+        roughTokenCountEstimationForMessages(messages.slice(i + 1))
+      )
+    }
+    i--
+  }
+  return roughTokenCountEstimationForMessages(messages)
+}

package/utils/toolErrors.ts

@@ -0,0 +1,132 @@
+import type { ZodError } from 'zod/v4'
+import { AbortError, ShellError } from './errors.js'
+import { INTERRUPT_MESSAGE_FOR_TOOL_USE } from './messages.js'
+
+export function formatError(error: unknown): string {
+  if (error instanceof AbortError) {
+    return error.message || INTERRUPT_MESSAGE_FOR_TOOL_USE
+  }
+  if (!(error instanceof Error)) {
+    return String(error)
+  }
+  const parts = getErrorParts(error)
+  const fullMessage =
+    parts.filter(Boolean).join('\n').trim() || 'Command failed with no output'
+  if (fullMessage.length <= 10000) {
+    return fullMessage
+  }
+  const halfLength = 5000
+  const start = fullMessage.slice(0, halfLength)
+  const end = fullMessage.slice(-halfLength)
+  return `${start}\n\n... [${fullMessage.length - 10000} characters truncated] ...\n\n${end}`
+}
+
+export function getErrorParts(error: Error): string[] {
+  if (error instanceof ShellError) {
+    return [
+      `Exit code ${error.code}`,
+      error.interrupted ? INTERRUPT_MESSAGE_FOR_TOOL_USE : '',
+      error.stderr,
+      error.stdout,
+    ]
+  }
+  const parts = [error.message]
+  if ('stderr' in error && typeof error.stderr === 'string') {
+    parts.push(error.stderr)
+  }
+  if ('stdout' in error && typeof error.stdout === 'string') {
+    parts.push(error.stdout)
+  }
+  return parts
+}
+
+/**
+ * Formats a Zod validation path into a readable string
+ * e.g., ['todos', 0, 'activeForm'] => 'todos[0].activeForm'
+ */
+function formatValidationPath(path: PropertyKey[]): string {
+  if (path.length === 0) return ''
+
+  return path.reduce((acc, segment, index) => {
+    const segmentStr = String(segment)
+    if (typeof segment === 'number') {
+      return `${String(acc)}[${segmentStr}]`
+    }
+    return index === 0 ? segmentStr : `${String(acc)}.${segmentStr}`
+  }, '') as string
+}
+
+/**
+ * Converts Zod validation errors into a human-readable and LLM friendly error message
+ *
+ * @param toolName The name of the tool that failed validation
+ * @param error The Zod error object
+ * @returns A formatted error message string
+ */
+export function formatZodValidationError(
+  toolName: string,
+  error: ZodError,
+): string {
+  const missingParams = error.issues
+    .filter(
+      err =>
+        err.code === 'invalid_type' &&
+        err.message.includes('received undefined'),
+    )
+    .map(err => formatValidationPath(err.path))
+
+  const unexpectedParams = error.issues
+    .filter(err => err.code === 'unrecognized_keys')
+    .flatMap(err => err.keys)
+
+  const typeMismatchParams = error.issues
+    .filter(
+      err =>
+        err.code === 'invalid_type' &&
+        !err.message.includes('received undefined'),
+    )
+    .map(err => {
+      const typeErr = err as { expected: string }
+      const receivedMatch = err.message.match(/received (\w+)/)
+      const received = receivedMatch ? receivedMatch[1] : 'unknown'
+      return {
+        param: formatValidationPath(err.path),
+        expected: typeErr.expected,
+        received,
+      }
+    })
+
+  // Default to original error message if we can't create a better one
+  let errorContent = error.message
+
+  // Build a human-readable error message
+  const errorParts = []
+
+  if (missingParams.length > 0) {
+    const missingParamErrors = missingParams.map(
+      param => `The required parameter \`${param}\` is missing`,
+    )
+    errorParts.push(...missingParamErrors)
+  }
+
+  if (unexpectedParams.length > 0) {
+    const unexpectedParamErrors = unexpectedParams.map(
+      param => `An unexpected parameter \`${param}\` was provided`,
+    )
+    errorParts.push(...unexpectedParamErrors)
+  }
+
+  if (typeMismatchParams.length > 0) {
+    const typeErrors = typeMismatchParams.map(
+      ({ param, expected, received }) =>
+        `The parameter \`${param}\` type is expected as \`${expected}\` but provided as \`${received}\``,
+    )
+    errorParts.push(...typeErrors)
+  }
+
+  if (errorParts.length > 0) {
+    errorContent = `${toolName} failed due to the following ${errorParts.length > 1 ? 'issues' : 'issue'}:\n${errorParts.join('\n')}`
+  }
+
+  return errorContent
+}

package/utils/toolPool.ts

@@ -0,0 +1,79 @@
+import { feature } from 'bun:bundle'
+import partition from 'lodash-es/partition.js'
+import uniqBy from 'lodash-es/uniqBy.js'
+import { COORDINATOR_MODE_ALLOWED_TOOLS } from '../constants/tools.js'
+import { isMcpTool } from '../services/mcp/utils.js'
+import type { Tool, ToolPermissionContext, Tools } from '../Tool.js'
+
+// MCP tool name suffixes for PR activity subscription. These are lightweight
+// orchestration actions the coordinator calls directly rather than delegating
+// to workers. Matched by suffix since the MCP server name prefix may vary.
+const PR_ACTIVITY_TOOL_SUFFIXES = [
+  'subscribe_pr_activity',
+  'unsubscribe_pr_activity',
+]
+
+export function isPrActivitySubscriptionTool(name: string): boolean {
+  return PR_ACTIVITY_TOOL_SUFFIXES.some(suffix => name.endsWith(suffix))
+}
+
+// Dead code elimination: conditional imports for feature-gated modules
+/* eslint-disable @typescript-eslint/no-require-imports */
+const coordinatorModeModule = feature('COORDINATOR_MODE')
+  ? (require('../coordinator/coordinatorMode.js') as typeof import('../coordinator/coordinatorMode.js'))
+  : null
+/* eslint-enable @typescript-eslint/no-require-imports */
+
+/**
+ * Filters a tool array to the set allowed in coordinator mode.
+ * Shared between the REPL path (mergeAndFilterTools) and the headless
+ * path (main.tsx) so both stay in sync.
+ *
+ * PR activity subscription tools are always allowed since subscription
+ * management is orchestration.
+ */
+export function applyCoordinatorToolFilter(tools: Tools): Tools {
+  return tools.filter(
+    t =>
+      COORDINATOR_MODE_ALLOWED_TOOLS.has(t.name) ||
+      isPrActivitySubscriptionTool(t.name),
+  )
+}
+
+/**
+ * Pure function that merges tool pools and applies coordinator mode filtering.
+ *
+ * Lives in a React-free file so print.ts can import it without pulling
+ * react/ink into the SDK module graph. The useMergedTools hook delegates
+ * to this function inside useMemo.
+ *
+ * @param initialTools - Extra tools to include (built-in + startup MCP from props).
+ * @param assembled - Tools from assembleToolPool (built-in + MCP, deduped).
+ * @param mode - The permission context mode.
+ * @returns Merged, deduplicated, and coordinator-filtered tool array.
+ */
+export function mergeAndFilterTools(
+  initialTools: Tools,
+  assembled: Tools,
+  mode: ToolPermissionContext['mode'],
+): Tools {
+  // Merge initialTools on top - they take precedence in deduplication.
+  // initialTools may include built-in tools (from getTools() in REPL.tsx) which
+  // overlap with assembled tools. uniqBy handles this deduplication.
+  // Partition-sort for prompt-cache stability (same as assembleToolPool):
+  // built-ins must stay a contiguous prefix for the server's cache policy.
+  const [mcp, builtIn] = partition(
+    uniqBy([...initialTools, ...assembled], 'name'),
+    isMcpTool,
+  )
+  const byName = (a: Tool, b: Tool) => a.name.localeCompare(b.name)
+  const tools = [...builtIn.sort(byName), ...mcp.sort(byName)]
+
+  if (feature('COORDINATOR_MODE') && coordinatorModeModule) {
+    if (coordinatorModeModule.isCoordinatorMode()) {
+      return applyCoordinatorToolFilter(tools)
+    }
+  }
+
+  return tools
+}