@xortex/xcode 3.0.0

package/utils/headlessProfiler.ts

@@ -0,0 +1,178 @@
+/**
+ * Headless mode profiling utility for measuring per-turn latency in -p (print) mode.
+ *
+ * Tracks key timing phases per turn:
+ * - Time to system message output (turn 0 only)
+ * - Time to first query started
+ * - Time to first API response (TTFT)
+ *
+ * Uses Node.js built-in performance hooks API for standard timing measurement.
+ * Sampled logging: 100% of ant users, 5% of external users.
+ *
+ * Set CLAUDE_CODE_PROFILE_STARTUP=1 for detailed logging output.
+ */
+
+import { getIsNonInteractiveSession } from '../bootstrap/state.js'
+import {
+  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  logEvent,
+} from '../services/analytics/index.js'
+import { logForDebugging } from './debug.js'
+import { isEnvTruthy } from './envUtils.js'
+import { getPerformance } from './profilerBase.js'
+import { jsonStringify } from './slowOperations.js'
+
+// Detailed profiling mode - same env var as startupProfiler
+// eslint-disable-next-line custom-rules/no-process-env-top-level
+const DETAILED_PROFILING = isEnvTruthy(process.env.CLAUDE_CODE_PROFILE_STARTUP)
+
+// Sampling for Statsig logging: 100% ant, 5% external
+// Decision made once at module load - non-sampled users pay no profiling cost
+const STATSIG_SAMPLE_RATE = 0.05
+// eslint-disable-next-line custom-rules/no-process-env-top-level
+const STATSIG_LOGGING_SAMPLED =
+  process.env.USER_TYPE === 'ant' || Math.random() < STATSIG_SAMPLE_RATE
+
+// Enable profiling if either detailed mode OR sampled for Statsig
+const SHOULD_PROFILE = DETAILED_PROFILING || STATSIG_LOGGING_SAMPLED
+
+// Use a unique prefix to avoid conflicts with other profiler marks
+const MARK_PREFIX = 'headless_'
+
+// Track current turn number (auto-incremented by headlessProfilerStartTurn)
+let currentTurnNumber = -1
+
+/**
+ * Clear all headless profiler marks from performance timeline
+ */
+function clearHeadlessMarks(): void {
+  const perf = getPerformance()
+  const allMarks = perf.getEntriesByType('mark')
+  for (const mark of allMarks) {
+    if (mark.name.startsWith(MARK_PREFIX)) {
+      perf.clearMarks(mark.name)
+    }
+  }
+}
+
+/**
+ * Start a new turn for profiling. Clears previous marks, increments turn number,
+ * and records turn_start. Call this at the beginning of each user message processing.
+ */
+export function headlessProfilerStartTurn(): void {
+  // Only profile in headless/non-interactive mode
+  if (!getIsNonInteractiveSession()) return
+  // Only profile if enabled
+  if (!SHOULD_PROFILE) return
+
+  currentTurnNumber++
+  clearHeadlessMarks()
+
+  const perf = getPerformance()
+  perf.mark(`${MARK_PREFIX}turn_start`)
+
+  if (DETAILED_PROFILING) {
+    logForDebugging(`[headlessProfiler] Started turn ${currentTurnNumber}`)
+  }
+}
+
+/**
+ * Record a checkpoint with the given name.
+ * Only records if in headless mode and profiling is enabled.
+ */
+export function headlessProfilerCheckpoint(name: string): void {
+  // Only profile in headless/non-interactive mode
+  if (!getIsNonInteractiveSession()) return
+  // Only profile if enabled
+  if (!SHOULD_PROFILE) return
+
+  const perf = getPerformance()
+  perf.mark(`${MARK_PREFIX}${name}`)
+
+  if (DETAILED_PROFILING) {
+    logForDebugging(
+      `[headlessProfiler] Checkpoint: ${name} at ${perf.now().toFixed(1)}ms`,
+    )
+  }
+}
+
+/**
+ * Log headless latency metrics for the current turn to Statsig.
+ * Call this at the end of each turn (before processing next user message).
+ */
+export function logHeadlessProfilerTurn(): void {
+  // Only log in headless mode
+  if (!getIsNonInteractiveSession()) return
+  // Only log if enabled
+  if (!SHOULD_PROFILE) return
+
+  const perf = getPerformance()
+  const allMarks = perf.getEntriesByType('mark')
+
+  // Filter to only our headless marks
+  const marks = allMarks.filter(mark => mark.name.startsWith(MARK_PREFIX))
+  if (marks.length === 0) return
+
+  // Build checkpoint lookup (strip prefix for easier access)
+  const checkpointTimes = new Map<string, number>()
+  for (const mark of marks) {
+    const name = mark.name.slice(MARK_PREFIX.length)
+    checkpointTimes.set(name, mark.startTime)
+  }
+
+  const turnStart = checkpointTimes.get('turn_start')
+  if (turnStart === undefined) return
+
+  // Compute phase durations relative to turn_start
+  const metadata: Record<string, number | string | undefined> = {
+    turn_number: currentTurnNumber,
+  }
+
+  // Time to system message from process start (only meaningful for turn 0)
+  // Use absolute time since perf_hooks startTime is relative to process start
+  const systemMessageTime = checkpointTimes.get('system_message_yielded')
+  if (systemMessageTime !== undefined && currentTurnNumber === 0) {
+    metadata.time_to_system_message_ms = Math.round(systemMessageTime)
+  }
+
+  // Time to query start
+  const queryStartTime = checkpointTimes.get('query_started')
+  if (queryStartTime !== undefined) {
+    metadata.time_to_query_start_ms = Math.round(queryStartTime - turnStart)
+  }
+
+  // Time to first response (first chunk from API)
+  const firstChunkTime = checkpointTimes.get('first_chunk')
+  if (firstChunkTime !== undefined) {
+    metadata.time_to_first_response_ms = Math.round(firstChunkTime - turnStart)
+  }
+
+  // Query overhead (time between query start and API request sent)
+  const apiRequestTime = checkpointTimes.get('api_request_sent')
+  if (queryStartTime !== undefined && apiRequestTime !== undefined) {
+    metadata.query_overhead_ms = Math.round(apiRequestTime - queryStartTime)
+  }
+
+  // Add checkpoint count for debugging
+  metadata.checkpoint_count = marks.length
+
+  // Add entrypoint for segmentation (sdk-ts, sdk-py, sdk-cli, or undefined)
+  if (process.env.CLAUDE_CODE_ENTRYPOINT) {
+    metadata.entrypoint = process.env.CLAUDE_CODE_ENTRYPOINT
+  }
+
+  // Log to Statsig if sampled
+  if (STATSIG_LOGGING_SAMPLED) {
+    logEvent(
+      'tengu_headless_latency',
+      metadata as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    )
+  }
+
+  // Log detailed output if CLAUDE_CODE_PROFILE_STARTUP=1
+  if (DETAILED_PROFILING) {
+    logForDebugging(
+      `[headlessProfiler] Turn ${currentTurnNumber} metrics: ${jsonStringify(metadata)}`,
+    )
+  }
+}

package/utils/heapDumpService.ts

@@ -0,0 +1,303 @@
+/**
+ * Service for heap dump capture.
+ * Used by the /heapdump command.
+ */
+
+import { createWriteStream, writeFileSync } from 'fs'
+import { readdir, readFile, writeFile } from 'fs/promises'
+import { join } from 'path'
+import { pipeline } from 'stream/promises'
+import {
+  getHeapSnapshot,
+  getHeapSpaceStatistics,
+  getHeapStatistics,
+  type HeapSpaceInfo,
+} from 'v8'
+import { getSessionId } from '../bootstrap/state.js'
+import { logEvent } from '../services/analytics/index.js'
+import { logForDebugging } from './debug.js'
+import { toError } from './errors.js'
+import { getDesktopPath } from './file.js'
+import { getFsImplementation } from './fsOperations.js'
+import { logError } from './log.js'
+import { jsonStringify } from './slowOperations.js'
+
+export type HeapDumpResult = {
+  success: boolean
+  heapPath?: string
+  diagPath?: string
+  error?: string
+}
+
+/**
+ * Memory diagnostics captured alongside heap dump.
+ * Helps identify if leak is in V8 heap (captured in snapshot) or native memory (not captured).
+ */
+export type MemoryDiagnostics = {
+  timestamp: string
+  sessionId: string
+  trigger: 'manual' | 'auto-1.5GB'
+  dumpNumber: number // 1st, 2nd, etc. auto dump in this session (0 for manual)
+  uptimeSeconds: number
+  memoryUsage: {
+    heapUsed: number
+    heapTotal: number
+    external: number
+    arrayBuffers: number
+    rss: number
+  }
+  memoryGrowthRate: {
+    bytesPerSecond: number
+    mbPerHour: number
+  }
+  v8HeapStats: {
+    heapSizeLimit: number // Max heap size allowed
+    mallocedMemory: number // Memory allocated outside V8 heap
+    peakMallocedMemory: number // Peak native memory
+    detachedContexts: number // Leaked contexts - key leak indicator!
+    nativeContexts: number // Active contexts
+  }
+  v8HeapSpaces?: Array<{
+    name: string
+    size: number
+    used: number
+    available: number
+  }>
+  resourceUsage: {
+    maxRSS: number // Peak RSS in bytes
+    userCPUTime: number
+    systemCPUTime: number
+  }
+  activeHandles: number // Leaked timers, sockets, file handles
+  activeRequests: number // Pending async operations
+  openFileDescriptors?: number // Linux/macOS - indicates resource leaks
+  analysis: {
+    potentialLeaks: string[]
+    recommendation: string
+  }
+  smapsRollup?: string // Linux only - detailed memory breakdown
+  platform: string
+  nodeVersion: string
+  ccVersion: string
+}
+
+/**
+ * Capture memory diagnostics.
+ * This helps identify if the leak is in V8 heap (captured) or native memory (not captured).
+ */
+export async function captureMemoryDiagnostics(
+  trigger: 'manual' | 'auto-1.5GB',
+  dumpNumber = 0,
+): Promise<MemoryDiagnostics> {
+  const usage = process.memoryUsage()
+  const heapStats = getHeapStatistics()
+  const resourceUsage = process.resourceUsage()
+  const uptimeSeconds = process.uptime()
+
+  // getHeapSpaceStatistics() is not available in Bun
+  let heapSpaceStats: HeapSpaceInfo[] | undefined
+  try {
+    heapSpaceStats = getHeapSpaceStatistics()
+  } catch {
+    // Not available in Bun runtime
+  }
+
+  // Get active handles/requests count (these are internal APIs but stable)
+  const activeHandles = (
+    process as unknown as { _getActiveHandles: () => unknown[] }
+  )._getActiveHandles().length
+  const activeRequests = (
+    process as unknown as { _getActiveRequests: () => unknown[] }
+  )._getActiveRequests().length
+
+  // Try to count open file descriptors (Linux/macOS)
+  let openFileDescriptors: number | undefined
+  try {
+    openFileDescriptors = (await readdir('/proc/self/fd')).length
+  } catch {
+    // Not on Linux - try macOS approach would require lsof, skip for now
+  }
+
+  // Try to read Linux smaps_rollup for detailed memory breakdown
+  let smapsRollup: string | undefined
+  try {
+    smapsRollup = await readFile('/proc/self/smaps_rollup', 'utf8')
+  } catch {
+    // Not on Linux or no access - this is fine
+  }
+
+  // Calculate native memory (RSS - heap) and growth rate
+  const nativeMemory = usage.rss - usage.heapUsed
+  const bytesPerSecond = uptimeSeconds > 0 ? usage.rss / uptimeSeconds : 0
+  const mbPerHour = (bytesPerSecond * 3600) / (1024 * 1024)
+
+  // Identify potential leaks
+  const potentialLeaks: string[] = []
+  if (heapStats.number_of_detached_contexts > 0) {
+    potentialLeaks.push(
+      `${heapStats.number_of_detached_contexts} detached context(s) - possible iframe/context leak`,
+    )
+  }
+  if (activeHandles > 100) {
+    potentialLeaks.push(
+      `${activeHandles} active handles - possible timer/socket leak`,
+    )
+  }
+  if (nativeMemory > usage.heapUsed) {
+    potentialLeaks.push(
+      'Native memory > heap - leak may be in native addons (node-pty, sharp, etc.)',
+    )
+  }
+  if (mbPerHour > 100) {
+    potentialLeaks.push(
+      `High memory growth rate: ${mbPerHour.toFixed(1)} MB/hour`,
+    )
+  }
+  if (openFileDescriptors && openFileDescriptors > 500) {
+    potentialLeaks.push(
+      `${openFileDescriptors} open file descriptors - possible file/socket leak`,
+    )
+  }
+
+  return {
+    timestamp: new Date().toISOString(),
+    sessionId: getSessionId(),
+    trigger,
+    dumpNumber,
+    uptimeSeconds,
+    memoryUsage: {
+      heapUsed: usage.heapUsed,
+      heapTotal: usage.heapTotal,
+      external: usage.external,
+      arrayBuffers: usage.arrayBuffers,
+      rss: usage.rss,
+    },
+    memoryGrowthRate: {
+      bytesPerSecond,
+      mbPerHour,
+    },
+    v8HeapStats: {
+      heapSizeLimit: heapStats.heap_size_limit,
+      mallocedMemory: heapStats.malloced_memory,
+      peakMallocedMemory: heapStats.peak_malloced_memory,
+      detachedContexts: heapStats.number_of_detached_contexts,
+      nativeContexts: heapStats.number_of_native_contexts,
+    },
+    v8HeapSpaces: heapSpaceStats?.map(space => ({
+      name: space.space_name,
+      size: space.space_size,
+      used: space.space_used_size,
+      available: space.space_available_size,
+    })),
+    resourceUsage: {
+      maxRSS: resourceUsage.maxRSS * 1024, // Convert KB to bytes
+      userCPUTime: resourceUsage.userCPUTime,
+      systemCPUTime: resourceUsage.systemCPUTime,
+    },
+    activeHandles,
+    activeRequests,
+    openFileDescriptors,
+    analysis: {
+      potentialLeaks,
+      recommendation:
+        potentialLeaks.length > 0
+          ? `WARNING: ${potentialLeaks.length} potential leak indicator(s) found. See potentialLeaks array.`
+          : 'No obvious leak indicators. Check heap snapshot for retained objects.',
+    },
+    smapsRollup,
+    platform: process.platform,
+    nodeVersion: process.version,
+    ccVersion: MACRO.VERSION,
+  }
+}
+
+/**
+ * Core heap dump function — captures heap snapshot + diagnostics to ~/Desktop.
+ *
+ * Diagnostics are written BEFORE the heap snapshot is captured, because the
+ * V8 heap snapshot serialization can crash for very large heaps. By writing
+ * diagnostics first, we still get useful memory info even if the snapshot fails.
+ */
+export async function performHeapDump(
+  trigger: 'manual' | 'auto-1.5GB' = 'manual',
+  dumpNumber = 0,
+): Promise<HeapDumpResult> {
+  try {
+    const sessionId = getSessionId()
+
+    // Capture diagnostics before any other async I/O —
+    // the heap dump itself allocates memory and would skew the numbers.
+    const diagnostics = await captureMemoryDiagnostics(trigger, dumpNumber)
+
+    const toGB = (bytes: number): string =>
+      (bytes / 1024 / 1024 / 1024).toFixed(3)
+    logForDebugging(`[HeapDump] Memory state:
+  heapUsed: ${toGB(diagnostics.memoryUsage.heapUsed)} GB (in snapshot)
+  external: ${toGB(diagnostics.memoryUsage.external)} GB (NOT in snapshot)
+  rss: ${toGB(diagnostics.memoryUsage.rss)} GB (total process)
+  ${diagnostics.analysis.recommendation}`)
+
+    const dumpDir = getDesktopPath()
+    await getFsImplementation().mkdir(dumpDir)
+
+    const suffix = dumpNumber > 0 ? `-dump${dumpNumber}` : ''
+    const heapFilename = `${sessionId}${suffix}.heapsnapshot`
+    const diagFilename = `${sessionId}${suffix}-diagnostics.json`
+    const heapPath = join(dumpDir, heapFilename)
+    const diagPath = join(dumpDir, diagFilename)
+
+    // Write diagnostics first (cheap, unlikely to fail)
+    await writeFile(diagPath, jsonStringify(diagnostics, null, 2), {
+      mode: 0o600,
+    })
+    logForDebugging(`[HeapDump] Diagnostics written to ${diagPath}`)
+
+    // Write heap snapshot (this can crash for very large heaps)
+    await writeHeapSnapshot(heapPath)
+    logForDebugging(`[HeapDump] Heap dump written to ${heapPath}`)
+
+    logEvent('tengu_heap_dump', {
+      triggerManual: trigger === 'manual',
+      triggerAuto15GB: trigger === 'auto-1.5GB',
+      dumpNumber,
+      success: true,
+    })
+
+    return { success: true, heapPath, diagPath }
+  } catch (err) {
+    const error = toError(err)
+    logError(error)
+    logEvent('tengu_heap_dump', {
+      triggerManual: trigger === 'manual',
+      triggerAuto15GB: trigger === 'auto-1.5GB',
+      dumpNumber,
+      success: false,
+    })
+    return { success: false, error: error.message }
+  }
+}
+
+/**
+ * Write heap snapshot to a file.
+ * Uses pipeline() which handles stream cleanup automatically on errors.
+ */
+async function writeHeapSnapshot(filepath: string): Promise<void> {
+  if (typeof Bun !== 'undefined') {
+    // In Bun, heapsnapshots are currently not streaming.
+    // Use synchronous I/O despite potentially large filesize so that we avoid cloning the string for cross-thread usage.
+    //
+    /* eslint-disable custom-rules/no-sync-fs -- intentionally sync to avoid cloning large heap snapshot string for cross-thread usage */
+    // @ts-expect-error 2nd argument is in the next version of Bun
+    writeFileSync(filepath, Bun.generateHeapSnapshot('v8', 'arraybuffer'), {
+      mode: 0o600,
+    })
+    /* eslint-enable custom-rules/no-sync-fs */
+
+    // Force GC to try to free that heap snapshot sooner.
+    Bun.gc(true)
+    return
+  }
+  const writeStream = createWriteStream(filepath, { mode: 0o600 })
+  const heapSnapshotStream = getHeapSnapshot()
+  await pipeline(heapSnapshotStream, writeStream)
+}

package/utils/heatmap.ts

@@ -0,0 +1,198 @@
+import chalk from 'chalk'
+import type { DailyActivity } from './stats.js'
+import { toDateString } from './statsCache.js'
+
+export type HeatmapOptions = {
+  terminalWidth?: number // Terminal width in characters
+  showMonthLabels?: boolean
+}
+
+type Percentiles = {
+  p25: number
+  p50: number
+  p75: number
+}
+
+/**
+ * Pre-calculates percentiles from activity data for use in intensity calculations
+ */
+function calculatePercentiles(
+  dailyActivity: DailyActivity[],
+): Percentiles | null {
+  const counts = dailyActivity
+    .map(a => a.messageCount)
+    .filter(c => c > 0)
+    .sort((a, b) => a - b)
+
+  if (counts.length === 0) return null
+
+  return {
+    p25: counts[Math.floor(counts.length * 0.25)]!,
+    p50: counts[Math.floor(counts.length * 0.5)]!,
+    p75: counts[Math.floor(counts.length * 0.75)]!,
+  }
+}
+
+/**
+ * Generates a GitHub-style activity heatmap for the terminal
+ */
+export function generateHeatmap(
+  dailyActivity: DailyActivity[],
+  options: HeatmapOptions = {},
+): string {
+  const { terminalWidth = 80, showMonthLabels = true } = options
+
+  // Day labels take 4 characters ("Mon "), calculate weeks that fit
+  // Cap at 52 weeks (1 year) to match GitHub style
+  const dayLabelWidth = 4
+  const availableWidth = terminalWidth - dayLabelWidth
+  const width = Math.min(52, Math.max(10, availableWidth))
+
+  // Build activity map by date
+  const activityMap = new Map<string, DailyActivity>()
+  for (const activity of dailyActivity) {
+    activityMap.set(activity.date, activity)
+  }
+
+  // Pre-calculate percentiles once for all intensity lookups
+  const percentiles = calculatePercentiles(dailyActivity)
+
+  // Calculate date range - end at today, go back N weeks
+  const today = new Date()
+  today.setHours(0, 0, 0, 0)
+
+  // Find the Sunday of the current week (start of the week containing today)
+  const currentWeekStart = new Date(today)
+  currentWeekStart.setDate(today.getDate() - today.getDay())
+
+  // Go back (width - 1) weeks from the current week start
+  const startDate = new Date(currentWeekStart)
+  startDate.setDate(startDate.getDate() - (width - 1) * 7)
+
+  // Generate grid (7 rows for days of week, width columns for weeks)
+  // Also track which week each month starts for labels
+  const grid: string[][] = Array.from({ length: 7 }, () =>
+    Array(width).fill(''),
+  )
+  const monthStarts: { month: number; week: number }[] = []
+  let lastMonth = -1
+
+  const currentDate = new Date(startDate)
+  for (let week = 0; week < width; week++) {
+    for (let day = 0; day < 7; day++) {
+      // Don't show future dates
+      if (currentDate > today) {
+        grid[day]![week] = ' '
+        currentDate.setDate(currentDate.getDate() + 1)
+        continue
+      }
+
+      const dateStr = toDateString(currentDate)
+      const activity = activityMap.get(dateStr)
+
+      // Track month changes (on day 0 = Sunday of each week)
+      if (day === 0) {
+        const month = currentDate.getMonth()
+        if (month !== lastMonth) {
+          monthStarts.push({ month, week })
+          lastMonth = month
+        }
+      }
+
+      // Determine intensity level based on message count
+      const intensity = getIntensity(activity?.messageCount || 0, percentiles)
+      grid[day]![week] = getHeatmapChar(intensity)
+
+      currentDate.setDate(currentDate.getDate() + 1)
+    }
+  }
+
+  // Build output
+  const lines: string[] = []
+
+  // Month labels - evenly spaced across the grid
+  if (showMonthLabels) {
+    const monthNames = [
+      'Jan',
+      'Feb',
+      'Mar',
+      'Apr',
+      'May',
+      'Jun',
+      'Jul',
+      'Aug',
+      'Sep',
+      'Oct',
+      'Nov',
+      'Dec',
+    ]
+
+    // Build label line with fixed-width month labels
+    const uniqueMonths = monthStarts.map(m => m.month)
+    const labelWidth = Math.floor(width / Math.max(uniqueMonths.length, 1))
+    const monthLabels = uniqueMonths
+      .map(month => monthNames[month]!.padEnd(labelWidth))
+      .join('')
+
+    // 4 spaces for day label column prefix
+    lines.push('    ' + monthLabels)
+  }
+
+  // Day labels
+  const dayLabels = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']
+
+  // Grid
+  for (let day = 0; day < 7; day++) {
+    // Only show labels for Mon, Wed, Fri
+    const label = [1, 3, 5].includes(day) ? dayLabels[day]!.padEnd(3) : '   '
+    const row = label + ' ' + grid[day]!.join('')
+    lines.push(row)
+  }
+
+  // Legend
+  lines.push('')
+  lines.push(
+    '    Less ' +
+      [
+        claudeOrange('░'),
+        claudeOrange('▒'),
+        claudeOrange('▓'),
+        claudeOrange('█'),
+      ].join(' ') +
+      ' More',
+  )
+
+  return lines.join('\n')
+}
+
+function getIntensity(
+  messageCount: number,
+  percentiles: Percentiles | null,
+): number {
+  if (messageCount === 0 || !percentiles) return 0
+
+  if (messageCount >= percentiles.p75) return 4
+  if (messageCount >= percentiles.p50) return 3
+  if (messageCount >= percentiles.p25) return 2
+  return 1
+}
+
+// Claude orange color (hex #da7756)
+const claudeOrange = chalk.hex('#da7756')
+
+function getHeatmapChar(intensity: number): string {
+  switch (intensity) {
+    case 0:
+      return chalk.gray('·')
+    case 1:
+      return claudeOrange('░')
+    case 2:
+      return claudeOrange('▒')
+    case 3:
+      return claudeOrange('▓')
+    case 4:
+      return claudeOrange('█')
+    default:
+      return chalk.gray('·')
+  }
+}