@xortex/xcode 3.0.0

package/utils/fileOperationAnalytics.ts

@@ -0,0 +1,71 @@
+import { createHash } from 'crypto'
+import type { AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS } from 'src/services/analytics/index.js'
+import { logEvent } from 'src/services/analytics/index.js'
+
+/**
+ * Creates a truncated SHA256 hash (16 chars) for file paths
+ * Used for privacy-preserving analytics on file operations
+ */
+function hashFilePath(
+  filePath: string,
+): AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS {
+  return createHash('sha256')
+    .update(filePath)
+    .digest('hex')
+    .slice(0, 16) as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
+}
+
+/**
+ * Creates a full SHA256 hash (64 chars) for file contents
+ * Used for deduplication and change detection analytics
+ */
+function hashFileContent(
+  content: string,
+): AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS {
+  return createHash('sha256')
+    .update(content)
+    .digest('hex') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
+}
+
+// Maximum content size to hash (100KB)
+// Prevents memory exhaustion when hashing large files (e.g., base64-encoded images)
+const MAX_CONTENT_HASH_SIZE = 100 * 1024
+
+/**
+ * Logs file operation analytics to Statsig
+ */
+export function logFileOperation(params: {
+  operation: 'read' | 'write' | 'edit'
+  tool: 'FileReadTool' | 'FileWriteTool' | 'FileEditTool'
+  filePath: string
+  content?: string
+  type?: 'create' | 'update'
+}): void {
+  const metadata: Record<
+    string,
+    | AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
+    | number
+    | boolean
+  > = {
+    operation:
+      params.operation as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    tool: params.tool as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    filePathHash: hashFilePath(params.filePath),
+  }
+
+  // Only hash content if it's provided and below size limit
+  // This prevents memory exhaustion from hashing large files (e.g., base64-encoded images)
+  if (
+    params.content !== undefined &&
+    params.content.length <= MAX_CONTENT_HASH_SIZE
+  ) {
+    metadata.contentHash = hashFileContent(params.content)
+  }
+
+  if (params.type !== undefined) {
+    metadata.type =
+      params.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
+  }
+
+  logEvent('tengu_file_operation', metadata)
+}

package/utils/filePersistence/filePersistence.ts

@@ -0,0 +1,287 @@
+/**
+ * File persistence orchestrator
+ *
+ * This module provides the main orchestration logic for persisting files
+ * at the end of each turn:
+ * - BYOC mode: Upload files to Files API and collect file IDs
+ * - 1P/Cloud mode: Query Files API listDirectory for file IDs (rclone handles sync)
+ */
+
+import { feature } from 'bun:bundle'
+import { join, relative } from 'path'
+import {
+  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  logEvent,
+} from '../../services/analytics/index.js'
+import {
+  type FilesApiConfig,
+  uploadSessionFiles,
+} from '../../services/api/filesApi.js'
+import { getCwd } from '../cwd.js'
+import { errorMessage } from '../errors.js'
+import { logError } from '../log.js'
+import { getSessionIngressAuthToken } from '../sessionIngressAuth.js'
+import {
+  findModifiedFiles,
+  getEnvironmentKind,
+  logDebug,
+} from './outputsScanner.js'
+import {
+  DEFAULT_UPLOAD_CONCURRENCY,
+  type FailedPersistence,
+  FILE_COUNT_LIMIT,
+  type FilesPersistedEventData,
+  OUTPUTS_SUBDIR,
+  type PersistedFile,
+  type TurnStartTime,
+} from './types.js'
+
+/**
+ * Execute file persistence for modified files in the outputs directory.
+ *
+ * Assembles all config internally:
+ * - Checks environment kind (CLAUDE_CODE_ENVIRONMENT_KIND)
+ * - Retrieves session access token
+ * - Requires CLAUDE_CODE_REMOTE_SESSION_ID for session ID
+ *
+ * @param turnStartTime - The timestamp when the turn started
+ * @param signal - Optional abort signal for cancellation
+ * @returns Event data, or null if not enabled or no files to persist
+ */
+export async function runFilePersistence(
+  turnStartTime: TurnStartTime,
+  signal?: AbortSignal,
+): Promise<FilesPersistedEventData | null> {
+  const environmentKind = getEnvironmentKind()
+  if (environmentKind !== 'byoc') {
+    return null
+  }
+
+  const sessionAccessToken = getSessionIngressAuthToken()
+  if (!sessionAccessToken) {
+    return null
+  }
+
+  const sessionId = process.env.CLAUDE_CODE_REMOTE_SESSION_ID
+  if (!sessionId) {
+    logError(
+      new Error(
+        'File persistence enabled but CLAUDE_CODE_REMOTE_SESSION_ID is not set',
+      ),
+    )
+    return null
+  }
+
+  const config: FilesApiConfig = {
+    oauthToken: sessionAccessToken,
+    sessionId,
+  }
+
+  const outputsDir = join(getCwd(), sessionId, OUTPUTS_SUBDIR)
+
+  // Check if aborted
+  if (signal?.aborted) {
+    logDebug('Persistence aborted before processing')
+    return null
+  }
+
+  const startTime = Date.now()
+  logEvent('tengu_file_persistence_started', {
+    mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  })
+
+  try {
+    let result: FilesPersistedEventData
+    if (environmentKind === 'byoc') {
+      result = await executeBYOCPersistence(
+        turnStartTime,
+        config,
+        outputsDir,
+        signal,
+      )
+    } else {
+      result = await executeCloudPersistence()
+    }
+
+    // Nothing to report
+    if (result.files.length === 0 && result.failed.length === 0) {
+      return null
+    }
+
+    const durationMs = Date.now() - startTime
+    logEvent('tengu_file_persistence_completed', {
+      success_count: result.files.length,
+      failure_count: result.failed.length,
+      duration_ms: durationMs,
+      mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    })
+
+    return result
+  } catch (error) {
+    logError(error)
+    logDebug(`File persistence failed: ${error}`)
+
+    const durationMs = Date.now() - startTime
+    logEvent('tengu_file_persistence_completed', {
+      success_count: 0,
+      failure_count: 0,
+      duration_ms: durationMs,
+      mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+      error:
+        'exception' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    })
+
+    return {
+      files: [],
+      failed: [
+        {
+          filename: outputsDir,
+          error: errorMessage(error),
+        },
+      ],
+    }
+  }
+}
+
+/**
+ * Execute BYOC mode persistence: scan local filesystem for modified files,
+ * then upload to Files API.
+ */
+async function executeBYOCPersistence(
+  turnStartTime: TurnStartTime,
+  config: FilesApiConfig,
+  outputsDir: string,
+  signal?: AbortSignal,
+): Promise<FilesPersistedEventData> {
+  // Find modified files via local filesystem scan
+  // Uses same directory structure as downloads: {cwd}/{sessionId}/outputs
+  const modifiedFiles = await findModifiedFiles(turnStartTime, outputsDir)
+
+  if (modifiedFiles.length === 0) {
+    logDebug('No modified files to persist')
+    return { files: [], failed: [] }
+  }
+
+  logDebug(`Found ${modifiedFiles.length} modified files`)
+
+  if (signal?.aborted) {
+    return { files: [], failed: [] }
+  }
+
+  // Enforce file count limit
+  if (modifiedFiles.length > FILE_COUNT_LIMIT) {
+    logDebug(
+      `File count limit exceeded: ${modifiedFiles.length} > ${FILE_COUNT_LIMIT}`,
+    )
+    logEvent('tengu_file_persistence_limit_exceeded', {
+      file_count: modifiedFiles.length,
+      limit: FILE_COUNT_LIMIT,
+    })
+    return {
+      files: [],
+      failed: [
+        {
+          filename: outputsDir,
+          error: `Too many files modified (${modifiedFiles.length}). Maximum: ${FILE_COUNT_LIMIT}.`,
+        },
+      ],
+    }
+  }
+
+  const filesToProcess = modifiedFiles
+    .map(filePath => ({
+      path: filePath,
+      relativePath: relative(outputsDir, filePath),
+    }))
+    .filter(({ relativePath }) => {
+      // Security: skip files that resolve outside the outputs directory
+      if (relativePath.startsWith('..')) {
+        logDebug(`Skipping file outside outputs directory: ${relativePath}`)
+        return false
+      }
+      return true
+    })
+
+  logDebug(`BYOC mode: uploading ${filesToProcess.length} files`)
+
+  // Upload files in parallel
+  const results = await uploadSessionFiles(
+    filesToProcess,
+    config,
+    DEFAULT_UPLOAD_CONCURRENCY,
+  )
+
+  // Separate successful and failed uploads
+  const persistedFiles: PersistedFile[] = []
+  const failedFiles: FailedPersistence[] = []
+
+  for (const result of results) {
+    if (result.success) {
+      persistedFiles.push({
+        filename: result.path,
+        file_id: result.fileId,
+      })
+    } else {
+      failedFiles.push({
+        filename: result.path,
+        error: result.error,
+      })
+    }
+  }
+
+  logDebug(
+    `BYOC persistence complete: ${persistedFiles.length} uploaded, ${failedFiles.length} failed`,
+  )
+
+  return {
+    files: persistedFiles,
+    failed: failedFiles,
+  }
+}
+
+/**
+ * Execute Cloud (1P) mode persistence.
+ * TODO: Read file_id from xattr on output files. xattr-based file IDs are
+ * currently being added for 1P environments.
+ */
+function executeCloudPersistence(): FilesPersistedEventData {
+  logDebug('Cloud mode: xattr-based file ID reading not yet implemented')
+  return { files: [], failed: [] }
+}
+
+/**
+ * Execute file persistence and emit result via callback.
+ * Handles errors internally.
+ */
+export async function executeFilePersistence(
+  turnStartTime: TurnStartTime,
+  signal: AbortSignal,
+  onResult: (result: FilesPersistedEventData) => void,
+): Promise<void> {
+  try {
+    const result = await runFilePersistence(turnStartTime, signal)
+    if (result) {
+      onResult(result)
+    }
+  } catch (error) {
+    logError(error)
+  }
+}
+
+/**
+ * Check if file persistence is enabled.
+ * Requires: feature flag ON, valid environment kind, session access token,
+ * and CLAUDE_CODE_REMOTE_SESSION_ID.
+ * This ensures only public-api/sessions users trigger file persistence,
+ * not normal Claude Code CLI users.
+ */
+export function isFilePersistenceEnabled(): boolean {
+  if (feature('FILE_PERSISTENCE')) {
+    return (
+      getEnvironmentKind() === 'byoc' &&
+      !!getSessionIngressAuthToken() &&
+      !!process.env.CLAUDE_CODE_REMOTE_SESSION_ID
+    )
+  }
+  return false
+}

package/utils/filePersistence/outputsScanner.ts

@@ -0,0 +1,126 @@
+/**
+ * Outputs directory scanner for file persistence
+ *
+ * This module provides utilities to:
+ * - Detect the session type from environment variables
+ * - Capture turn start timestamp
+ * - Find modified files by comparing file mtimes against turn start time
+ */
+
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { logForDebugging } from '../debug.js'
+import type { EnvironmentKind } from '../teleport/environments.js'
+import type { TurnStartTime } from './types.js'
+
+/** Shared debug logger for file persistence modules */
+export function logDebug(message: string): void {
+  logForDebugging(`[file-persistence] ${message}`)
+}
+
+/**
+ * Get the environment kind from CLAUDE_CODE_ENVIRONMENT_KIND.
+ * Returns null if not set or not a recognized value.
+ */
+export function getEnvironmentKind(): EnvironmentKind | null {
+  const kind = process.env.CLAUDE_CODE_ENVIRONMENT_KIND
+  if (kind === 'byoc' || kind === 'anthropic_cloud') {
+    return kind
+  }
+  return null
+}
+
+function hasParentPath(
+  entry: object,
+): entry is { parentPath: string; name: string } {
+  return 'parentPath' in entry && typeof entry.parentPath === 'string'
+}
+
+function hasPath(entry: object): entry is { path: string; name: string } {
+  return 'path' in entry && typeof entry.path === 'string'
+}
+
+function getEntryParentPath(entry: object, fallback: string): string {
+  if (hasParentPath(entry)) {
+    return entry.parentPath
+  }
+  if (hasPath(entry)) {
+    return entry.path
+  }
+  return fallback
+}
+
+/**
+ * Find files that have been modified since the turn started.
+ * Returns paths of files with mtime >= turnStartTime.
+ *
+ * Uses recursive directory listing and parallelized stat calls for efficiency.
+ *
+ * @param turnStartTime - The timestamp when the turn started
+ * @param outputsDir - The directory to scan for modified files
+ */
+export async function findModifiedFiles(
+  turnStartTime: TurnStartTime,
+  outputsDir: string,
+): Promise<string[]> {
+  // Use recursive flag to get all entries in one call
+  let entries: Awaited<ReturnType<typeof fs.readdir>>
+  try {
+    entries = await fs.readdir(outputsDir, {
+      withFileTypes: true,
+      recursive: true,
+    })
+  } catch {
+    // Directory doesn't exist or is not accessible
+    return []
+  }
+
+  // Filter to regular files only (skip symlinks for security) and build full paths
+  const filePaths: string[] = []
+  for (const entry of entries) {
+    if (entry.isSymbolicLink()) {
+      continue
+    }
+    if (entry.isFile()) {
+      // entry.parentPath is available in Node 20+, fallback to entry.path for older versions
+      const parentPath = getEntryParentPath(entry, outputsDir)
+      filePaths.push(path.join(parentPath, entry.name))
+    }
+  }
+
+  if (filePaths.length === 0) {
+    logDebug('No files found in outputs directory')
+    return []
+  }
+
+  // Parallelize stat calls for all files
+  const statResults = await Promise.all(
+    filePaths.map(async filePath => {
+      try {
+        const stat = await fs.lstat(filePath)
+        // Skip if it became a symlink between readdir and stat (race condition)
+        if (stat.isSymbolicLink()) {
+          return null
+        }
+        return { filePath, mtimeMs: stat.mtimeMs }
+      } catch {
+        // File may have been deleted between readdir and stat
+        return null
+      }
+    }),
+  )
+
+  // Filter to files modified since turn start
+  const modifiedFiles: string[] = []
+  for (const result of statResults) {
+    if (result && result.mtimeMs >= turnStartTime) {
+      modifiedFiles.push(result.filePath)
+    }
+  }
+
+  logDebug(
+    `Found ${modifiedFiles.length} modified files since turn start (scanned ${filePaths.length} total)`,
+  )
+
+  return modifiedFiles
+}

package/utils/fileRead.ts

@@ -0,0 +1,102 @@
+/**
+ * Sync file-read path, extracted from file.ts.
+ *
+ * file.ts sits in the settings SCC via log.ts → types/logs.ts → types/message.ts →
+ * Tool.ts → commands.ts → … Anything that needs readFileSync from file.ts
+ * pulls in the whole chain. This leaf imports only fsOperations and debug,
+ * both of which terminate in Node builtins.
+ *
+ * detectFileEncoding/detectLineEndings stay in file.ts — they call logError
+ * (log.ts → SCC) on unexpected failures. The -ForResolvedPath/-ForString
+ * helpers here are the pure parts; callers who need the logging wrappers
+ * import from file.ts.
+ */
+
+import { logForDebugging } from './debug.js'
+import { getFsImplementation, safeResolvePath } from './fsOperations.js'
+
+export type LineEndingType = 'CRLF' | 'LF'
+
+export function detectEncodingForResolvedPath(
+  resolvedPath: string,
+): BufferEncoding {
+  const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, {
+    length: 4096,
+  })
+
+  // Empty files should default to utf8, not ascii
+  // This fixes a bug where writing emojis/CJK to empty files caused corruption
+  if (bytesRead === 0) {
+    return 'utf8'
+  }
+
+  if (bytesRead >= 2) {
+    if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf16le'
+  }
+
+  if (
+    bytesRead >= 3 &&
+    buffer[0] === 0xef &&
+    buffer[1] === 0xbb &&
+    buffer[2] === 0xbf
+  ) {
+    return 'utf8'
+  }
+
+  // For non-empty files, default to utf8 since it's a superset of ascii
+  // and handles all Unicode characters properly
+  return 'utf8'
+}
+
+export function detectLineEndingsForString(content: string): LineEndingType {
+  let crlfCount = 0
+  let lfCount = 0
+
+  for (let i = 0; i < content.length; i++) {
+    if (content[i] === '\n') {
+      if (i > 0 && content[i - 1] === '\r') {
+        crlfCount++
+      } else {
+        lfCount++
+      }
+    }
+  }
+
+  return crlfCount > lfCount ? 'CRLF' : 'LF'
+}
+
+/**
+ * Like readFileSync but also returns the detected encoding and original line
+ * ending style in one filesystem pass. Callers writing the file back (e.g.
+ * FileEditTool) can reuse these instead of calling detectFileEncoding /
+ * detectLineEndings separately, which would each redo safeResolvePath +
+ * readSync(4KB).
+ */
+export function readFileSyncWithMetadata(filePath: string): {
+  content: string
+  encoding: BufferEncoding
+  lineEndings: LineEndingType
+} {
+  const fs = getFsImplementation()
+  const { resolvedPath, isSymlink } = safeResolvePath(fs, filePath)
+
+  if (isSymlink) {
+    logForDebugging(`Reading through symlink: ${filePath} -> ${resolvedPath}`)
+  }
+
+  const encoding = detectEncodingForResolvedPath(resolvedPath)
+  const raw = fs.readFileSync(resolvedPath, { encoding })
+  // Detect line endings from the raw head before CRLF normalization erases
+  // the distinction. 4096 code units is ≥ detectLineEndings's 4096-byte
+  // readSync sample (line endings are ASCII, so the unit mismatch is moot).
+  const lineEndings = detectLineEndingsForString(raw.slice(0, 4096))
+  return {
+    content: raw.replaceAll('\r\n', '\n'),
+    encoding,
+    lineEndings,
+  }
+}
+
+export function readFileSync(filePath: string): string {
+  return readFileSyncWithMetadata(filePath).content
+}

package/utils/fileReadCache.ts

@@ -0,0 +1,96 @@
+import { detectFileEncoding } from './file.js'
+import { getFsImplementation } from './fsOperations.js'
+
+type CachedFileData = {
+  content: string
+  encoding: BufferEncoding
+  mtime: number
+}
+
+/**
+ * A simple in-memory cache for file contents with automatic invalidation based on modification time.
+ * This eliminates redundant file reads in FileEditTool operations.
+ */
+class FileReadCache {
+  private cache = new Map<string, CachedFileData>()
+  private readonly maxCacheSize = 1000
+
+  /**
+   * Reads a file with caching. Returns both content and encoding.
+   * Cache key includes file path and modification time for automatic invalidation.
+   */
+  readFile(filePath: string): { content: string; encoding: BufferEncoding } {
+    const fs = getFsImplementation()
+
+    // Get file stats for cache invalidation
+    let stats
+    try {
+      stats = fs.statSync(filePath)
+    } catch (error) {
+      // File was deleted, remove from cache and re-throw
+      this.cache.delete(filePath)
+      throw error
+    }
+
+    const cacheKey = filePath
+    const cachedData = this.cache.get(cacheKey)
+
+    // Check if we have valid cached data
+    if (cachedData && cachedData.mtime === stats.mtimeMs) {
+      return {
+        content: cachedData.content,
+        encoding: cachedData.encoding,
+      }
+    }
+
+    // Cache miss or stale data - read the file
+    const encoding = detectFileEncoding(filePath)
+    const content = fs
+      .readFileSync(filePath, { encoding })
+      .replaceAll('\r\n', '\n')
+
+    // Update cache
+    this.cache.set(cacheKey, {
+      content,
+      encoding,
+      mtime: stats.mtimeMs,
+    })
+
+    // Evict oldest entries if cache is too large
+    if (this.cache.size > this.maxCacheSize) {
+      const firstKey = this.cache.keys().next().value
+      if (firstKey) {
+        this.cache.delete(firstKey)
+      }
+    }
+
+    return { content, encoding }
+  }
+
+  /**
+   * Clears the entire cache. Useful for testing or memory management.
+   */
+  clear(): void {
+    this.cache.clear()
+  }
+
+  /**
+   * Removes a specific file from the cache.
+   */
+  invalidate(filePath: string): void {
+    this.cache.delete(filePath)
+  }
+
+  /**
+   * Gets cache statistics for debugging/monitoring.
+   */
+  getStats(): { size: number; entries: string[] } {
+    return {
+      size: this.cache.size,
+      entries: Array.from(this.cache.keys()),
+    }
+  }
+}
+
+// Export a singleton instance
+export const fileReadCache = new FileReadCache()