@xortex/xcode 3.0.0

package/utils/mcpOutputStorage.ts

@@ -0,0 +1,189 @@
+import { writeFile } from 'fs/promises'
+import { join } from 'path'
+import {
+  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  logEvent,
+} from '../services/analytics/index.js'
+import type { MCPResultType } from '../services/mcp/client.js'
+import { toError } from './errors.js'
+import { formatFileSize } from './format.js'
+import { logError } from './log.js'
+import { ensureToolResultsDir, getToolResultsDir } from './toolResultStorage.js'
+
+/**
+ * Generates a format description string based on the MCP result type and schema.
+ */
+export function getFormatDescription(
+  type: MCPResultType,
+  schema?: unknown,
+): string {
+  switch (type) {
+    case 'toolResult':
+      return 'Plain text'
+    case 'structuredContent':
+      return schema ? `JSON with schema: ${schema}` : 'JSON'
+    case 'contentArray':
+      return schema ? `JSON array with schema: ${schema}` : 'JSON array'
+  }
+}
+
+/**
+ * Generates instruction text for Claude to read from a saved output file.
+ *
+ * @param rawOutputPath - Path to the saved output file
+ * @param contentLength - Length of the content in characters
+ * @param formatDescription - Description of the content format
+ * @param maxReadLength - Optional max chars for Read tool (for Bash output context)
+ * @returns Instruction text to include in the tool result
+ */
+export function getLargeOutputInstructions(
+  rawOutputPath: string,
+  contentLength: number,
+  formatDescription: string,
+  maxReadLength?: number,
+): string {
+  const baseInstructions =
+    `Error: result (${contentLength.toLocaleString()} characters) exceeds maximum allowed tokens. Output has been saved to ${rawOutputPath}.\n` +
+    `Format: ${formatDescription}\n` +
+    `Use offset and limit parameters to read specific portions of the file, search within it for specific content, and jq to make structured queries.\n` +
+    `REQUIREMENTS FOR SUMMARIZATION/ANALYSIS/REVIEW:\n` +
+    `- You MUST read the content from the file at ${rawOutputPath} in sequential chunks until 100% of the content has been read.\n`
+
+  const truncationWarning = maxReadLength
+    ? `- If you receive truncation warnings when reading the file ("[N lines truncated]"), reduce the chunk size until you have read 100% of the content without truncation ***DO NOT PROCEED UNTIL YOU HAVE DONE THIS***. Bash output is limited to ${maxReadLength.toLocaleString()} chars.\n`
+    : `- If you receive truncation warnings when reading the file, reduce the chunk size until you have read 100% of the content without truncation.\n`
+
+  const completionRequirement = `- Before producing ANY summary or analysis, you MUST explicitly describe what portion of the content you have read. ***If you did not read the entire content, you MUST explicitly state this.***\n`
+
+  return baseInstructions + truncationWarning + completionRequirement
+}
+
+/**
+ * Map a mime type to a file extension. Conservative: known types get their
+ * proper extension; unknown types get 'bin'. The extension matters because
+ * the Read tool dispatches on it (PDFs, images, etc. need the right ext).
+ */
+export function extensionForMimeType(mimeType: string | undefined): string {
+  if (!mimeType) return 'bin'
+  // Strip any charset/boundary parameter
+  const mt = (mimeType.split(';')[0] ?? '').trim().toLowerCase()
+  switch (mt) {
+    case 'application/pdf':
+      return 'pdf'
+    case 'application/json':
+      return 'json'
+    case 'text/csv':
+      return 'csv'
+    case 'text/plain':
+      return 'txt'
+    case 'text/html':
+      return 'html'
+    case 'text/markdown':
+      return 'md'
+    case 'application/zip':
+      return 'zip'
+    case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+      return 'docx'
+    case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+      return 'xlsx'
+    case 'application/vnd.openxmlformats-officedocument.presentationml.presentation':
+      return 'pptx'
+    case 'application/msword':
+      return 'doc'
+    case 'application/vnd.ms-excel':
+      return 'xls'
+    case 'audio/mpeg':
+      return 'mp3'
+    case 'audio/wav':
+      return 'wav'
+    case 'audio/ogg':
+      return 'ogg'
+    case 'video/mp4':
+      return 'mp4'
+    case 'video/webm':
+      return 'webm'
+    case 'image/png':
+      return 'png'
+    case 'image/jpeg':
+      return 'jpg'
+    case 'image/gif':
+      return 'gif'
+    case 'image/webp':
+      return 'webp'
+    case 'image/svg+xml':
+      return 'svg'
+    default:
+      return 'bin'
+  }
+}
+
+/**
+ * Heuristic for whether a content-type header indicates binary content that
+ * should be saved to disk rather than put into the model context.
+ * Text-ish types (text/*, json, xml, form data) are treated as non-binary.
+ */
+export function isBinaryContentType(contentType: string): boolean {
+  if (!contentType) return false
+  const mt = (contentType.split(';')[0] ?? '').trim().toLowerCase()
+  if (mt.startsWith('text/')) return false
+  // Structured text formats delivered with an application/ type. Use suffix
+  // or exact match rather than substring so 'openxmlformats' (docx/xlsx) stays binary.
+  if (mt.endsWith('+json') || mt === 'application/json') return false
+  if (mt.endsWith('+xml') || mt === 'application/xml') return false
+  if (mt.startsWith('application/javascript')) return false
+  if (mt === 'application/x-www-form-urlencoded') return false
+  return true
+}
+
+export type PersistBinaryResult =
+  | { filepath: string; size: number; ext: string }
+  | { error: string }
+
+/**
+ * Write raw binary bytes to the tool-results directory with a mime-derived
+ * extension. Unlike persistToolResult (which stringifies), this writes the
+ * bytes as-is so the resulting file can be opened with native tools (Read
+ * for PDFs, pandas for xlsx, etc.).
+ */
+export async function persistBinaryContent(
+  bytes: Buffer,
+  mimeType: string | undefined,
+  persistId: string,
+): Promise<PersistBinaryResult> {
+  await ensureToolResultsDir()
+  const ext = extensionForMimeType(mimeType)
+  const filepath = join(getToolResultsDir(), `${persistId}.${ext}`)
+
+  try {
+    await writeFile(filepath, bytes)
+  } catch (error) {
+    const err = toError(error)
+    logError(err)
+    return { error: err.message }
+  }
+
+  // mime type and extension are safe fixed-vocabulary strings (not paths/code)
+  logEvent('tengu_binary_content_persisted', {
+    mimeType: (mimeType ??
+      'unknown') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    sizeBytes: bytes.length,
+    ext: ext as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  })
+
+  return { filepath, size: bytes.length, ext }
+}
+
+/**
+ * Build a short message telling Claude where binary content was saved.
+ * Just states the path — no prescriptive hint, since what the model can
+ * actually do with the file depends on provider/tooling.
+ */
+export function getBinaryBlobSavedMessage(
+  filepath: string,
+  mimeType: string | undefined,
+  size: number,
+  sourceDescription: string,
+): string {
+  const mt = mimeType || 'unknown type'
+  return `${sourceDescription}Binary content (${mt}, ${formatFileSize(size)}) saved to ${filepath}`
+}

package/utils/mcpValidation.ts

@@ -0,0 +1,208 @@
+import type {
+  ContentBlockParam,
+  ImageBlockParam,
+  TextBlockParam,
+} from '@anthropic-ai/sdk/resources/index.mjs'
+import { getFeatureValue_CACHED_MAY_BE_STALE } from '../services/analytics/growthbook.js'
+import {
+  countMessagesTokensWithAPI,
+  roughTokenCountEstimation,
+} from '../services/tokenEstimation.js'
+import { compressImageBlock } from './imageResizer.js'
+import { logError } from './log.js'
+
+export const MCP_TOKEN_COUNT_THRESHOLD_FACTOR = 0.5
+export const IMAGE_TOKEN_ESTIMATE = 1600
+const DEFAULT_MAX_MCP_OUTPUT_TOKENS = 25000
+
+/**
+ * Resolve the MCP output token cap. Precedence:
+ *   1. MAX_MCP_OUTPUT_TOKENS env var (explicit user override)
+ *   2. tengu_satin_quoll GrowthBook flag's `mcp_tool` key (tokens, not chars —
+ *      unlike the other keys in that map which getPersistenceThreshold reads
+ *      as chars; MCP has its own truncation layer upstream of that)
+ *   3. Hardcoded default
+ */
+export function getMaxMcpOutputTokens(): number {
+  const envValue = process.env.MAX_MCP_OUTPUT_TOKENS
+  if (envValue) {
+    const parsed = parseInt(envValue, 10)
+    if (Number.isFinite(parsed) && parsed > 0) {
+      return parsed
+    }
+  }
+  const overrides = getFeatureValue_CACHED_MAY_BE_STALE<Record<
+    string,
+    number
+  > | null>('tengu_satin_quoll', {})
+  const override = overrides?.['mcp_tool']
+  if (
+    typeof override === 'number' &&
+    Number.isFinite(override) &&
+    override > 0
+  ) {
+    return override
+  }
+  return DEFAULT_MAX_MCP_OUTPUT_TOKENS
+}
+
+export type MCPToolResult = string | ContentBlockParam[] | undefined
+
+function isTextBlock(block: ContentBlockParam): block is TextBlockParam {
+  return block.type === 'text'
+}
+
+function isImageBlock(block: ContentBlockParam): block is ImageBlockParam {
+  return block.type === 'image'
+}
+
+export function getContentSizeEstimate(content: MCPToolResult): number {
+  if (!content) return 0
+
+  if (typeof content === 'string') {
+    return roughTokenCountEstimation(content)
+  }
+
+  return content.reduce((total, block) => {
+    if (isTextBlock(block)) {
+      return total + roughTokenCountEstimation(block.text)
+    } else if (isImageBlock(block)) {
+      // Estimate for image tokens
+      return total + IMAGE_TOKEN_ESTIMATE
+    }
+    return total
+  }, 0)
+}
+
+function getMaxMcpOutputChars(): number {
+  return getMaxMcpOutputTokens() * 4
+}
+
+function getTruncationMessage(): string {
+  return `\n\n[OUTPUT TRUNCATED - exceeded ${getMaxMcpOutputTokens()} token limit]
+
+The tool output was truncated. If this MCP server provides pagination or filtering tools, use them to retrieve specific portions of the data. If pagination is not available, inform the user that you are working with truncated output and results may be incomplete.`
+}
+
+function truncateString(content: string, maxChars: number): string {
+  if (content.length <= maxChars) {
+    return content
+  }
+  return content.slice(0, maxChars)
+}
+
+async function truncateContentBlocks(
+  blocks: ContentBlockParam[],
+  maxChars: number,
+): Promise<ContentBlockParam[]> {
+  const result: ContentBlockParam[] = []
+  let currentChars = 0
+
+  for (const block of blocks) {
+    if (isTextBlock(block)) {
+      const remainingChars = maxChars - currentChars
+      if (remainingChars <= 0) break
+
+      if (block.text.length <= remainingChars) {
+        result.push(block)
+        currentChars += block.text.length
+      } else {
+        result.push({ type: 'text', text: block.text.slice(0, remainingChars) })
+        break
+      }
+    } else if (isImageBlock(block)) {
+      // Include images but count their estimated size
+      const imageChars = IMAGE_TOKEN_ESTIMATE * 4
+      if (currentChars + imageChars <= maxChars) {
+        result.push(block)
+        currentChars += imageChars
+      } else {
+        // Image exceeds budget - try to compress it to fit remaining space
+        const remainingChars = maxChars - currentChars
+        if (remainingChars > 0) {
+          // Convert remaining chars to bytes for compression
+          // base64 uses ~4/3 the original size, so we calculate max bytes
+          const remainingBytes = Math.floor(remainingChars * 0.75)
+          try {
+            const compressedBlock = await compressImageBlock(
+              block,
+              remainingBytes,
+            )
+            result.push(compressedBlock)
+            // Update currentChars based on compressed image size
+            if (compressedBlock.source.type === 'base64') {
+              currentChars += compressedBlock.source.data.length
+            } else {
+              currentChars += imageChars
+            }
+          } catch {
+            // If compression fails, skip the image
+          }
+        }
+      }
+    } else {
+      result.push(block)
+    }
+  }
+
+  return result
+}
+
+export async function mcpContentNeedsTruncation(
+  content: MCPToolResult,
+): Promise<boolean> {
+  if (!content) return false
+
+  // Use size check as a heuristic to avoid unnecessary token counting API calls
+  const contentSizeEstimate = getContentSizeEstimate(content)
+  if (
+    contentSizeEstimate <=
+    getMaxMcpOutputTokens() * MCP_TOKEN_COUNT_THRESHOLD_FACTOR
+  ) {
+    return false
+  }
+
+  try {
+    const messages =
+      typeof content === 'string'
+        ? [{ role: 'user' as const, content }]
+        : [{ role: 'user' as const, content }]
+
+    const tokenCount = await countMessagesTokensWithAPI(messages, [])
+    return !!(tokenCount && tokenCount > getMaxMcpOutputTokens())
+  } catch (error) {
+    logError(error)
+    // Assume no truncation needed on error
+    return false
+  }
+}
+
+export async function truncateMcpContent(
+  content: MCPToolResult,
+): Promise<MCPToolResult> {
+  if (!content) return content
+
+  const maxChars = getMaxMcpOutputChars()
+  const truncationMsg = getTruncationMessage()
+
+  if (typeof content === 'string') {
+    return truncateString(content, maxChars) + truncationMsg
+  } else {
+    const truncatedBlocks = await truncateContentBlocks(
+      content as ContentBlockParam[],
+      maxChars,
+    )
+    truncatedBlocks.push({ type: 'text', text: truncationMsg })
+    return truncatedBlocks
+  }
+}
+
+export async function truncateMcpContentIfNeeded(
+  content: MCPToolResult,
+): Promise<MCPToolResult> {
+  if (!(await mcpContentNeedsTruncation(content))) {
+    return content
+  }
+
+  return await truncateMcpContent(content)
+}

package/utils/mcpWebSocketTransport.ts

@@ -0,0 +1,200 @@
+import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'
+import {
+  type JSONRPCMessage,
+  JSONRPCMessageSchema,
+} from '@modelcontextprotocol/sdk/types.js'
+import type WsWebSocket from 'ws'
+import { logForDiagnosticsNoPII } from './diagLogs.js'
+import { toError } from './errors.js'
+import { jsonParse, jsonStringify } from './slowOperations.js'
+
+// WebSocket readyState constants (same for both native and ws)
+const WS_CONNECTING = 0
+const WS_OPEN = 1
+
+// Minimal interface shared by globalThis.WebSocket and ws.WebSocket
+type WebSocketLike = {
+  readonly readyState: number
+  close(): void
+  send(data: string): void
+}
+
+export class WebSocketTransport implements Transport {
+  private started = false
+  private opened: Promise<void>
+  private isBun = typeof Bun !== 'undefined'
+
+  constructor(private ws: WebSocketLike) {
+    this.opened = new Promise((resolve, reject) => {
+      if (this.ws.readyState === WS_OPEN) {
+        resolve()
+      } else if (this.isBun) {
+        const nws = this.ws as unknown as globalThis.WebSocket
+        const onOpen = () => {
+          nws.removeEventListener('open', onOpen)
+          nws.removeEventListener('error', onError)
+          resolve()
+        }
+        const onError = (event: Event) => {
+          nws.removeEventListener('open', onOpen)
+          nws.removeEventListener('error', onError)
+          logForDiagnosticsNoPII('error', 'mcp_websocket_connect_fail')
+          reject(event)
+        }
+        nws.addEventListener('open', onOpen)
+        nws.addEventListener('error', onError)
+      } else {
+        const nws = this.ws as unknown as WsWebSocket
+        nws.on('open', () => {
+          resolve()
+        })
+        nws.on('error', error => {
+          logForDiagnosticsNoPII('error', 'mcp_websocket_connect_fail')
+          reject(error)
+        })
+      }
+    })
+
+    // Attach persistent event handlers
+    if (this.isBun) {
+      const nws = this.ws as unknown as globalThis.WebSocket
+      nws.addEventListener('message', this.onBunMessage)
+      nws.addEventListener('error', this.onBunError)
+      nws.addEventListener('close', this.onBunClose)
+    } else {
+      const nws = this.ws as unknown as WsWebSocket
+      nws.on('message', this.onNodeMessage)
+      nws.on('error', this.onNodeError)
+      nws.on('close', this.onNodeClose)
+    }
+  }
+
+  onclose?: () => void
+  onerror?: (error: Error) => void
+  onmessage?: (message: JSONRPCMessage) => void
+
+  // Bun (native WebSocket) event handlers
+  private onBunMessage = (event: MessageEvent) => {
+    try {
+      const data =
+        typeof event.data === 'string' ? event.data : String(event.data)
+      const messageObj = jsonParse(data)
+      const message = JSONRPCMessageSchema.parse(messageObj)
+      this.onmessage?.(message)
+    } catch (error) {
+      this.handleError(error)
+    }
+  }
+
+  private onBunError = () => {
+    this.handleError(new Error('WebSocket error'))
+  }
+
+  private onBunClose = () => {
+    this.handleCloseCleanup()
+  }
+
+  // Node (ws package) event handlers
+  private onNodeMessage = (data: Buffer) => {
+    try {
+      const messageObj = jsonParse(data.toString('utf-8'))
+      const message = JSONRPCMessageSchema.parse(messageObj)
+      this.onmessage?.(message)
+    } catch (error) {
+      this.handleError(error)
+    }
+  }
+
+  private onNodeError = (error: unknown) => {
+    this.handleError(error)
+  }
+
+  private onNodeClose = () => {
+    this.handleCloseCleanup()
+  }
+
+  // Shared error handler
+  private handleError(error: unknown): void {
+    logForDiagnosticsNoPII('error', 'mcp_websocket_message_fail')
+    this.onerror?.(toError(error))
+  }
+
+  // Shared close handler with listener cleanup
+  private handleCloseCleanup(): void {
+    this.onclose?.()
+    // Clean up listeners after close
+    if (this.isBun) {
+      const nws = this.ws as unknown as globalThis.WebSocket
+      nws.removeEventListener('message', this.onBunMessage)
+      nws.removeEventListener('error', this.onBunError)
+      nws.removeEventListener('close', this.onBunClose)
+    } else {
+      const nws = this.ws as unknown as WsWebSocket
+      nws.off('message', this.onNodeMessage)
+      nws.off('error', this.onNodeError)
+      nws.off('close', this.onNodeClose)
+    }
+  }
+
+  /**
+   * Starts listening for messages on the WebSocket.
+   */
+  async start(): Promise<void> {
+    if (this.started) {
+      throw new Error('Start can only be called once per transport.')
+    }
+    await this.opened
+    if (this.ws.readyState !== WS_OPEN) {
+      logForDiagnosticsNoPII('error', 'mcp_websocket_start_not_opened')
+      throw new Error('WebSocket is not open. Cannot start transport.')
+    }
+    this.started = true
+    // Unlike stdio, WebSocket connections are typically already established when the transport is created.
+    // No explicit connection action needed here, just attaching listeners.
+  }
+
+  /**
+   * Closes the WebSocket connection.
+   */
+  async close(): Promise<void> {
+    if (
+      this.ws.readyState === WS_OPEN ||
+      this.ws.readyState === WS_CONNECTING
+    ) {
+      this.ws.close()
+    }
+    // Ensure listeners are removed even if close was called externally or connection was already closed
+    this.handleCloseCleanup()
+  }
+
+  /**
+   * Sends a JSON-RPC message over the WebSocket connection.
+   */
+  async send(message: JSONRPCMessage): Promise<void> {
+    if (this.ws.readyState !== WS_OPEN) {
+      logForDiagnosticsNoPII('error', 'mcp_websocket_send_not_opened')
+      throw new Error('WebSocket is not open. Cannot send message.')
+    }
+    const json = jsonStringify(message)
+
+    try {
+      if (this.isBun) {
+        // Native WebSocket.send() is synchronous (no callback)
+        this.ws.send(json)
+      } else {
+        await new Promise<void>((resolve, reject) => {
+          ;(this.ws as unknown as WsWebSocket).send(json, error => {
+            if (error) {
+              reject(error)
+            } else {
+              resolve()
+            }
+          })
+        })
+      }
+    } catch (error) {
+      this.handleError(error)
+      throw error
+    }
+  }
+}