illuma-agents 1.0.16 → 1.0.18

package/src/stream.ts

@@ -107,24 +107,27 @@ export function getChunkContent({
         | undefined
     )?.summary?.[0]?.text;
   }
-  if (
-    provider === Providers.OPENROUTER &&
-    chunk?.additional_kwargs?.reasoning_details != null &&
-    Array.isArray(chunk.additional_kwargs.reasoning_details)
-  ) {
-    // Extract text from reasoning_details array (for Gemini, DeepSeek, etc.)
-    const textEntries = chunk.additional_kwargs.reasoning_details
-      .filter(
-        (detail) =>
-          detail.type === 'reasoning.text' &&
-          detail.text != null &&
-          detail.text !== ''
-      )
-      .map((detail) => detail.text)
-      .join('');
-    if (textEntries) {
-      return textEntries;
+  /**
+   * For OpenRouter, reasoning is stored in additional_kwargs.reasoning (not reasoning_content).
+   * NOTE: We intentionally do NOT extract text from reasoning_details here.
+   * The reasoning_details array contains the FULL accumulated reasoning text (set only on final chunk),
+   * but individual reasoning tokens are already streamed via additional_kwargs.reasoning.
+   * Extracting from reasoning_details would cause duplication.
+   * The reasoning_details is only used for:
+   * 1. Detecting reasoning mode in handleReasoning()
+   * 2. Final message storage (for thought signatures)
+   */
+  if (provider === Providers.OPENROUTER) {
+    // Content presence signals end of reasoning phase - prefer content over reasoning
+    // This handles transitional chunks that may have both reasoning and content
+    if (typeof chunk?.content === 'string' && chunk.content !== '') {
+      return chunk.content;
+    }
+    const reasoning = chunk?.additional_kwargs?.reasoning as string | undefined;
+    if (reasoning != null && reasoning !== '') {
+      return reasoning;
     }
+    return chunk?.content;
   }
   return (
     ((chunk?.additional_kwargs?.[reasoningKey] as string | undefined) ?? '') ||
@@ -376,9 +379,14 @@ hasToolCallChunks: ${hasToolCallChunks}
       reasoning_content = 'valid';
     } else if (
       agentContext.provider === Providers.OPENROUTER &&
-      chunk.additional_kwargs?.reasoning_details != null &&
-      Array.isArray(chunk.additional_kwargs.reasoning_details) &&
-      chunk.additional_kwargs.reasoning_details.length > 0
+      // Only set reasoning as valid if content is NOT present (content signals end of reasoning)
+      (chunk.content == null || chunk.content === '') &&
+      // Check for reasoning_details (final chunk) OR reasoning string (intermediate chunks)
+      ((chunk.additional_kwargs?.reasoning_details != null &&
+        Array.isArray(chunk.additional_kwargs.reasoning_details) &&
+        chunk.additional_kwargs.reasoning_details.length > 0) ||
+        (typeof chunk.additional_kwargs?.reasoning === 'string' &&
+          chunk.additional_kwargs.reasoning !== ''))
     ) {
       reasoning_content = 'valid';
     }

package/src/tools/ProgrammaticToolCalling.ts

@@ -38,42 +38,35 @@ const ProgrammaticToolCallingSchema = z.object({
     .string()
     .min(1)
     .describe(
-      `Python code that calls tools programmatically. Tools are automatically available as async Python functions - DO NOT define them yourself.
-
-The Code API generates async function stubs from the tool definitions. Just call them directly:
-
-Example (Simple call):
-  result = await get_weather(city="San Francisco")
-  print(result)
-
-Example (Parallel - Fastest):
-  results = await asyncio.gather(
-      get_weather(city="SF"),
-      get_weather(city="NYC"),
-      get_weather(city="London")
-  )
-  for city, weather in zip(["SF", "NYC", "London"], results):
-      print(f"{city}: {weather['temperature']}°F")
-
-Example (Loop with processing):
-  team = await get_team_members()
-  for member in team:
-      expenses = await get_expenses(user_id=member['id'])
-      total = sum(e['amount'] for e in expenses)
-      print(f"{member['name']}: \${total:.2f}")
-
-Example (Conditional logic):
-  data = await fetch_data(source="primary")
-  if not data:
-      data = await fetch_data(source="backup")
-  print(f"Got {len(data)} records")
-
-Requirements:
-- Tools are pre-defined as async functions - DO NOT write function definitions
-- Use await for all tool calls
-- Use asyncio.gather() for parallel execution of independent calls
-- Only print() output flows back to the context window
-- Tool results from programmatic calls do NOT consume context tokens`
+      `Python code that calls tools programmatically. Tools are available as async functions.
+
+CRITICAL - STATELESS EXECUTION:
+Each call is a fresh Python interpreter. Variables, imports, and data do NOT persist between calls.
+You MUST complete your entire workflow in ONE code block: query → process → output.
+DO NOT split work across multiple calls expecting to reuse variables.
+
+Your code is auto-wrapped in async context. Just write logic with await—no boilerplate needed.
+
+Example (Complete workflow in one call):
+  # Query data
+  data = await query_database(sql="SELECT * FROM users")
+  # Process it
+  df = pd.DataFrame(data)
+  summary = df.groupby('region').sum()
+  # Output results
+  await write_to_sheet(spreadsheet_id=sid, data=summary.to_dict())
+  print(f"Wrote {len(summary)} rows")
+
+Example (Parallel calls):
+  sf, ny = await asyncio.gather(get_weather(city="SF"), get_weather(city="NY"))
+  print(f"SF: {sf}, NY: {ny}")
+
+Rules:
+- EVERYTHING in one call—no state persists between executions
+- Just write code with await—auto-wrapped in async context
+- DO NOT define async def main() or call asyncio.run()
+- Tools are pre-defined—DO NOT write function definitions
+- Only print() output returns to the model`
     ),
   session_id: z
     .string()
@@ -234,6 +227,67 @@ export function filterToolsByUsage(
   return toolDefs.filter((tool) => usedToolNames.has(tool.name));
 }
 
+/**
+ * Fetches files from a previous session to make them available for the current execution.
+ * Files are returned as CodeEnvFile references to be included in the request.
+ * @param baseUrl - The base URL for the Code API
+ * @param apiKey - The API key for authentication
+ * @param sessionId - The session ID to fetch files from
+ * @param proxy - Optional HTTP proxy URL
+ * @returns Array of CodeEnvFile references, or empty array if fetch fails
+ */
+export async function fetchSessionFiles(
+  baseUrl: string,
+  apiKey: string,
+  sessionId: string,
+  proxy?: string
+): Promise<t.CodeEnvFile[]> {
+  try {
+    const filesEndpoint = `${baseUrl}/files/${sessionId}?detail=full`;
+    const fetchOptions: RequestInit = {
+      method: 'GET',
+      headers: {
+        'User-Agent': 'LibreChat/1.0',
+        'X-API-Key': apiKey,
+      },
+    };
+
+    if (proxy != null && proxy !== '') {
+      fetchOptions.agent = new HttpsProxyAgent(proxy);
+    }
+
+    const response = await fetch(filesEndpoint, fetchOptions);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch files for session: ${response.status}`);
+    }
+
+    const files = await response.json();
+    if (!Array.isArray(files) || files.length === 0) {
+      return [];
+    }
+
+    return files.map((file: Record<string, unknown>) => {
+      // Extract the ID from the file name (part after session ID prefix and before extension)
+      const nameParts = (file.name as string).split('/');
+      const id = nameParts.length > 1 ? nameParts[1].split('.')[0] : '';
+
+      return {
+        session_id: sessionId,
+        id,
+        name: (file.metadata as Record<string, unknown>)[
+          'original-filename'
+        ] as string,
+      };
+    });
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.warn(
+      `Failed to fetch files for session: ${sessionId}, ${(error as Error).message}`
+    );
+    return [];
+  }
+}
+
 /**
  * Makes an HTTP request to the Code API.
  * @param endpoint - The API endpoint URL
@@ -274,9 +328,140 @@ export async function makeRequest(
   return (await response.json()) as t.ProgrammaticExecutionResponse;
 }
 
+/**
+ * Unwraps tool responses that may be formatted as tuples or content blocks.
+ * MCP tools return [content, artifacts], we need to extract the raw data.
+ * @param result - The raw result from tool.invoke()
+ * @param isMCPTool - Whether this is an MCP tool (has mcp property)
+ * @returns Unwrapped raw data (string, object, or parsed JSON)
+ */
+export function unwrapToolResponse(
+  result: unknown,
+  isMCPTool: boolean
+): unknown {
+  // Only unwrap if this is an MCP tool and result is a tuple
+  if (!isMCPTool) {
+    return result;
+  }
+
+  /**
+   * Checks if a value is a content block object (has type and text).
+   */
+  const isContentBlock = (value: unknown): boolean => {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+      return false;
+    }
+    const obj = value as Record<string, unknown>;
+    return typeof obj.type === 'string';
+  };
+
+  /**
+   * Checks if an array is an array of content blocks.
+   */
+  const isContentBlockArray = (arr: unknown[]): boolean => {
+    return arr.length > 0 && arr.every(isContentBlock);
+  };
+
+  /**
+   * Extracts text from a single content block object.
+   * Returns the text if it's a text block, otherwise returns null.
+   */
+  const extractTextFromBlock = (block: unknown): string | null => {
+    if (typeof block !== 'object' || block === null) return null;
+    const b = block as Record<string, unknown>;
+    if (b.type === 'text' && typeof b.text === 'string') {
+      return b.text;
+    }
+    return null;
+  };
+
+  /**
+   * Extracts text from content blocks (array or single object).
+   * Returns combined text or null if no text blocks found.
+   */
+  const extractTextFromContent = (content: unknown): string | null => {
+    // Single content block object: { type: 'text', text: '...' }
+    if (
+      typeof content === 'object' &&
+      content !== null &&
+      !Array.isArray(content)
+    ) {
+      const text = extractTextFromBlock(content);
+      if (text !== null) return text;
+    }
+
+    // Array of content blocks: [{ type: 'text', text: '...' }, ...]
+    if (Array.isArray(content) && content.length > 0) {
+      const texts = content
+        .map(extractTextFromBlock)
+        .filter((t): t is string => t !== null);
+      if (texts.length > 0) {
+        return texts.join('\n');
+      }
+    }
+
+    return null;
+  };
+
+  /**
+   * Tries to parse a string as JSON if it looks like JSON.
+   */
+  const maybeParseJSON = (str: string): unknown => {
+    const trimmed = str.trim();
+    if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+      try {
+        return JSON.parse(trimmed);
+      } catch {
+        return str;
+      }
+    }
+    return str;
+  };
+
+  // Handle array of content blocks at top level FIRST
+  // (before checking for tuple, since both are arrays)
+  if (Array.isArray(result) && isContentBlockArray(result)) {
+    const extractedText = extractTextFromContent(result);
+    if (extractedText !== null) {
+      return maybeParseJSON(extractedText);
+    }
+  }
+
+  // Check if result is a tuple/array with [content, artifacts]
+  if (Array.isArray(result) && result.length >= 1) {
+    const [content] = result;
+
+    // If first element is a string, return it (possibly parsed as JSON)
+    if (typeof content === 'string') {
+      return maybeParseJSON(content);
+    }
+
+    // Try to extract text from content blocks
+    const extractedText = extractTextFromContent(content);
+    if (extractedText !== null) {
+      return maybeParseJSON(extractedText);
+    }
+
+    // If first element is an object (but not a text block), return it
+    if (typeof content === 'object' && content !== null) {
+      return content;
+    }
+  }
+
+  // Handle single content block object at top level (not in tuple)
+  const extractedText = extractTextFromContent(result);
+  if (extractedText !== null) {
+    return maybeParseJSON(extractedText);
+  }
+
+  // Not a formatted response, return as-is
+  return result;
+}
+
 /**
  * Executes tools in parallel when requested by the API.
  * Uses Promise.all for parallel execution, catching individual errors.
+ * Unwraps formatted responses (e.g., MCP tool tuples) to raw data.
  * @param toolCalls - Array of tool calls from the API
  * @param toolMap - Map of tool names to executable tools
  * @returns Array of tool results
@@ -301,9 +486,13 @@ export async function executeTools(
       const result = await tool.invoke(call.input, {
         metadata: { [Constants.PROGRAMMATIC_TOOL_CALLING]: true },
       });
+
+      const isMCPTool = tool.mcp === true;
+      const unwrappedResult = unwrapToolResponse(result, isMCPTool);
+
       return {
         call_id: call.id,
-        result,
+        result: unwrappedResult,
         is_error: false,
       };
     } catch (error) {
@@ -414,25 +603,23 @@ export function createProgrammaticToolCallingTool(
   const EXEC_ENDPOINT = `${baseUrl}/exec/programmatic`;
 
   const description = `
-Run tools by writing Python code. Tools are available as async functions - just call them with await.
+Run tools via Python code. Auto-wrapped in async context—just use \`await\` directly.
 
-This is different from execute_code: here you can call your tools (like get_weather, get_expenses, etc.) directly in Python code.
+CRITICAL - STATELESS: Each call is a fresh interpreter. Variables/imports do NOT persist.
+Complete your ENTIRE workflow in ONE call: fetch → process → save. No splitting across calls.
 
-Usage:
-- Tools are pre-defined as async functions - call them with await
-- Use asyncio.gather() to run multiple tools in parallel
-- Only print() output is returned - tool results stay in Python
+Rules:
+- Everything in ONE code block—no state carries over between executions
+- Do NOT define \`async def main()\` or call \`asyncio.run()\`—just write code with await
+- Tools are pre-defined—DO NOT write function definitions
+- Only \`print()\` output returns; tool results are raw dicts/lists/strings
+- Use \`session_id\` param for file persistence across calls
+- Tool names normalized: hyphens→underscores, keywords get \`_tool\` suffix
 
-Examples:
-- Simple: result = await get_weather(city="NYC")
-- Loop: for user in users: data = await get_expenses(user_id=user['id'])
-- Parallel: sf, ny = await asyncio.gather(get_weather(city="SF"), get_weather(city="NY"))
+When to use: loops, conditionals, parallel (\`asyncio.gather\`), multi-step pipelines.
 
-When to use this instead of calling tools directly:
-- You need to call tools in a loop (process many items)
-- You want parallel execution (asyncio.gather)
-- You need conditionals based on tool results
-- You want to aggregate/filter data before returning
+Example (complete pipeline):
+  data = await query_db(sql="..."); df = process(data); await save_to_sheet(data=df); print("Done")
 `.trim();
 
   return tool<typeof ProgrammaticToolCallingSchema>(
@@ -474,6 +661,12 @@ When to use this instead of calling tools directly:
           );
         }
 
+        // Fetch files from previous session if session_id is provided
+        let files: t.CodeEnvFile[] | undefined;
+        if (session_id != null && session_id.length > 0) {
+          files = await fetchSessionFiles(baseUrl, apiKey, session_id, proxy);
+        }
+
         let response = await makeRequest(
           EXEC_ENDPOINT,
           apiKey,
@@ -482,6 +675,7 @@ When to use this instead of calling tools directly:
             tools: effectiveTools,
             session_id,
             timeout,
+            ...(files && files.length > 0 ? { files } : {}),
           },
           proxy
         );

package/src/tools/ToolNode.ts

@@ -20,6 +20,7 @@ import type { BaseMessage, AIMessage } from '@langchain/core/messages';
 import type { StructuredToolInterface } from '@langchain/core/tools';
 import type * as t from '@/types';
 import { RunnableCallable } from '@/utils';
+import { processToolOutput } from '@/utils/toonFormat';
 import { Constants } from '@/common';
 
 /**
@@ -29,6 +30,43 @@ function isSend(value: unknown): value is Send {
   return value instanceof Send;
 }
 
+/**
+ * Extract text content from a ToolMessage content field.
+ * Handles both string and MessageContentComplex[] formats.
+ * For array content (e.g., from content_and_artifact tools), extracts text from text blocks.
+ */
+function extractStringContent(content: unknown): string {
+  // Already a string - return as is
+  if (typeof content === 'string') {
+    return content;
+  }
+
+  // Array of content blocks - extract text from each
+  if (Array.isArray(content)) {
+    const textParts: string[] = [];
+    for (const block of content) {
+      if (typeof block === 'string') {
+        textParts.push(block);
+      } else if (block && typeof block === 'object') {
+        // Handle {type: 'text', text: '...'} format
+        const obj = block as Record<string, unknown>;
+        if (obj.type === 'text' && typeof obj.text === 'string') {
+          textParts.push(obj.text);
+        } else if (typeof obj.text === 'string') {
+          // Just has 'text' property
+          textParts.push(obj.text);
+        }
+      }
+    }
+    if (textParts.length > 0) {
+      return textParts.join('\n');
+    }
+  }
+
+  // Fallback: stringify whatever it is
+  return JSON.stringify(content);
+}
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export class ToolNode<T = any> extends RunnableCallable<T, T> {
   private toolMap: Map<string, StructuredToolInterface | RunnableToolLike>;
@@ -140,16 +178,51 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
       }
 
       const output = await tool.invoke(invokeParams, config);
-      if (
-        (isBaseMessage(output) && output._getType() === 'tool') ||
-        isCommand(output)
-      ) {
+
+      // Handle Command outputs directly
+      if (isCommand(output)) {
         return output;
+      }
+
+      // ========================================================================
+      // TOOL OUTPUT PROCESSING - Single point for all tools (MCP and non-MCP)
+      // 1. Extract string content from any output format
+      // 2. Apply TOON conversion if content contains JSON
+      // 3. Apply truncation if still too large
+      // 4. Return ToolMessage with processed string content
+      // ========================================================================
+
+      // Step 1: Extract string content from the output
+      let rawContent: string;
+      if (isBaseMessage(output) && output._getType() === 'tool') {
+        const toolMsg = output as ToolMessage;
+        rawContent = extractStringContent(toolMsg.content);
+      } else {
+        rawContent = extractStringContent(output);
+      }
+
+      // Step 2 & 3: Apply TOON conversion and truncation
+      const processed = processToolOutput(rawContent, {
+        maxLength: 100000, // 100K char limit
+        enableToon: true,
+        minSizeForToon: 1000,
+        minReductionPercent: 10, // Only apply TOON when clearly beneficial
+      });
+
+      // Step 4: Return ToolMessage with processed string content
+      if (isBaseMessage(output) && output._getType() === 'tool') {
+        const toolMsg = output as ToolMessage;
+        return new ToolMessage({
+          status: toolMsg.status,
+          name: toolMsg.name,
+          content: processed.content,
+          tool_call_id: toolMsg.tool_call_id,
+        });
       } else {
         return new ToolMessage({
           status: 'success',
           name: tool.name,
-          content: typeof output === 'string' ? output : JSON.stringify(output),
+          content: processed.content,
           tool_call_id: call.id!,
         });
       }