illuma-agents 1.0.36 → 1.0.38

package/src/scripts/tools.ts

@@ -18,9 +18,13 @@ async function testStandardStreaming(): Promise<void> {
   const { userName, location, provider, currentDate } = await getArgs();
   const { contentParts, aggregateContent } = createContentAggregator();
   const customHandlers = {
-    [GraphEvents.TOOL_END]: new ToolEndHandler(undefined, (name?: string) => {
-      return true;
-    }),
+    [GraphEvents.TOOL_END]: new ToolEndHandler(
+      undefined,
+      undefined,
+      (name?: string) => {
+        return true;
+      }
+    ),
     [GraphEvents.CHAT_MODEL_END]: {
       handle: (
         _event: string,

package/src/stream.ts

@@ -339,7 +339,8 @@ hasToolCallChunks: ${hasToolCallChunks}
         (c) =>
           (c.type?.startsWith(ContentTypes.THINKING) ?? false) ||
           (c.type?.startsWith(ContentTypes.REASONING) ?? false) ||
-          (c.type?.startsWith(ContentTypes.REASONING_CONTENT) ?? false)
+          (c.type?.startsWith(ContentTypes.REASONING_CONTENT) ?? false) ||
+          c.type === 'redacted_thinking'
       )
     ) {
       await graph.dispatchReasoningDelta(stepId, {
@@ -365,7 +366,8 @@ hasToolCallChunks: ${hasToolCallChunks}
       Array.isArray(chunk.content) &&
       (chunk.content[0]?.type === ContentTypes.THINKING ||
         chunk.content[0]?.type === ContentTypes.REASONING ||
-        chunk.content[0]?.type === ContentTypes.REASONING_CONTENT)
+        chunk.content[0]?.type === ContentTypes.REASONING_CONTENT ||
+        chunk.content[0]?.type === 'redacted_thinking')
     ) {
       reasoning_content = 'valid';
     } else if (

package/src/tools/BrowserTools.ts

@@ -17,6 +17,8 @@ export const EBrowserTools = {
   BACK: 'browser_back',
   SCREENSHOT: 'browser_screenshot',
   GET_PAGE_STATE: 'browser_get_page_state',
+  KEYPRESS: 'browser_keypress',
+  SWITCH_TAB: 'browser_switch_tab',
 } as const;
 
 export type BrowserToolName =
@@ -74,19 +76,23 @@ const BrowserClickSchema = z.object({
   index: z
     .number()
     .describe(
-      'The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'
+      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'
     ),
 });
 
 const BrowserTypeSchema = z.object({
   index: z
     .number()
-    .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel="To recipients" vs fieldLabel="CC recipients"). Do NOT type into parent container <div> elements.'),
+    .describe(
+      'The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'
+    ),
   text: z.string().describe('The text to type into the element'),
   pressEnter: z
     .boolean()
     .optional()
-    .describe('Whether to press Enter after typing (useful for search forms)'),
+    .describe(
+      'Whether to press Enter after typing (useful for search forms and submitting)'
+    ),
 });
 
 const BrowserNavigateSchema = z.object({
@@ -129,6 +135,22 @@ const BrowserScreenshotSchema = z.object({});
 
 const BrowserGetPageStateSchema = z.object({});
 
+const BrowserKeypressSchema = z.object({
+  keys: z
+    .string()
+    .describe(
+      'Keyboard keys to press. Use "+" to combine modifiers (e.g., "Control+Enter", "Control+a", "Escape", "Tab", "Enter"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'
+    ),
+});
+
+const BrowserSwitchTabSchema = z.object({
+  tabId: z
+    .number()
+    .describe(
+      'The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'
+    ),
+});
+
 /**
  * Browser tool response interface
  * This is what the extension returns after executing the action
@@ -173,7 +195,9 @@ function formatResultForLLM(
   }
   if (result.elementList != null && result.elementList !== '') {
     // Add hint about fieldLabel and targeting inputs for form interactions
-    parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
+    parts.push(
+      `\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`
+    );
   }
   if (result.screenshot != null && result.screenshot !== '') {
     parts.push('\n[Screenshot captured and displayed to user]');
@@ -249,7 +273,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('click'), {
       name: EBrowserTools.CLICK,
-      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
+      description:
+        'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
       schema: BrowserClickSchema,
     })
   );
@@ -258,7 +283,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('type'), {
       name: EBrowserTools.TYPE,
-      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
+      description:
+        'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
       schema: BrowserTypeSchema,
     })
   );
@@ -267,7 +293,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('navigate'), {
       name: EBrowserTools.NAVIGATE,
-      description: 'Navigate to URL (include https://). Returns new page element list.',
+      description:
+        'Navigate to URL (include https://). Returns new page element list.',
       schema: BrowserNavigateSchema,
     })
   );
@@ -276,7 +303,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('scroll'), {
       name: EBrowserTools.SCROLL,
-      description: 'Scroll page (up/down/left/right). Returns updated element list.',
+      description:
+        'Scroll page (up/down/left/right). Returns updated element list.',
       schema: BrowserScrollSchema,
     })
   );
@@ -285,7 +313,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('extract'), {
       name: EBrowserTools.EXTRACT,
-      description: 'Extract page content. Returns URL, title, and element list.',
+      description:
+        'Extract page content. Returns URL, title, and element list.',
       schema: BrowserExtractSchema,
     })
   );
@@ -294,7 +323,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('hover'), {
       name: EBrowserTools.HOVER,
-      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',
+      description:
+        'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',
       schema: BrowserHoverSchema,
     })
   );
@@ -303,7 +333,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('wait'), {
       name: EBrowserTools.WAIT,
-      description: 'Wait for async content to load. Returns updated element list.',
+      description:
+        'Wait for async content to load. Returns updated element list.',
       schema: BrowserWaitSchema,
     })
   );
@@ -312,7 +343,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('back'), {
       name: EBrowserTools.BACK,
-      description: 'Go back in browser history. Returns previous page element list.',
+      description:
+        'Go back in browser history. Returns previous page element list.',
       schema: BrowserBackSchema,
     })
   );
@@ -321,7 +353,8 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('screenshot'), {
       name: EBrowserTools.SCREENSHOT,
-      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',
+      description:
+        'Capture screenshot. Displayed to user. Use get_page_state for automation.',
       schema: BrowserScreenshotSchema,
     })
   );
@@ -330,10 +363,31 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('get_page_state'), {
       name: EBrowserTools.GET_PAGE_STATE,
-      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',
+      description:
+        'Get page URL, title, and interactive elements with [index] for actions. Start here.',
       schema: BrowserGetPageStateSchema,
     })
   );
 
+  // browser_keypress - for keyboard shortcuts
+  tools.push(
+    tool(createToolFunction('keypress'), {
+      name: EBrowserTools.KEYPRESS,
+      description:
+        'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',
+      schema: BrowserKeypressSchema,
+    })
+  );
+
+  // browser_switch_tab - for switching between tabs
+  tools.push(
+    tool(createToolFunction('switch_tab'), {
+      name: EBrowserTools.SWITCH_TAB,
+      description:
+        'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',
+      schema: BrowserSwitchTabSchema,
+    })
+  );
+
   return tools;
 }

package/src/tools/CodeExecutor.ts

@@ -17,7 +17,7 @@ export const getCodeBaseURL = (): string =>
 const imageMessage = 'Image is already displayed to the user';
 const otherMessage = 'File is already downloaded by the user';
 const accessMessage =
-  'Note: Files are READ-ONLY. Save changes to NEW filenames. To access these files in future executions, provide the `session_id` as a parameter (not in your code).';
+  'Note: Files from previous executions are automatically available and can be modified.';
 const emptyOutputMessage =
   'stdout: Empty. Ensure you\'re writing output explicitly.\n';
 
@@ -41,7 +41,8 @@ const CodeExecutionToolSchema = z.object({
   code: z.string()
     .describe(`The complete, self-contained code to execute, without any truncation or minimization.
 - The environment is stateless; variables and imports don't persist between executions.
-- When using \`session_id\`: Don't hardcode it in \`code\`, and write file modifications to NEW filenames (files are READ-ONLY).
+- Generated files from previous executions are automatically available in "/mnt/data/".
+- Files from previous executions are automatically available and can be modified in place.
 - Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.
 - Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.
 - IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.
@@ -50,17 +51,6 @@ const CodeExecutionToolSchema = z.object({
 - js: use the \`console\` or \`process\` methods for all outputs.
 - r: IMPORTANT: No X11 display available. ALL graphics MUST use Cairo library (library(Cairo)).
 - Other languages: use appropriate output functions.`),
-  session_id: z
-    .string()
-    .optional()
-    .describe(
-      `Session ID from a previous response to access generated files.
-- Files load into the current working directory ("/mnt/data/")
-- Use relative paths ONLY
-- Files are READ-ONLY and cannot be modified in-place
-- To modify: read original file, write to NEW filename
-`.trim()
-    ),
   args: z
     .array(z.string())
     .optional()
@@ -107,15 +97,33 @@ Rules:
 `.trim();
 
   return tool<typeof CodeExecutionToolSchema>(
-    async ({ lang, code, session_id, ...rest }) => {
-      const postData = {
+    async ({ lang, code, ...rest }, config) => {
+      /**
+       * Extract session context from config.toolCall (injected by ToolNode).
+       * - session_id: For API to associate with previous session
+       * - _injected_files: File refs to pass directly (avoids /files endpoint race condition)
+       */
+      const { session_id, _injected_files } = (config.toolCall ?? {}) as {
+        session_id?: string;
+        _injected_files?: t.CodeEnvFile[];
+      };
+
+      const postData: Record<string, unknown> = {
         lang,
         code,
         ...rest,
         ...params,
       };
 
-      if (session_id != null && session_id.length > 0) {
+      /**
+       * File injection priority:
+       * 1. Use _injected_files from ToolNode (avoids /files endpoint race condition)
+       * 2. Fall back to fetching from /files endpoint if session_id provided but no injected files
+       */
+      if (_injected_files && _injected_files.length > 0) {
+        postData.files = _injected_files;
+      } else if (session_id != null && session_id.length > 0) {
+        /** Fallback: fetch from /files endpoint (may have race condition issues) */
         try {
           const filesEndpoint = `${baseEndpoint}/files/${session_id}?detail=full`;
           const fetchOptions: RequestInit = {
@@ -140,7 +148,6 @@ Rules:
           const files = await response.json();
           if (Array.isArray(files) && files.length > 0) {
             const fileReferences: t.CodeEnvFile[] = files.map((file) => {
-              // Extract the ID from the file name (part after session ID prefix and before extension)
               const nameParts = file.name.split('/');
               const id = nameParts.length > 1 ? nameParts[1].split('.')[0] : '';
 
@@ -151,11 +158,7 @@ Rules:
               };
             });
 
-            if (!postData.files) {
-              postData.files = fileReferences;
-            } else if (Array.isArray(postData.files)) {
-              postData.files = [...postData.files, ...fileReferences];
-            }
+            postData.files = fileReferences;
           }
         } catch {
           // eslint-disable-next-line no-console
@@ -204,7 +207,7 @@ Rules:
             }
           }
 
-          formattedOutput += `\nsession_id: ${result.session_id}\n\n${accessMessage}`;
+          formattedOutput += `\n\n${accessMessage}`;
           return [
             formattedOutput.trim(),
             {

package/src/tools/ProgrammaticToolCalling.ts

@@ -19,7 +19,7 @@ config();
 const imageMessage = 'Image is already displayed to the user';
 const otherMessage = 'File is already downloaded by the user';
 const accessMessage =
-  'Note: Files are READ-ONLY. Save changes to NEW filenames. To access these files in future executions, provide the `session_id` as a parameter (not in your code).';
+  'Note: Files from previous executions are automatically available and can be modified.';
 const emptyOutputMessage =
   'stdout: Empty. Ensure you\'re writing output explicitly.\n';
 
@@ -68,12 +68,6 @@ Rules:
 - Tools are pre-defined—DO NOT write function definitions
 - Only print() output returns to the model`
     ),
-  session_id: z
-    .string()
-    .optional()
-    .describe(
-      'Session ID for file access (same as regular code execution). Files load into /mnt/data/ and are READ-ONLY.'
-    ),
   timeout: z
     .number()
     .int()
@@ -542,7 +536,7 @@ export function formatCompletedResponse(
       }
     }
 
-    formatted += `\nsession_id: ${response.session_id}\n\n${accessMessage}`;
+    formatted += `\n\n${accessMessage}`;
   }
 
   return [
@@ -613,7 +607,7 @@ Rules:
 - Do NOT define \`async def main()\` or call \`asyncio.run()\`—just write code with await
 - Tools are pre-defined—DO NOT write function definitions
 - Only \`print()\` output returns; tool results are raw dicts/lists/strings
-- Use \`session_id\` param for file persistence across calls
+- Generated files are automatically available in /mnt/data/ for subsequent executions
 - Tool names normalized: hyphens→underscores, keywords get \`_tool\` suffix
 
 When to use: loops, conditionals, parallel (\`asyncio.gather\`), multi-step pipelines.
@@ -624,11 +618,15 @@ Example (complete pipeline):
 
   return tool<typeof ProgrammaticToolCallingSchema>(
     async (params, config) => {
-      const { code, session_id, timeout = DEFAULT_TIMEOUT } = params;
+      const { code, timeout = DEFAULT_TIMEOUT } = params;
 
       // Extra params injected by ToolNode (follows web_search pattern)
-      const { toolMap, toolDefs } = (config.toolCall ?? {}) as ToolCall &
-        Partial<t.ProgrammaticCache>;
+      const { toolMap, toolDefs, session_id, _injected_files } =
+        (config.toolCall ?? {}) as ToolCall &
+          Partial<t.ProgrammaticCache> & {
+            session_id?: string;
+            _injected_files?: t.CodeEnvFile[];
+          };
 
       if (toolMap == null || toolMap.size === 0) {
         throw new Error(
@@ -661,9 +659,15 @@ Example (complete pipeline):
           );
         }
 
-        // Fetch files from previous session if session_id is provided
+        /**
+         * File injection priority:
+         * 1. Use _injected_files from ToolNode (avoids /files endpoint race condition)
+         * 2. Fall back to fetching from /files endpoint if session_id provided but no injected files
+         */
         let files: t.CodeEnvFile[] | undefined;
-        if (session_id != null && session_id.length > 0) {
+        if (_injected_files && _injected_files.length > 0) {
+          files = _injected_files;
+        } else if (session_id != null && session_id.length > 0) {
           files = await fetchSessionFiles(baseUrl, apiKey, session_id, proxy);
         }
 

package/src/tools/ToolNode.ts

@@ -80,6 +80,8 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
   private toolRegistry?: t.LCToolRegistry;
   /** Cached programmatic tools (computed once on first PTC call) */
   private programmaticCache?: t.ProgrammaticCache;
+  /** Reference to Graph's sessions map for automatic session injection */
+  private sessions?: t.ToolSessionMap;
 
   constructor({
     tools,
@@ -91,6 +93,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     handleToolErrors,
     loadRuntimeTools,
     toolRegistry,
+    sessions,
   }: t.ToolNodeConstructorParams) {
     super({ name, tags, func: (input, config) => this.run(input, config) });
     this.toolMap = toolMap ?? new Map(tools.map((tool) => [tool.name, tool]));
@@ -100,6 +103,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     this.errorHandler = errorHandler;
     this.toolUsageCount = new Map<string, number>();
     this.toolRegistry = toolRegistry;
+    this.sessions = sessions;
   }
 
   /**
@@ -170,13 +174,46 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           toolMap,
           toolDefs,
         };
-      } else if (call.name === Constants.TOOL_SEARCH_REGEX) {
+      } else if (call.name === Constants.TOOL_SEARCH) {
         invokeParams = {
           ...invokeParams,
           toolRegistry: this.toolRegistry,
         };
       }
 
+      /**
+       * Inject session context for code execution tools when available.
+       * Each file uses its own session_id (supporting multi-session file tracking).
+       * Both session_id and _injected_files are injected directly to invokeParams
+       * (not inside args) so they bypass Zod schema validation and reach config.toolCall.
+       */
+      if (
+        call.name === Constants.EXECUTE_CODE ||
+        call.name === Constants.PROGRAMMATIC_TOOL_CALLING
+      ) {
+        const codeSession = this.sessions?.get(Constants.EXECUTE_CODE) as
+          | t.CodeSessionContext
+          | undefined;
+        if (codeSession?.files != null && codeSession.files.length > 0) {
+          /**
+           * Convert tracked files to CodeEnvFile format for the API.
+           * Each file uses its own session_id (set when file was created).
+           * This supports files from multiple parallel/sequential executions.
+           */
+          const fileRefs: t.CodeEnvFile[] = codeSession.files.map((file) => ({
+            session_id: file.session_id ?? codeSession.session_id,
+            id: file.id,
+            name: file.name,
+          }));
+          /** Inject latest session_id and files - bypasses Zod, reaches config.toolCall */
+          invokeParams = {
+            ...invokeParams,
+            session_id: codeSession.session_id,
+            _injected_files: fileRefs,
+          };
+        }
+      }
+
       const output = await tool.invoke(invokeParams, config);
 
       // Handle Command outputs directly
@@ -354,6 +391,13 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     )[] = [];
     let parentCommand: Command | null = null;
 
+    /**
+     * Collect handoff commands (Commands with string goto and Command.PARENT)
+     * for potential parallel handoff aggregation
+     */
+    const handoffCommands: Command[] = [];
+    const nonCommandOutputs: BaseMessage[] = [];
+
     for (const output of outputs) {
       if (isCommand(output)) {
         if (
@@ -361,6 +405,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           Array.isArray(output.goto) &&
           output.goto.every((send): send is Send => isSend(send))
         ) {
+          /** Aggregate Send-based commands */
           if (parentCommand) {
             (parentCommand.goto as Send[]).push(...(output.goto as Send[]));
           } else {
@@ -369,16 +414,84 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
               goto: output.goto,
             });
           }
+        } else if (output.graph === Command.PARENT) {
+          /**
+           * Handoff Command with destination.
+           * Handle both string ('agent') and array (['agent']) formats.
+           * Collect for potential parallel aggregation.
+           */
+          const goto = output.goto;
+          const isSingleStringDest = typeof goto === 'string';
+          const isSingleArrayDest =
+            Array.isArray(goto) &&
+            goto.length === 1 &&
+            typeof goto[0] === 'string';
+
+          if (isSingleStringDest || isSingleArrayDest) {
+            handoffCommands.push(output);
+          } else {
+            /** Multi-destination or other command - pass through */
+            combinedOutputs.push(output);
+          }
         } else {
+          /** Other commands - pass through */
           combinedOutputs.push(output);
         }
       } else {
+        nonCommandOutputs.push(output);
         combinedOutputs.push(
           Array.isArray(input) ? [output] : { messages: [output] }
         );
       }
     }
 
+    /**
+     * Handle handoff commands - convert to Send objects for parallel execution
+     * when multiple handoffs are requested
+     */
+    if (handoffCommands.length > 1) {
+      /**
+       * Multiple parallel handoffs - convert to Send objects.
+       * Each Send carries its own state with the appropriate messages.
+       * This enables LLM-initiated parallel execution when calling multiple
+       * transfer tools simultaneously.
+       */
+
+      /** Collect all destinations for sibling tracking */
+      const allDestinations = handoffCommands.map((cmd) => {
+        const goto = cmd.goto;
+        return typeof goto === 'string' ? goto : (goto as string[])[0];
+      });
+
+      const sends = handoffCommands.map((cmd, idx) => {
+        const destination = allDestinations[idx];
+        /** Get siblings (other destinations, not this one) */
+        const siblings = allDestinations.filter((d) => d !== destination);
+
+        /** Add siblings to ToolMessage additional_kwargs */
+        const update = cmd.update as { messages?: BaseMessage[] } | undefined;
+        if (update && update.messages) {
+          for (const msg of update.messages) {
+            if (msg.getType() === 'tool') {
+              (msg as ToolMessage).additional_kwargs.handoff_parallel_siblings =
+                siblings;
+            }
+          }
+        }
+
+        return new Send(destination, cmd.update);
+      });
+
+      const parallelCommand = new Command({
+        graph: Command.PARENT,
+        goto: sends,
+      });
+      combinedOutputs.push(parallelCommand);
+    } else if (handoffCommands.length === 1) {
+      /** Single handoff - pass through as-is */
+      combinedOutputs.push(handoffCommands[0]);
+    }
+
     if (parentCommand) {
       combinedOutputs.push(parentCommand);
     }