illuma-agents 1.0.19 → 1.0.21

package/src/tools/BrowserTools.ts

@@ -117,6 +117,12 @@ const BrowserScreenshotSchema = z.object({
   ),
 });
 
+const BrowserGetPageStateSchema = z.object({
+  reason: z.string().optional().describe(
+    'Why you need fresh page state (e.g., "after navigation", "to see updated elements")'
+  ),
+});
+
 // ============================================
 // Tool Implementations
 // ============================================
@@ -241,8 +247,18 @@ Use this tool when you need to:
 - Navigate to a different page
 - Open a new URL
 
+**IMPORTANT**: After calling browser_navigate, you MUST call browser_get_page_state 
+before using browser_click or browser_type. This is because navigation changes the page,
+and you need to see the new page's elements before you can interact with them.
+
 Provide the full URL including the protocol (https://).
 
+**Correct workflow**:
+1. browser_navigate({ url: "https://www.amazon.com" })
+2. browser_get_page_state({ reason: "see elements on Amazon" })
+3. Now find the search input's [index] in the returned state
+4. browser_type({ index: <search_input_index>, text: "query", pressEnter: true })
+
 Example: browser_navigate({ url: "https://www.google.com" })`,
       schema: BrowserNavigateSchema,
     }
@@ -444,6 +460,52 @@ Example: browser_screenshot({ fullPage: false })`,
   );
 }
 
+/**
+ * Browser get page state tool - gets fresh page context after navigation or actions
+ * CRITICAL: Use this after browser_navigate or any action that changes the page
+ */
+export function createBrowserGetPageStateTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema> {
+  return tool<typeof BrowserGetPageStateSchema>(
+    async ({ reason }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'get_page_state',
+          reason,
+        },
+        requiresBrowserExecution: true,
+        // Special flag: extension should inject fresh context into the conversation
+        requiresContextRefresh: true,
+        // IMPORTANT: Tell the agent to wait
+        message: 'Page state is being captured by the browser extension. The element list will be provided in the next message. DO NOT proceed with click or type actions until you receive the actual element list.',
+      });
+    },
+    {
+      name: EBrowserTools.GET_PAGE_STATE,
+      description: `Get fresh page state showing current interactive elements.
+
+**CRITICAL WORKFLOW**: After calling this tool, you MUST STOP and WAIT. The browser extension will capture the page state and return the element list. DO NOT plan any browser_click or browser_type actions in the same response - you don't have the element indices yet!
+
+**When to use**:
+- After browser_navigate (to see elements on the new page)
+- After browser_click (if it caused navigation or page changes)
+- Any time you need to see what elements are currently on the page
+
+**IMPORTANT**: This tool captures the page state asynchronously. The actual element list will be provided AFTER this tool completes. You should:
+1. Call this tool
+2. STOP and wait for the response with the element list
+3. In your NEXT response, use the element indices for click/type actions
+
+Example workflow:
+- Turn 1: browser_navigate to amazon.com, then browser_get_page_state
+- Turn 2: (After receiving element list) browser_type with the correct search input index
+
+Example: browser_get_page_state({ reason: "to see elements after navigation" })`,
+      schema: BrowserGetPageStateSchema,
+    }
+  );
+}
+
 // ============================================
 // Tool Collection
 // ============================================
@@ -467,6 +529,8 @@ export type BrowserToolsConfig = {
   enableBack?: boolean;
   /** Enable screenshot tool */
   enableScreenshot?: boolean;
+  /** Enable get page state tool */
+  enableGetPageState?: boolean;
 };
 
 /**
@@ -504,6 +568,7 @@ export function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStru
     enableWait = true,
     enableBack = true,
     enableScreenshot = true,
+    enableGetPageState = true,
   } = config;
   
   if (enableClick) tools.push(createBrowserClickTool());
@@ -515,6 +580,7 @@ export function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStru
   if (enableWait) tools.push(createBrowserWaitTool());
   if (enableBack) tools.push(createBrowserGoBackTool());
   if (enableScreenshot) tools.push(createBrowserScreenshotTool());
+  if (enableGetPageState) tools.push(createBrowserGetPageStateTool());
   
   return tools;
 }
@@ -533,6 +599,7 @@ export const EBrowserTools = {
   WAIT: 'browser_wait',
   BACK: 'browser_back',
   SCREENSHOT: 'browser_screenshot',
+  GET_PAGE_STATE: 'browser_get_page_state',
 } as const;
 
 /**
@@ -548,6 +615,7 @@ export const BROWSER_TOOL_NAMES = [
   EBrowserTools.WAIT,
   EBrowserTools.BACK,
   EBrowserTools.SCREENSHOT,
+  EBrowserTools.GET_PAGE_STATE,
 ] as const;
 
 export type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];

package/src/types/run.ts

@@ -115,6 +115,14 @@ export type RunConfig = {
   returnContent?: boolean;
   tokenCounter?: TokenCounter;
   indexTokenCountMap?: Record<string, number>;
+  /** 
+   * Enable browser extension mode with interrupt-based tool execution.
+   * When true:
+   * - Uses MemorySaver checkpointer for pause/resume
+   * - Browser tools will interrupt execution and wait for extension results
+   * - Extension must call resume endpoint with Command to continue
+   */
+  browserMode?: boolean;
 };
 
 export type ProvidedCallbacks =