illuma-agents 1.0.20 → 1.0.22

package/src/run.ts

@@ -4,6 +4,8 @@ import { CallbackHandler } from '@langfuse/langchain';
 import { PromptTemplate } from '@langchain/core/prompts';
 import { RunnableLambda } from '@langchain/core/runnables';
 import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
+import { MemorySaver } from '@langchain/langgraph-checkpoint';
+import { Command, INTERRUPT, isInterrupted } from '@langchain/langgraph';
 import type {
   MessageContentComplex,
   BaseMessage,
@@ -22,6 +24,7 @@ import { StandardGraph } from '@/graphs/Graph';
 import { HandlerRegistry } from '@/events';
 import { isOpenAILike } from '@/utils/llm';
 import { isPresent } from '@/utils/misc';
+import type { BrowserInterrupt, BrowserActionResult } from '@/tools/BrowserInterruptTools';
 
 export const defaultOmitOptions = new Set([
   'stream',
@@ -36,11 +39,35 @@ export const defaultOmitOptions = new Set([
   'additionalModelRequestFields',
 ]);
 
+/** Global checkpointer store for browser mode (keyed by runId) */
+const browserCheckpointers = new Map<string, MemorySaver>();
+
+/**
+ * Get or create a checkpointer for browser mode
+ */
+export function getBrowserCheckpointer(runId: string): MemorySaver {
+  let checkpointer = browserCheckpointers.get(runId);
+  if (!checkpointer) {
+    checkpointer = new MemorySaver();
+    browserCheckpointers.set(runId, checkpointer);
+  }
+  return checkpointer;
+}
+
+/**
+ * Clean up a browser checkpointer when done
+ */
+export function cleanupBrowserCheckpointer(runId: string): void {
+  browserCheckpointers.delete(runId);
+}
+
 export class Run<_T extends t.BaseGraphState> {
   id: string;
   private tokenCounter?: t.TokenCounter;
   private handlerRegistry?: HandlerRegistry;
   private indexTokenCountMap?: Record<string, number>;
+  /** Whether this run is in browser extension mode */
+  browserMode: boolean = false;
   graphRunnable?: t.CompiledStateWorkflow;
   Graph: StandardGraph | MultiAgentGraph | undefined;
   returnContent: boolean = false;
@@ -54,6 +81,7 @@ export class Run<_T extends t.BaseGraphState> {
     this.id = runId;
     this.tokenCounter = config.tokenCounter;
     this.indexTokenCountMap = config.indexTokenCountMap;
+    this.browserMode = config.browserMode ?? false;
 
     const handlerRegistry = new HandlerRegistry();
 
@@ -73,16 +101,24 @@ export class Run<_T extends t.BaseGraphState> {
 
     /** Handle different graph types */
     if (config.graphConfig.type === 'multi-agent') {
+      // For multi-agent, browser mode checkpointer support not yet implemented
       this.graphRunnable = this.createMultiAgentGraph(config.graphConfig);
       if (this.Graph) {
         this.Graph.handlerRegistry = handlerRegistry;
       }
     } else {
       /** Default to legacy graph for 'standard' or undefined type */
-      this.graphRunnable = this.createLegacyGraph(config.graphConfig);
+      // In browser mode, inject checkpointer into compileOptions BEFORE creating the graph
+      const graphConfig = { ...config.graphConfig };
+      if (this.browserMode) {
+        const checkpointer = getBrowserCheckpointer(runId);
+        graphConfig.compileOptions = {
+          ...graphConfig.compileOptions,
+          checkpointer,
+        };
+      }
+      this.graphRunnable = this.createLegacyGraph(graphConfig);
       if (this.Graph) {
-        this.Graph.compileOptions =
-          config.graphConfig.compileOptions ?? this.Graph.compileOptions;
         this.Graph.handlerRegistry = handlerRegistry;
       }
     }
@@ -453,4 +489,141 @@ export class Run<_T extends t.BaseGraphState> {
       );
     }
   }
+
+  /**
+   * Process stream with browser interrupt support.
+   * Uses regular stream() instead of streamEvents() to properly detect interrupts.
+   * Returns interrupt data when graph is paused waiting for browser action.
+   */
+  async *processBrowserStream(
+    inputs: t.IState,
+    config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string },
+    streamOptions?: t.EventStreamOptions
+  ): AsyncGenerator<
+    | { type: 'event'; data: unknown }
+    | { type: 'interrupt'; data: BrowserInterrupt }
+    | { type: 'done'; data?: MessageContentComplex[] }
+  > {
+    if (this.graphRunnable == null) {
+      throw new Error(
+        'Run not initialized. Make sure to use Run.create() to instantiate the Run.'
+      );
+    }
+    if (!this.Graph) {
+      throw new Error(
+        'Graph not initialized. Make sure to use Run.create() to instantiate the Run.'
+      );
+    }
+
+    this.Graph.resetValues(streamOptions?.keepContent);
+
+    if (!this.id) {
+      throw new Error('Run ID not provided');
+    }
+
+    config.run_id = this.id;
+    // Set up thread_id for checkpointing (required for interrupt/resume)
+    config.configurable = Object.assign(config.configurable ?? {}, {
+      run_id: this.id,
+      thread_id: this.id, // Use run ID as thread ID for browser sessions
+    });
+
+    // Use the values stream mode to detect interrupts
+    const stream = await this.graphRunnable.stream(inputs, {
+      ...config,
+      streamMode: ['values', 'updates'],
+    });
+
+    for await (const chunk of stream) {
+      // Check if this chunk contains an interrupt
+      if (isInterrupted(chunk)) {
+        const interrupts = chunk[INTERRUPT] as Array<{ value: BrowserInterrupt }>;
+        if (interrupts.length > 0) {
+          // Emit the interrupt data to the client
+          for (const interrupt of interrupts) {
+            if (interrupt.value?.type === 'browser_interrupt') {
+              yield { type: 'interrupt', data: interrupt.value };
+            }
+          }
+          // Stop yielding - graph is paused
+          return;
+        }
+      }
+
+      // Emit regular events
+      yield { type: 'event', data: chunk };
+    }
+
+    // Stream completed without interrupt
+    if (this.returnContent) {
+      yield { type: 'done', data: this.Graph.getContentParts() };
+    } else {
+      yield { type: 'done' };
+    }
+  }
+
+  /**
+   * Resume a browser stream after interrupt.
+   * Call this with the result from the browser extension.
+   */
+  async *resumeBrowserStream(
+    result: BrowserActionResult,
+    config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string }
+  ): AsyncGenerator<
+    | { type: 'event'; data: unknown }
+    | { type: 'interrupt'; data: BrowserInterrupt }
+    | { type: 'done'; data?: MessageContentComplex[] }
+  > {
+    if (this.graphRunnable == null) {
+      throw new Error(
+        'Run not initialized. Make sure to use Run.create() to instantiate the Run.'
+      );
+    }
+    if (!this.Graph) {
+      throw new Error(
+        'Graph not initialized. Make sure to use Run.create() to instantiate the Run.'
+      );
+    }
+
+    if (!this.id) {
+      throw new Error('Run ID not provided');
+    }
+
+    config.run_id = this.id;
+    config.configurable = Object.assign(config.configurable ?? {}, {
+      run_id: this.id,
+      thread_id: this.id,
+    });
+
+    // Use Command to resume with the browser result
+    const resumeCommand = new Command({ resume: result });
+
+    const stream = await this.graphRunnable.stream(resumeCommand, {
+      ...config,
+      streamMode: ['values', 'updates'],
+    });
+
+    for await (const chunk of stream) {
+      // Check if this chunk contains another interrupt
+      if (isInterrupted(chunk)) {
+        const interrupts = chunk[INTERRUPT] as Array<{ value: BrowserInterrupt }>;
+        if (interrupts.length > 0) {
+          for (const interrupt of interrupts) {
+            if (interrupt.value?.type === 'browser_interrupt') {
+              yield { type: 'interrupt', data: interrupt.value };
+            }
+          }
+          return;
+        }
+      }
+
+      yield { type: 'event', data: chunk };
+    }
+
+    if (this.returnContent) {
+      yield { type: 'done', data: this.Graph.getContentParts() };
+    } else {
+      yield { type: 'done' };
+    }
+  }
 }

package/src/specs/browser-interrupt-tools.test.ts

@@ -0,0 +1,235 @@
+/**
+ * Tests for Browser Interrupt Tools
+ * 
+ * These tests verify:
+ * 1. Tool creation and schema validation
+ * 2. Interrupt type exports and guards
+ * 3. Tool naming consistency
+ */
+
+import { z } from 'zod';
+import {
+  createBrowserInterruptTools,
+  createBrowserNavigateInterruptTool,
+  createBrowserClickInterruptTool,
+  createBrowserTypeInterruptTool,
+  createBrowserGetPageStateInterruptTool,
+  createBrowserScrollInterruptTool,
+  createBrowserExtractInterruptTool,
+  createBrowserHoverInterruptTool,
+  createBrowserWaitInterruptTool,
+  createBrowserGoBackInterruptTool,
+  createBrowserScreenshotInterruptTool,
+  isBrowserInterrupt,
+  isBrowserInterruptToolCall,
+  BROWSER_INTERRUPT_TOOL_NAMES,
+  EBrowserInterruptTools,
+  type BrowserInterrupt,
+  type BrowserActionResult,
+} from '../tools/BrowserInterruptTools';
+
+describe('BrowserInterruptTools', () => {
+  describe('createBrowserInterruptTools', () => {
+    it('should create all 10 browser tools', () => {
+      const tools = createBrowserInterruptTools();
+      expect(tools).toHaveLength(10);
+    });
+
+    it('should have unique tool names', () => {
+      const tools = createBrowserInterruptTools();
+      const names = tools.map(t => t.name);
+      const uniqueNames = new Set(names);
+      expect(uniqueNames.size).toBe(names.length);
+    });
+
+    it('should match BROWSER_INTERRUPT_TOOL_NAMES', () => {
+      const tools = createBrowserInterruptTools();
+      const names = tools.map(t => t.name);
+      expect(names.sort()).toEqual([...BROWSER_INTERRUPT_TOOL_NAMES].sort());
+    });
+  });
+
+  describe('individual tool creation', () => {
+    it('should create navigate tool with correct schema', () => {
+      const tool = createBrowserNavigateInterruptTool();
+      expect(tool.name).toBe('browser_navigate');
+      expect(tool.schema).toBeDefined();
+      
+      // Verify schema accepts valid input
+      const schema = tool.schema as z.ZodObject<any>;
+      const result = schema.safeParse({ url: 'https://example.com' });
+      expect(result.success).toBe(true);
+    });
+
+    it('should create click tool with correct schema', () => {
+      const tool = createBrowserClickInterruptTool();
+      expect(tool.name).toBe('browser_click');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      // Click by index
+      expect(schema.safeParse({ index: 5 }).success).toBe(true);
+      // Click by coordinates
+      expect(schema.safeParse({ coordinates: { x: 100, y: 200 } }).success).toBe(true);
+    });
+
+    it('should create type tool with correct schema', () => {
+      const tool = createBrowserTypeInterruptTool();
+      expect(tool.name).toBe('browser_type');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({ index: 3, text: 'hello world' }).success).toBe(true);
+      expect(schema.safeParse({ index: 3, text: 'search', pressEnter: true }).success).toBe(true);
+    });
+
+    it('should create get_page_state tool with correct schema', () => {
+      const tool = createBrowserGetPageStateInterruptTool();
+      expect(tool.name).toBe('browser_get_page_state');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({}).success).toBe(true);
+      expect(schema.safeParse({ reason: 'checking elements' }).success).toBe(true);
+    });
+
+    it('should create scroll tool with correct schema', () => {
+      const tool = createBrowserScrollInterruptTool();
+      expect(tool.name).toBe('browser_scroll');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({ direction: 'down' }).success).toBe(true);
+      expect(schema.safeParse({ direction: 'up', amount: 500 }).success).toBe(true);
+      expect(schema.safeParse({ direction: 'invalid' }).success).toBe(false);
+    });
+
+    it('should create extract tool with correct schema', () => {
+      const tool = createBrowserExtractInterruptTool();
+      expect(tool.name).toBe('browser_extract');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({}).success).toBe(true);
+      expect(schema.safeParse({ query: 'product prices' }).success).toBe(true);
+    });
+
+    it('should create hover tool with correct schema', () => {
+      const tool = createBrowserHoverInterruptTool();
+      expect(tool.name).toBe('browser_hover');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({ index: 5 }).success).toBe(true);
+      expect(schema.safeParse({}).success).toBe(false); // index is required
+    });
+
+    it('should create wait tool with correct schema', () => {
+      const tool = createBrowserWaitInterruptTool();
+      expect(tool.name).toBe('browser_wait');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({}).success).toBe(true);
+      expect(schema.safeParse({ duration: 2000 }).success).toBe(true);
+    });
+
+    it('should create back tool with correct schema', () => {
+      const tool = createBrowserGoBackInterruptTool();
+      expect(tool.name).toBe('browser_back');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({}).success).toBe(true);
+    });
+
+    it('should create screenshot tool with correct schema', () => {
+      const tool = createBrowserScreenshotInterruptTool();
+      expect(tool.name).toBe('browser_screenshot');
+      
+      const schema = tool.schema as z.ZodObject<any>;
+      expect(schema.safeParse({}).success).toBe(true);
+      expect(schema.safeParse({ fullPage: true }).success).toBe(true);
+    });
+  });
+
+  describe('isBrowserInterrupt', () => {
+    it('should return true for valid browser interrupt', () => {
+      const interrupt: BrowserInterrupt = {
+        type: 'browser_interrupt',
+        action: { type: 'navigate', url: 'https://example.com' },
+        interruptId: 'test_123',
+      };
+      expect(isBrowserInterrupt(interrupt)).toBe(true);
+    });
+
+    it('should return false for null/undefined', () => {
+      expect(isBrowserInterrupt(null)).toBe(false);
+      expect(isBrowserInterrupt(undefined)).toBe(false);
+    });
+
+    it('should return false for non-object', () => {
+      expect(isBrowserInterrupt('string')).toBe(false);
+      expect(isBrowserInterrupt(123)).toBe(false);
+    });
+
+    it('should return false for wrong type field', () => {
+      expect(isBrowserInterrupt({ type: 'other_type' })).toBe(false);
+    });
+  });
+
+  describe('isBrowserInterruptToolCall', () => {
+    it('should return true for valid tool names', () => {
+      expect(isBrowserInterruptToolCall('browser_navigate')).toBe(true);
+      expect(isBrowserInterruptToolCall('browser_click')).toBe(true);
+      expect(isBrowserInterruptToolCall('browser_type')).toBe(true);
+      expect(isBrowserInterruptToolCall('browser_get_page_state')).toBe(true);
+    });
+
+    it('should return false for invalid tool names', () => {
+      expect(isBrowserInterruptToolCall('invalid_tool')).toBe(false);
+      expect(isBrowserInterruptToolCall('browser_invalid')).toBe(false);
+      expect(isBrowserInterruptToolCall('')).toBe(false);
+    });
+  });
+
+  describe('EBrowserInterruptTools enum', () => {
+    it('should have all expected tool names', () => {
+      expect(EBrowserInterruptTools.CLICK).toBe('browser_click');
+      expect(EBrowserInterruptTools.TYPE).toBe('browser_type');
+      expect(EBrowserInterruptTools.NAVIGATE).toBe('browser_navigate');
+      expect(EBrowserInterruptTools.SCROLL).toBe('browser_scroll');
+      expect(EBrowserInterruptTools.EXTRACT).toBe('browser_extract');
+      expect(EBrowserInterruptTools.HOVER).toBe('browser_hover');
+      expect(EBrowserInterruptTools.WAIT).toBe('browser_wait');
+      expect(EBrowserInterruptTools.BACK).toBe('browser_back');
+      expect(EBrowserInterruptTools.SCREENSHOT).toBe('browser_screenshot');
+      expect(EBrowserInterruptTools.GET_PAGE_STATE).toBe('browser_get_page_state');
+    });
+  });
+
+  describe('BrowserActionResult type', () => {
+    it('should accept success result', () => {
+      const result: BrowserActionResult = {
+        success: true,
+      };
+      expect(result.success).toBe(true);
+    });
+
+    it('should accept result with page state', () => {
+      const result: BrowserActionResult = {
+        success: true,
+        pageState: {
+          url: 'https://example.com',
+          title: 'Example',
+          elementList: '[0] <button>Click me</button>',
+          elementCount: 1,
+          scrollPosition: 0,
+          scrollHeight: 1000,
+          viewportHeight: 800,
+        },
+      };
+      expect(result.pageState?.elementCount).toBe(1);
+    });
+
+    it('should accept error result', () => {
+      const result: BrowserActionResult = {
+        success: false,
+        error: 'Element not found',
+      };
+      expect(result.error).toBe('Element not found');
+    });
+  });
+});