@specsage/cli 0.1.15 → 0.1.17

package/lib/browser.js

@@ -6,7 +6,7 @@
 import { chromium } from "playwright";
 import fs from "fs";
 import path from "path";
-import { fileURLToPath } from "url";
+import { fileURLToPath, pathToFileURL } from "url";
 import {
   setupDialogHandler,
   hasDialog,
@@ -35,6 +35,15 @@ const tmpDir = tempDirArgIndex !== -1 && process.argv[tempDirArgIndex + 1]
   ? process.argv[tempDirArgIndex + 1]
   : path.join(scriptDir, '..', 'tmp');
 
+// Parse --eval-capture argument for eval capture module path
+const evalCaptureArgIndex = process.argv.indexOf('--eval-capture');
+const evalCapturePath = evalCaptureArgIndex !== -1 && process.argv[evalCaptureArgIndex + 1]
+  ? process.argv[evalCaptureArgIndex + 1]
+  : null;
+
+let evalCapture = null;
+let evalCaptureStepIndex = 0;
+
 async function init() {
   const visible = process.argv.includes('--visible');
   const record = process.argv.includes('--record');
@@ -76,6 +85,19 @@ async function init() {
 
   // Set up dialog handling - captures dialogs for AI to see and respond to
   setupDialogHandler(page);
+
+  // Load eval capture module if specified
+  if (evalCapturePath) {
+    try {
+      const evalCaptureModule = await import(pathToFileURL(evalCapturePath));
+      evalCapture = evalCaptureModule.default || evalCaptureModule;
+      console.error(`[browser.js] Eval capture loaded: ${evalCapturePath}`);
+    } catch (err) {
+      console.error(`[browser.js] Failed to load eval capture: ${evalCapturePath}`);
+      console.error(err);
+      process.exit(1);
+    }
+  }
 }
 
 function send(response) {
@@ -838,21 +860,70 @@ async function handleCommand(msg) {
         const centerX = x + w / 2;
         centerY = y + h / 2;
 
-        // Use Playwright's selectOption by locating the element at the position
-        const selectEl = page.locator('select').filter({
-          has: page.locator(`text="${value}"`)
-        }).or(page.locator('select')).first();
-
-        // Try to find the exact select element by position
-        const selectAtPoint = await page.evaluateHandle(
+        // Try to find a native <select> element by position
+        const selectHandle = await page.evaluateHandle(
           ({ x, y }) => document.elementFromPoint(x, y)?.closest('select'),
           { x: centerX, y: centerY }
         );
 
-        if (selectAtPoint) {
-          await selectAtPoint.selectOption({ label: value });
+        // evaluateHandle returns a JSHandle even for null — check the underlying value
+        const isNativeSelect = await selectHandle.evaluate(el => el !== null).catch(() => false);
+
+        if (isNativeSelect) {
+          await selectHandle.selectOption({ label: value });
         } else {
-          throw new Error(`No select element found at position for ${element_id}`);
+          // Custom combobox/select: click to open, then poll for the option.
+          // Options may load asynchronously (e.g. MUI selects that fetch data
+          // after a dependent field changes), so we retry instead of a fixed wait.
+
+          // Grab aria-controls before clicking so we can scope to the correct listbox
+          const listboxId = await page.evaluate(
+            ({ x, y }) => {
+              const el = document.elementFromPoint(x, y)?.closest('[role="combobox"]');
+              return el?.getAttribute('aria-controls') || null;
+            },
+            { x: centerX, y: centerY }
+          );
+
+          await page.mouse.click(centerX, centerY);
+
+          const SELECT_TIMEOUT = 10000;
+          const POLL_INTERVAL = 300;
+          const deadline = Date.now() + SELECT_TIMEOUT;
+          let clicked = false;
+
+          // Build a scoped listbox locator: prefer aria-controls ID, fall back to last visible listbox
+          const listboxLocator = listboxId
+            ? page.locator(`#${CSS.escape(listboxId)}`)
+            : page.locator('[role="listbox"]').last();
+
+          while (Date.now() < deadline) {
+            await new Promise(r => setTimeout(r, POLL_INTERVAL));
+
+            // Try scoped role=option first (MUI, Radix, Headless UI, etc.)
+            const optionLocator = listboxLocator.getByRole('option', { name: value, exact: true });
+            const optionCount = await optionLocator.count();
+            if (optionCount > 0) {
+              await optionLocator.first().click();
+              clicked = true;
+              break;
+            }
+
+            // Fallback: text match inside the scoped listbox or any ul within it
+            const textLocator = listboxLocator.locator(`text="${value}"`).or(
+              page.locator(`ul >> text="${value}"`)
+            ).first();
+            const textCount = await textLocator.count();
+            if (textCount > 0) {
+              await textLocator.click();
+              clicked = true;
+              break;
+            }
+          }
+
+          if (!clicked) {
+            throw new Error(`Option "${value}" not found in custom select after ${SELECT_TIMEOUT}ms`);
+          }
         }
 
         // Wait for page to settle after selection (form updates, validation)
@@ -1048,6 +1119,23 @@ async function handleCommand(msg) {
         throw new Error(`Unknown command: ${command}`);
     }
 
+    // Call eval capture hook after any command that produced real page elements
+    if (evalCapture?.afterStep && result.elements && !result.dialog_blocking) {
+      try {
+        await evalCapture.afterStep({
+          page,
+          stepIndex: evalCaptureStepIndex,
+          elements: result.elements,
+          command,
+          params,
+          evalCaptureData: msg.eval_capture_data || null
+        });
+        evalCaptureStepIndex++;
+      } catch (err) {
+        console.error('[browser.js] Eval capture afterStep error:', err);
+      }
+    }
+
     send({ request_id, ok: true, result, error: null });
 
   } catch (err) {

package/lib/runner.rb

@@ -27,7 +27,7 @@ class Runner
   # Initialize runner with scenario data from server
   # @param all_scenarios [Hash] optional map of scenario_id => scenario_data for pre-scenario lookup
   # @param step_client [StepClient, DirectStepClient] optional pre-configured client for step processing
-  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, all_scenarios: nil, step_client: nil)
+  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, all_scenarios: nil, step_client: nil, eval_capture: nil)
     @scenario = normalize_scenario_data(scenario_data)
     @scenario_id = @scenario['id']
     @scenario_name = @scenario['name'] || @scenario['id'] || 'unnamed'
@@ -46,6 +46,7 @@ class Runner
     @max_steps = nil # Max browser actions allowed, received from server on first step
     @temp_dir = nil # Unique temp directory for this runner's video recording
     @all_scenarios = all_scenarios || {} # Map of scenario_id => scenario_data for pre-scenario lookup
+    @eval_capture = eval_capture # Path to eval capture JS module (passed as --eval-capture to browser.js)
   end
 
   def run
@@ -161,6 +162,11 @@ class Runner
     scenario_credentials = {}
     scenario_max_steps = nil
 
+    # Build eval capture metadata (only when eval capture is active)
+    eval_capture_data = if @eval_capture
+      { 'scenario_name' => scenario_name }
+    end
+
     loop do
       # Get next action from server
       step_result = @step_client.submit_step(
@@ -207,7 +213,12 @@ class Runner
         next
       end
 
-      result = execute_action(action)
+      # Add eval instruction to eval capture data
+      if eval_capture_data && step_result[:eval_instruction]
+        eval_capture_data['eval_instruction'] = step_result[:eval_instruction]
+      end
+
+      result = execute_action(action, eval_capture_data: eval_capture_data)
 
       # Update state from action result
       if result[:screenshot_base64]
@@ -281,6 +292,7 @@ class Runner
     args << '--visible' if @visible
     args << '--record' if @record
     args.push('--temp-dir', @temp_dir) if @temp_dir
+    args.push('--eval-capture', @eval_capture) if @eval_capture
     @node_stdin, @node_stdout, @node_stderr, @node_wait_thread = Open3.popen3(*args)
 
     # Wait for ready signal from browser.js
@@ -417,11 +429,12 @@ class Runner
     @node_wait_thread = nil
   end
 
-  def send_to_node(command, params = {}, timeout: NODE_IO_TIMEOUT_SECONDS)
+  def send_to_node(command, params = {}, timeout: NODE_IO_TIMEOUT_SECONDS, eval_capture_data: nil)
     raise 'Protocol error: Node channel is poisoned, cannot send' if @node_channel_poisoned
 
     request_id = @next_request_id
     request = { request_id: request_id, command: command, params: params }
+    request[:eval_capture_data] = eval_capture_data if eval_capture_data
 
     begin
       response = Timeout.timeout(timeout) do
@@ -483,7 +496,7 @@ class Runner
     end
   end
 
-  def execute_action(action)
+  def execute_action(action, eval_capture_data: nil)
     case action['action']
     when 'navigate'
       # Substitute credentials in URL (e.g., https://<<API_KEY>>@api.example.com)
@@ -491,14 +504,14 @@ class Runner
       display_url = url # For logging (shows placeholders, not actual values)
       url = substitute_credentials(url) if contains_credential_placeholder?(url)
 
-      response = send_to_node('navigate', { url: url })
+      response = send_to_node('navigate', { url: url }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Navigated to #{display_url}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'click'
       element_id = action['element_id']
-      response = send_to_node('click_element', { element_id: element_id })
+      response = send_to_node('click_element', { element_id: element_id }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Clicked element #{element_id}", screenshot_base64: screenshot_base64, elements: elements }
@@ -509,7 +522,7 @@ class Runner
       display_value = value # For logging (shows placeholders, not actual values)
       value = substitute_credentials(value) if contains_credential_placeholder?(value)
 
-      response = send_to_node('select_option', { element_id: element_id, value: value })
+      response = send_to_node('select_option', { element_id: element_id, value: value }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Selected '#{display_value}' in element #{element_id}", screenshot_base64: screenshot_base64, elements: elements }
@@ -521,39 +534,39 @@ class Runner
       display_keys = keys # For logging (shows placeholders, not actual values)
       keys = substitute_credentials(keys) if contains_credential_placeholder?(keys)
 
-      response = send_to_node('type', { keys: keys })
+      response = send_to_node('type', { keys: keys }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Typed: #{display_keys}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'hotkey'
       keys = action['keys']
-      response = send_to_node('hotkey', { keys: keys })
+      response = send_to_node('hotkey', { keys: keys }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Hotkey: #{keys}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'wait'
-      response = send_to_node('wait', { ms: action['ms'] })
+      response = send_to_node('wait', { ms: action['ms'] }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Waited #{action['ms']}ms", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'scroll'
-      response = send_to_node('scroll', { direction: action['direction'] })
+      response = send_to_node('scroll', { direction: action['direction'] }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Scrolled #{action['direction']}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'accept_dialog'
       value = action['value'] # Optional, for prompt dialogs
-      response = send_to_node('accept_dialog', { value: value })
+      response = send_to_node('accept_dialog', { value: value }, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Accepted dialog#{value ? " with value '#{value}'" : ''}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'dismiss_dialog'
-      response = send_to_node('dismiss_dialog', {})
+      response = send_to_node('dismiss_dialog', {}, eval_capture_data: eval_capture_data)
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
       { result: "Dismissed dialog", screenshot_base64: screenshot_base64, elements: elements }

package/lib/step_client.rb

@@ -71,7 +71,8 @@ class StepClient
       step_number: response["step_number"],
       max_steps: response["max_steps"],
       continue: response["continue"],
-      credentials: response["credentials"] || {}
+      credentials: response["credentials"] || {},
+      eval_instruction: response["eval_instruction"]
     }
   end
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specsage/cli",
-  "version": "0.1.15",
+  "version": "0.1.17",
   "description": "SpecSage CLI - AI-powered end-to-end testing automation (Node wrapper for Ruby CLI)",
   "type": "module",
   "bin": {