@specsage/cli 0.1.12 → 0.1.14

package/lib/browser.js

@@ -128,229 +128,454 @@ async function captureState() {
   };
 }
 
-async function enumerateElements() {
-  const elements = [];
-
-  // Native interactive elements
-  const nativeSelectors = [
-    'button, [role="button"]',
-    'a, [role="link"]',
-    'input, textarea',
-    'select, [role="combobox"]',
-    '[role="option"]',
-    '[role="menuitem"]',
-    '[role="tab"]',
-    '[role="checkbox"]',
-    '[role="radio"]',
-  ];
+// ===== ARIA Snapshot: Constants & Helpers =====
+
+const INTERACTIVE_ROLES = new Set([
+  'button', 'link', 'textbox', 'checkbox', 'radio',
+  'combobox', 'option', 'menuitem', 'tab',
+  'switch', 'spinbutton', 'searchbox'
+]);
+
+const ROLE_TO_TYPE = {
+  button: 'button',
+  link: 'link',
+  textbox: 'input',
+  spinbutton: 'input',
+  searchbox: 'input',
+  checkbox: 'checkbox',
+  switch: 'checkbox',
+  radio: 'radio',
+  combobox: 'select',
+  option: 'option',
+  menuitem: 'option',
+  tab: 'tab',
+};
 
-  // Scripted interactive elements
-  const scriptedSelectors = [
-    { selector: '[onclick]', source: 'onclick' },
-    { selector: '[data-action]', source: 'data-action' },
-    { selector: '[data-testid]', source: 'data-testid' },
-    { selector: '[tabindex]', source: 'tabindex' },
-  ];
+/**
+ * Parse ariaSnapshot YAML into a tree using indentation-aware loop.
+ * Each node: { role, name, attributes, children, depth }
+ */
+function parseAriaSnapshot(yaml) {
+  if (!yaml) return [];
+
+  const lines = yaml.split('\n');
+  const root = { role: 'root', name: '', attributes: {}, children: [], depth: -1 };
+  const stack = [root];
+
+  for (const line of lines) {
+    const trimmed = line.trimStart();
+    if (!trimmed || !trimmed.startsWith('- ')) continue;
+
+    // Indentation depth: count leading spaces, each 2 = 1 level
+    const indent = line.length - line.trimStart().length;
+    const depth = Math.floor(indent / 2);
+
+    // Content after "- "
+    let content = trimmed.slice(2);
+
+    // Strip trailing colon (signals children) or inline text after ": "
+    const trailingColonIdx = content.lastIndexOf(':');
+    if (trailingColonIdx !== -1) {
+      const afterColon = content.slice(trailingColonIdx + 1);
+      if (afterColon === '' || afterColon.startsWith(' ')) {
+        content = content.slice(0, trailingColonIdx).trim();
+      }
+    }
+
+    // Extract role (first word)
+    let role = '';
+    const spaceIdx = content.indexOf(' ');
+    const quoteIdx = content.indexOf('"');
+    const bracketIdx = content.indexOf('[');
+
+    if (spaceIdx === -1 && quoteIdx === -1 && bracketIdx === -1) {
+      role = content;
+      content = '';
+    } else {
+      const endCandidates = [spaceIdx, quoteIdx, bracketIdx].filter(i => i !== -1);
+      const firstEnd = Math.min(...endCandidates);
+      role = content.slice(0, firstEnd);
+      content = content.slice(firstEnd).trim();
+    }
+
+    // Extract name (quoted string)
+    let name = '';
+    const nameMatch = content.match(/^"([^"]*)"/);
+    if (nameMatch) {
+      name = nameMatch[1].trim();
+      content = content.slice(nameMatch[0].length).trim();
+    }
+
+    // Extract attributes [key=value] or [key]
+    const attributes = {};
+    const attrRegex = /\[([^\]]+)\]/g;
+    let attrMatch;
+    while ((attrMatch = attrRegex.exec(content)) !== null) {
+      const attr = attrMatch[1];
+      const eqIdx = attr.indexOf('=');
+      if (eqIdx !== -1) {
+        attributes[attr.slice(0, eqIdx).trim()] = attr.slice(eqIdx + 1).trim();
+      } else {
+        attributes[attr.trim()] = true;
+      }
+    }
+
+    const node = { role, name, attributes, children: [], depth };
+
+    // Pop stack to find parent at lower depth
+    while (stack.length > 1 && stack[stack.length - 1].depth >= depth) {
+      stack.pop();
+    }
+
+    stack[stack.length - 1].children.push(node);
+    stack.push(node);
+  }
+
+  return root.children;
+}
+
+/**
+ * Walk the ARIA tree, collecting interactive nodes with their ancestor chain.
+ * parentChain enables scoped locator construction (e.g. main > navigation > link).
+ */
+function collectInteractiveNodes(nodes, parentChain = []) {
+  const result = [];
+
+  for (const node of nodes) {
+    if (INTERACTIVE_ROLES.has(node.role)) {
+      result.push({
+        role: node.role,
+        name: node.name,
+        attributes: node.attributes,
+        parentChain: [...parentChain]
+      });
+    }
+
+    // Recurse into children with this node added to the chain
+    if (node.children.length > 0) {
+      const newChain = [...parentChain, { role: node.role, name: node.name }];
+      result.push(...collectInteractiveNodes(node.children, newChain));
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Assign occurrence indices so duplicate (parentChain + role + name) combos
+ * can be disambiguated with .nth(i).
+ */
+function assignOccurrenceIndices(nodes) {
+  const counts = new Map();
+  for (const node of nodes) {
+    const key = JSON.stringify([node.parentChain, node.role, node.name]);
+    const count = counts.get(key) || 0;
+    node.occurrenceIndex = count;
+    counts.set(key, count + 1);
+  }
+  for (const node of nodes) {
+    const key = JSON.stringify([node.parentChain, node.role, node.name]);
+    node.totalOccurrences = counts.get(key);
+  }
+}
+
+/**
+ * Build a scoped Playwright locator from parentChain + interactive node.
+ * e.g. page.getByRole('main').getByRole('navigation').getByRole('link', { name: 'Home' })
+ */
+function buildScopedLocator(pg, node) {
+  let locator = pg;
+
+  for (const ancestor of node.parentChain) {
+    const opts = ancestor.name ? { name: ancestor.name, exact: true } : {};
+    locator = locator.getByRole(ancestor.role, opts);
+  }
+
+  const opts = { exact: true };
+  if (node.name) opts.name = node.name;
+  locator = locator.getByRole(node.role, opts);
+
+  if (node.totalOccurrences > 1) {
+    locator = locator.nth(node.occurrenceIndex);
+  }
+
+  return locator;
+}
+
+/**
+ * Check if two bounding boxes overlap significantly (>=50% of smaller area).
+ * Used to prevent "double vision" in the scripted fallback.
+ */
+function bboxOverlaps(a, b) {
+  const overlapX = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
+  const overlapY = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
+  const overlapArea = overlapX * overlapY;
+  const smallerArea = Math.min(a.w * a.h, b.w * b.h);
+  return smallerArea > 0 && (overlapArea / smallerArea) >= 0.5;
+}
+
+/**
+ * Extract element details from a DOM element (runs in browser context via locator.evaluate).
+ * Returns null if the element is invisible (opacity:0).
+ */
+function extractElementDetails(el) {
+  const style = getComputedStyle(el);
+  if (style.opacity === '0') return null;
+
+  const tagName = el.tagName.toLowerCase();
+  const ariaLabel = el.getAttribute('aria-label');
+  const name = el.getAttribute('name');
+  const placeholder = el.getAttribute('placeholder');
+  const type = el.getAttribute('type');
+  const href = el.getAttribute('href');
+  const visibleText = (el.textContent || '').trim().substring(0, 100);
+  const disabled = el.disabled === true || el.getAttribute('aria-disabled') === 'true';
+  const checked = el.checked === true || el.getAttribute('aria-checked') === 'true';
+
+  let backgroundColor = null;
+  const bg = style.backgroundColor;
+  if (bg && bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent') {
+    backgroundColor = bg;
+  }
+  if (!backgroundColor) {
+    for (const child of el.children) {
+      const childBg = getComputedStyle(child).backgroundColor;
+      if (childBg && childBg !== 'rgba(0, 0, 0, 0)' && childBg !== 'transparent'
+          && childBg !== 'rgb(255, 255, 255)') {
+        backgroundColor = childBg;
+        break;
+      }
+    }
+  }
+
+  // Generate domPath for scroll-into-view fallback
+  const parts = [];
+  let node = el;
+  while (node && node.nodeType === Node.ELEMENT_NODE) {
+    let seg = node.tagName.toLowerCase();
+    if (node.id) {
+      seg += `#${node.id}`;
+      parts.unshift(seg);
+      break;
+    } else {
+      const siblings = node.parentNode
+        ? Array.from(node.parentNode.children).filter(c => c.tagName === node.tagName)
+        : [];
+      if (siblings.length > 1) {
+        const index = siblings.indexOf(node) + 1;
+        seg += `:nth-of-type(${index})`;
+      }
+      parts.unshift(seg);
+    }
+    node = node.parentNode;
+  }
+
+  return {
+    tagName, ariaLabel, name, placeholder, type, href,
+    disabled, checked, visibleText,
+    accessibleName: ariaLabel || visibleText || name || placeholder || '',
+    backgroundColor,
+    domPath: parts.join(' > ')
+  };
+}
 
-  // Process native elements
-  for (const selector of nativeSelectors) {
-    const locators = page.locator(selector);
-    const count = await locators.count();
+async function enumerateElements() {
+  // ---- Step 1: Get the Aria Snapshot ----
+  let snapshot;
+  try {
+    snapshot = await page.locator('body').ariaSnapshot();
+  } catch {
+    snapshot = null;
+  }
 
-    for (let i = 0; i < count; i++) {
-      const el = locators.nth(i);
+  // ---- Step 2: Parse into tree, extract interactive nodes ----
+  const tree = parseAriaSnapshot(snapshot);
+  const interactiveNodes = collectInteractiveNodes(tree);
+  assignOccurrenceIndices(interactiveNodes);
 
+  // ---- Step 3: Resolve each to DOM via scoped locators ----
+  const ariaElements = (await Promise.all(
+    interactiveNodes.map(async (node) => {
       try {
-        const isVisible = await el.isVisible();
-        if (!isVisible) continue;
-
-        const box = await el.boundingBox();
-        if (!box) continue;
-
-        const tagName = await el.evaluate(e => e.tagName.toLowerCase());
-        const role = await el.getAttribute('role');
-        const ariaLabel = await el.getAttribute('aria-label');
-        const name = await el.getAttribute('name');
-        const placeholder = await el.getAttribute('placeholder');
-        const type = await el.getAttribute('type');
-        const href = await el.getAttribute('href');
-        const disabled = await el.isDisabled();
-
-        // Get DOM path for stable identification
-        const domPath = await el.evaluate(e => {
-          const parts = [];
-          let node = e;
-          while (node && node.nodeType === Node.ELEMENT_NODE) {
-            let selector = node.tagName.toLowerCase();
-            if (node.id) {
-              selector += `#${node.id}`;
-              parts.unshift(selector);
-              break; // ID is unique, stop here
-            } else {
-              const siblings = node.parentNode ? Array.from(node.parentNode.children).filter(c => c.tagName === node.tagName) : [];
-              if (siblings.length > 1) {
-                const index = siblings.indexOf(node) + 1;
-                selector += `:nth-of-type(${index})`;
-              }
-              parts.unshift(selector);
-            }
-            node = node.parentNode;
-          }
-          return parts.join(' > ');
-        });
+        const locator = buildScopedLocator(page, node);
 
-        let accessibleName = ariaLabel;
-        if (!accessibleName) {
-          accessibleName = await el.evaluate(e => e.textContent?.trim().substring(0, 100));
-        }
-        if (!accessibleName) {
-          accessibleName = name || placeholder || '';
-        }
+        const [box, details] = await Promise.all([
+          locator.boundingBox({ timeout: 3000 }).catch(() => null),
+          locator.evaluate(extractElementDetails).catch(() => null)
+        ]);
 
-        const visibleText = await el.evaluate(e => e.textContent?.trim().substring(0, 100) || '');
-
-        let elementType;
-        const effectiveRole = role || tagName;
-        if (tagName === 'button' || role === 'button') {
-          elementType = 'button';
-        } else if (tagName === 'a' || role === 'link') {
-          elementType = 'link';
-        } else if (tagName === 'input' || tagName === 'textarea') {
-          elementType = 'input';
-        } else if (tagName === 'select' || role === 'combobox') {
-          elementType = 'select';
-        } else if (role === 'option' || role === 'menuitem') {
-          elementType = 'option';
-        } else if (role === 'checkbox') {
-          elementType = 'checkbox';
-        } else if (role === 'radio') {
-          elementType = 'radio';
-        } else if (role === 'tab') {
-          elementType = 'tab';
-        } else {
-          elementType = tagName;
-        }
+        if (!details || !box || box.width === 0 || box.height === 0) return null;
+
+        const elementType = ROLE_TO_TYPE[node.role] || node.role;
 
-        // Generate stable key for this element
         const stableKey = generateStableKey({
-          tagName, role, name, type, placeholder, ariaLabel, href, domPath
+          tagName: details.tagName,
+          role: node.role,
+          name: details.name,
+          type: details.type,
+          placeholder: details.placeholder,
+          ariaLabel: details.ariaLabel,
+          href: details.href,
+          domPath: details.domPath
         });
 
-        elements.push({
-          id: null,  // Will be assigned after deduplication
+        return {
+          id: null,
           stable_key: stableKey,
           type: elementType,
-          role: effectiveRole,
-          accessible_name: accessibleName || '',
-          visible_text: visibleText,
-          disabled: disabled,
-          input_type: type || null,
+          role: node.role,
+          accessible_name: node.name || details.accessibleName || '',
+          visible_text: details.visibleText,
+          disabled: details.disabled || !!node.attributes.disabled,
+          checked: details.checked || !!node.attributes.checked,
+          input_type: details.type || null,
+          background_color: details.backgroundColor || null,
           bounding_box: {
             x: Math.round(box.x),
             y: Math.round(box.y),
             w: Math.round(box.width),
             h: Math.round(box.height)
           },
-          locator: el,
+          domPath: details.domPath,
           mechanism: 'native',
           source: 'native'
-        });
-      } catch (err) {
-        continue;
+        };
+      } catch {
+        return null;
       }
+    })
+  )).filter(Boolean);
+
+  // ---- Step 5: Scripted element fallback (anti-double-vision) ----
+  const scriptedRaw = await page.evaluate(() => {
+    const results = [];
+    const scriptedSelectors = [
+      { selector: '[onclick]', source: 'onclick' },
+      { selector: '[data-action]', source: 'data-action' },
+      { selector: '[data-testid]', source: 'data-testid' },
+      { selector: '[tabindex]', source: 'tabindex' },
+    ];
+
+    function isElementVisible(el) {
+      const style = getComputedStyle(el);
+      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
+      const rect = el.getBoundingClientRect();
+      return rect.width > 0 && rect.height > 0;
     }
-  }
 
-  // Process scripted elements
-  for (const { selector, source } of scriptedSelectors) {
-    const locators = page.locator(selector);
-    const count = await locators.count();
-
-    for (let i = 0; i < count; i++) {
-      const el = locators.nth(i);
-
-      try {
-        const isVisible = await el.isVisible();
-        if (!isVisible) continue;
-
-        const box = await el.boundingBox();
-        if (!box) continue;
-
-        const tagName = await el.evaluate(e => e.tagName.toLowerCase());
-        const role = await el.getAttribute('role');
-        const ariaLabel = await el.getAttribute('aria-label');
-        const name = await el.getAttribute('name');
-        const placeholder = await el.getAttribute('placeholder');
-        const type = await el.getAttribute('type');
-        const href = await el.getAttribute('href');
-        const disabled = await el.isDisabled();
-
-        // Get DOM path for stable identification
-        const domPath = await el.evaluate(e => {
-          const parts = [];
-          let node = e;
-          while (node && node.nodeType === Node.ELEMENT_NODE) {
-            let selector = node.tagName.toLowerCase();
-            if (node.id) {
-              selector += `#${node.id}`;
-              parts.unshift(selector);
-              break; // ID is unique, stop here
-            } else {
-              const siblings = node.parentNode ? Array.from(node.parentNode.children).filter(c => c.tagName === node.tagName) : [];
-              if (siblings.length > 1) {
-                const index = siblings.indexOf(node) + 1;
-                selector += `:nth-of-type(${index})`;
-              }
-              parts.unshift(selector);
-            }
-            node = node.parentNode;
+    function getDomPath(el) {
+      const parts = [];
+      let node = el;
+      while (node && node.nodeType === Node.ELEMENT_NODE) {
+        let seg = node.tagName.toLowerCase();
+        if (node.id) {
+          seg += `#${node.id}`;
+          parts.unshift(seg);
+          break;
+        } else {
+          const siblings = node.parentNode ? Array.from(node.parentNode.children).filter(c => c.tagName === node.tagName) : [];
+          if (siblings.length > 1) {
+            const index = siblings.indexOf(node) + 1;
+            seg += `:nth-of-type(${index})`;
           }
-          return parts.join(' > ');
-        });
+          parts.unshift(seg);
+        }
+        node = node.parentNode;
+      }
+      return parts.join(' > ');
+    }
 
-        let accessibleName = ariaLabel;
-        if (!accessibleName) {
-          accessibleName = await el.evaluate(e => e.textContent?.trim().substring(0, 100));
+    for (const { selector, source } of scriptedSelectors) {
+      for (const el of document.querySelectorAll(selector)) {
+        if (!isElementVisible(el)) continue;
+
+        const rect = el.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) continue;
+
+        const tagName = el.tagName.toLowerCase();
+        const role = el.getAttribute('role');
+        const ariaLabel = el.getAttribute('aria-label');
+        const name = el.getAttribute('name');
+        const placeholder = el.getAttribute('placeholder');
+        const type = el.getAttribute('type');
+        const href = el.getAttribute('href');
+        const disabled = el.disabled === true || el.getAttribute('aria-disabled') === 'true';
+        const checked = el.checked === true || el.getAttribute('aria-checked') === 'true';
+        const visibleText = (el.textContent || '').trim().substring(0, 100);
+        const accessibleName = ariaLabel || visibleText || name || placeholder || '';
+
+        let backgroundColor = null;
+        const style = getComputedStyle(el);
+        const bg = style.backgroundColor;
+        if (bg && bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent') {
+          backgroundColor = bg;
         }
-        if (!accessibleName) {
-          accessibleName = name || placeholder || '';
+        if (!backgroundColor) {
+          for (const child of el.children) {
+            const childBg = getComputedStyle(child).backgroundColor;
+            if (childBg && childBg !== 'rgba(0, 0, 0, 0)' && childBg !== 'transparent'
+                && childBg !== 'rgb(255, 255, 255)') {
+              backgroundColor = childBg;
+              break;
+            }
+          }
         }
 
-        const visibleText = await el.evaluate(e => e.textContent?.trim().substring(0, 100) || '');
-
-        const effectiveRole = role || tagName;
-
-        // Generate stable key for this element
-        const stableKey = generateStableKey({
-          tagName, role, name, type, placeholder, ariaLabel, href, domPath
-        });
-
-        elements.push({
-          id: null,  // Will be assigned after deduplication
-          stable_key: stableKey,
-          type: tagName,
-          role: effectiveRole,
-          accessible_name: accessibleName || '',
-          visible_text: visibleText,
-          disabled: disabled,
-          input_type: type || null,
+        results.push({
+          tagName, role, ariaLabel, name, placeholder, type, href,
+          disabled, checked, visibleText, accessibleName, backgroundColor,
+          domPath: getDomPath(el),
           bounding_box: {
-            x: Math.round(box.x),
-            y: Math.round(box.y),
-            w: Math.round(box.width),
-            h: Math.round(box.height)
+            x: Math.round(rect.x),
+            y: Math.round(rect.y),
+            w: Math.round(rect.width),
+            h: Math.round(rect.height)
           },
-          locator: el,
-          mechanism: 'scripted',
-          source: source
+          source
         });
-      } catch (err) {
-        continue;
       }
     }
-  }
+    return results;
+  });
 
-  // Deduplicate by bounding box (some elements match multiple selectors)
+  // Only add scripted elements whose bbox does NOT overlap with any ARIA element
+  const scriptedElements = scriptedRaw
+    .filter(raw => !ariaElements.some(aria => bboxOverlaps(aria.bounding_box, raw.bounding_box)))
+    .map(raw => {
+      const effectiveRole = raw.role || raw.tagName;
+      const stableKey = generateStableKey({
+        tagName: raw.tagName,
+        role: raw.role,
+        name: raw.name,
+        type: raw.type,
+        placeholder: raw.placeholder,
+        ariaLabel: raw.ariaLabel,
+        href: raw.href,
+        domPath: raw.domPath
+      });
+
+      return {
+        id: null,
+        stable_key: stableKey,
+        type: raw.tagName,
+        role: effectiveRole,
+        accessible_name: raw.accessibleName || '',
+        visible_text: raw.visibleText,
+        disabled: raw.disabled,
+        checked: raw.checked,
+        input_type: raw.type || null,
+        background_color: raw.backgroundColor || null,
+        bounding_box: raw.bounding_box,
+        domPath: raw.domPath,
+        mechanism: 'scripted',
+        source: raw.source
+      };
+    });
+
+  // ---- Post-processing: dedup, stable keys, display IDs (unchanged logic) ----
+  const elements = [...ariaElements, ...scriptedElements];
+
+  // Deduplicate by bounding box
   const seen = new Set();
   const uniqueElements = [];
   for (const el of elements) {
@@ -362,7 +587,6 @@ async function enumerateElements() {
   }
 
   // Disambiguate stable_keys by appending occurrence index for collisions
-  // Group elements by their base stable_key
   const keyGroups = new Map();
   for (const el of uniqueElements) {
     const baseKey = el.stable_key;
@@ -372,13 +596,11 @@ async function enumerateElements() {
     keyGroups.get(baseKey).push(el);
   }
 
-  // For each group with collisions, sort by position (top-to-bottom, left-to-right)
-  // then assign deterministic indices
   for (const [baseKey, group] of keyGroups) {
     if (group.length > 1) {
       group.sort((a, b) => {
         const yDiff = a.bounding_box.y - b.bounding_box.y;
-        if (Math.abs(yDiff) > 5) return yDiff; // 5px threshold for "same row"
+        if (Math.abs(yDiff) > 5) return yDiff;
         return a.bounding_box.x - b.bounding_box.x;
       });
     }
@@ -388,11 +610,9 @@ async function enumerateElements() {
   }
 
   // Assign stable display IDs based on stable_key
-  // Elements that existed before keep their display ID, new elements get the next available ID
   const usedDisplayIds = new Set();
   const newElementKeyToDisplayId = new Map();
 
-  // First pass: assign existing display IDs to elements we've seen before
   for (const el of uniqueElements) {
     if (elementKeyToDisplayId.has(el.stable_key)) {
       const existingId = elementKeyToDisplayId.get(el.stable_key);
@@ -402,11 +622,9 @@ async function enumerateElements() {
     }
   }
 
-  // Second pass: assign new display IDs to new elements
   let nextId = 1;
   for (const el of uniqueElements) {
     if (el.id === null) {
-      // Find next available ID
       while (usedDisplayIds.has(`E${nextId}`)) {
         nextId++;
       }
@@ -432,8 +650,7 @@ async function enumerateElements() {
   }
 
   lastElements = uniqueElements;
-  // Return elements without the locator property (not JSON-serializable)
-  return lastElements.map(({ locator, ...rest }) => rest);
+  return lastElements.map(({ domPath, ...rest }) => rest);
 }
 
 async function debugOverlay(x, y) {
@@ -461,7 +678,7 @@ async function debugOverlay(x, y) {
 }
 
 // Commands that should be blocked when a dialog is pending
-const DIALOG_BLOCKING_COMMANDS = ['navigate', 'click_element', 'select_option', 'keypress', 'scroll', 'wait', 'capture'];
+const DIALOG_BLOCKING_COMMANDS = ['navigate', 'click_element', 'select_option', 'type', 'hotkey', 'scroll', 'wait', 'capture'];
 
 async function handleCommand(msg) {
   const { request_id, command, params } = msg;
@@ -503,23 +720,32 @@ async function handleCommand(msg) {
 
         const element = displayIdToElement.get(element_id);
         if (!element) throw new Error(`Element not found: ${element_id}`);
-        lastClickedElement = element; // Store for keypress context
+        lastClickedElement = element; // Store for type command context
 
         let { x, y, w, h } = element.bounding_box;
 
         // Auto-scroll element into view if its center is outside the viewport
         const viewportSize = page.viewportSize();
         let centerY = y + h / 2;
-        if (element.locator && (centerY < 0 || centerY >= viewportSize.height)) {
-          await element.locator.scrollIntoViewIfNeeded({ timeout: 3000 });
-          await new Promise(r => setTimeout(r, 200));
-          const newBox = await element.locator.boundingBox();
-          if (newBox) {
-            x = Math.round(newBox.x);
-            y = Math.round(newBox.y);
-            w = Math.round(newBox.width);
-            h = Math.round(newBox.height);
-            element.bounding_box = { x, y, w, h };
+        if (element.domPath && (centerY < 0 || centerY >= viewportSize.height)) {
+          try {
+            const locator = page.locator(element.domPath);
+            await locator.scrollIntoViewIfNeeded({ timeout: 3000 });
+            await new Promise(r => setTimeout(r, 200));
+            const newBox = await locator.boundingBox();
+            if (newBox) {
+              x = Math.round(newBox.x);
+              y = Math.round(newBox.y);
+              w = Math.round(newBox.width);
+              h = Math.round(newBox.height);
+              element.bounding_box = { x, y, w, h };
+            }
+          } catch (scrollErr) {
+            // Strict mode violation or other locator error — fall back to
+            // coordinate-based scroll so the click can still proceed.
+            const targetY = y + h / 2 - viewportSize.height / 2;
+            await page.evaluate((scrollY) => window.scrollBy(0, scrollY), targetY);
+            await new Promise(r => setTimeout(r, 200));
           }
         }
 
@@ -587,16 +813,25 @@ async function handleCommand(msg) {
         // Auto-scroll element into view if its center is outside the viewport
         const viewportSize = page.viewportSize();
         let centerY = y + h / 2;
-        if (element.locator && (centerY < 0 || centerY >= viewportSize.height)) {
-          await element.locator.scrollIntoViewIfNeeded({ timeout: 3000 });
-          await new Promise(r => setTimeout(r, 200));
-          const newBox = await element.locator.boundingBox();
-          if (newBox) {
-            x = Math.round(newBox.x);
-            y = Math.round(newBox.y);
-            w = Math.round(newBox.width);
-            h = Math.round(newBox.height);
-            element.bounding_box = { x, y, w, h };
+        if (element.domPath && (centerY < 0 || centerY >= viewportSize.height)) {
+          try {
+            const locator = page.locator(element.domPath);
+            await locator.scrollIntoViewIfNeeded({ timeout: 3000 });
+            await new Promise(r => setTimeout(r, 200));
+            const newBox = await locator.boundingBox();
+            if (newBox) {
+              x = Math.round(newBox.x);
+              y = Math.round(newBox.y);
+              w = Math.round(newBox.width);
+              h = Math.round(newBox.height);
+              element.bounding_box = { x, y, w, h };
+            }
+          } catch (scrollErr) {
+            // Strict mode violation or other locator error — fall back to
+            // coordinate-based scroll so the select can still proceed.
+            const targetY = y + h / 2 - viewportSize.height / 2;
+            await page.evaluate((scrollY) => window.scrollBy(0, scrollY), targetY);
+            await new Promise(r => setTimeout(r, 200));
           }
         }
 
@@ -627,18 +862,14 @@ async function handleCommand(msg) {
         break;
       }
 
-      case "keypress": {
-        if (!params?.keys) throw new Error("keypress requires keys");
+      case "type": {
+        if (!params?.keys) throw new Error("type requires keys");
 
-        // Handle Ctrl+A (select all) as a special case
-        // Use Meta+a on macOS (Command key) and Control+a on other platforms
-        if (params.keys === 'ctrl+a' || params.keys === 'Ctrl+A') {
-          const modifier = process.platform === 'darwin' ? 'Meta' : 'Control';
-          await page.keyboard.press(`${modifier}+a`);
-          await new Promise(r => setTimeout(r, 100));
-          result.screenshot_base64 = await screenshotBase64();
-          result.elements = await enumerateElements();
-          break;
+        // Reject modifier combos — those belong in the "hotkey" command
+        if (/^(ctrl|alt|meta|shift)\+/i.test(params.keys)) {
+          throw new Error(
+            `"type" is for typing text only. Use "hotkey" for keyboard shortcuts like "${params.keys}".`
+          );
         }
 
         // Map special characters to Playwright key names
@@ -666,10 +897,22 @@ async function handleCommand(msg) {
           }
         }
 
-        // Type each character, mapping special chars to Playwright key names
+        // Split into runs of normal text (use keyboard.type) and special chars (use keyboard.press)
+        let textBuffer = '';
         for (const char of params.keys) {
-          const key = specialKeyMap[char] || char;
-          await page.keyboard.press(key);
+          const special = specialKeyMap[char];
+          if (special) {
+            if (textBuffer) {
+              await page.keyboard.type(textBuffer, { delay: 20 });
+              textBuffer = '';
+            }
+            await page.keyboard.press(special);
+          } else {
+            textBuffer += char;
+          }
+        }
+        if (textBuffer) {
+          await page.keyboard.type(textBuffer, { delay: 20 });
         }
 
         // Wait for search/autocomplete to settle
@@ -679,8 +922,40 @@ async function handleCommand(msg) {
         break;
       }
 
+      case "hotkey": {
+        if (!params?.keys) throw new Error("hotkey requires keys");
+
+        // Map common shorthand to Playwright key names
+        // Supports: "ctrl+a", "Enter", "Backspace", "Tab", "Escape", "ctrl+c", etc.
+        const hotkeyMap = {
+          'ctrl': process.platform === 'darwin' ? 'Meta' : 'Control',
+          'alt': 'Alt',
+          'shift': 'Shift',
+          'meta': 'Meta',
+        };
+
+        const combo = params.keys;
+
+        if (/\+/.test(combo)) {
+          // Modifier combo like "ctrl+a", "ctrl+shift+z"
+          const parts = combo.split('+');
+          const key = parts.pop(); // Last part is the actual key
+          const modifiers = parts.map(m => hotkeyMap[m.toLowerCase()] || m);
+          const playwrightCombo = [...modifiers, key].join('+');
+          await page.keyboard.press(playwrightCombo);
+        } else {
+          // Single special key like "Enter", "Backspace", "Tab", "Escape"
+          await page.keyboard.press(combo);
+        }
+
+        await new Promise(r => setTimeout(r, 100));
+        result.screenshot_base64 = await screenshotBase64();
+        result.elements = await enumerateElements();
+        break;
+      }
+
       case "wait": {
-        if (!params?.ms) throw new Error("wait requires ms");
+        if (params?.ms == null) throw new Error("wait requires ms");
         await new Promise(r => setTimeout(r, params.ms));
         result.screenshot_base64 = await screenshotBase64();
         result.elements = await enumerateElements();

package/lib/client_errors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Shared base error type for step clients (StepClient, DirectStepClient).
+# Runner can rescue this base type to handle errors from either client.
+# Each client defines its own StepError subclass for API stability.
+class StepClientError < StandardError; end

package/lib/runner.rb

@@ -11,13 +11,14 @@ require 'fileutils'
 # Determine SpecSage home directory for locating resources
 SPECSAGE_HOME ||= File.expand_path('..', __dir__)
 
+require_relative 'client_errors'
 require_relative 'step_client'
 
 class Runner
   NODE_IO_TIMEOUT_SECONDS = 30
   NODE_SHUTDOWN_TIMEOUT_SECONDS = 45
 
-  BROWSER_ACTIONS = %w[navigate click select keypress wait scroll accept_dialog dismiss_dialog].freeze
+  BROWSER_ACTIONS = %w[navigate click select type hotkey wait scroll accept_dialog dismiss_dialog].freeze
 
   # Pattern for safe path segment: alphanumeric, underscore, hyphen only
   # Prevents directory traversal, special chars, and filesystem issues
@@ -25,7 +26,8 @@ class Runner
 
   # Initialize runner with scenario data from server
   # @param all_scenarios [Hash] optional map of scenario_id => scenario_data for pre-scenario lookup
-  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, all_scenarios: nil)
+  # @param step_client [StepClient, DirectStepClient] optional pre-configured client for step processing
+  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, all_scenarios: nil, step_client: nil)
     @scenario = normalize_scenario_data(scenario_data)
     @scenario_id = @scenario['id']
     @scenario_name = @scenario['name'] || @scenario['id'] || 'unnamed'
@@ -38,7 +40,7 @@ class Runner
     @next_request_id = 1
     @node_channel_poisoned = false
     @publisher = publisher
-    @step_client = nil
+    @step_client = step_client # Accept pre-configured client (nil = create StepClient in run())
     @server_run_id = server_run_id
     @credentials = {} # Credentials received from server { "NAME" => "value" }
     @max_steps = nil # Max browser actions allowed, received from server on first step
@@ -51,7 +53,10 @@ class Runner
 
     raise ArgumentError, 'server_run_id is required' unless @server_run_id
 
-    @step_client = StepClient.new(
+    # Only create StepClient if not injected (CLI path uses HTTP, Sidekiq injects DirectStepClient)
+    # TODO: Validate publisher presence here when step_client is nil. Currently crashes with
+    # NoMethodError on @publisher.base_url if caller omits both step_client and publisher.
+    @step_client ||= StepClient.new(
       base_url: @publisher.base_url,
       server_run_id: @server_run_id,
       api_key: @publisher.api_key
@@ -124,7 +129,7 @@ class Runner
     cleanup_temp_dir
 
     result[:verdict]
-  rescue StepClient::StepError => e
+  rescue StepClientError => e
     send_client_verdict_if_needed('ERROR', "Server error: #{e.message}")
     stop_node_process
     upload_video
@@ -245,7 +250,7 @@ class Runner
       status: status,
       reason: reason
     )
-  rescue StepClient::StepError => e
+  rescue StepClientError => e
     log "Warning: Failed to send main scenario verdict: #{e.message}"
   end
 
@@ -310,8 +315,8 @@ class Runner
       break if line.nil? || line.empty?
       line.each_line { |l| log "Node: #{l.strip}" unless l.strip.empty? }
     end
-  rescue IO::WaitReadable, EOFError
-    # No more data available
+  rescue IO::WaitReadable, EOFError, IOError
+    # No more data available (IOError covers closed stream on second stop_node_process call)
   end
 
   def stop_node_process
@@ -376,10 +381,13 @@ class Runner
     # Read any remaining stderr output from Node process for debugging
     drain_node_stderr
 
-    # Close IO streams
+    # Close IO streams and nil references to prevent IOError on subsequent calls
     @node_stdin&.close rescue nil
     @node_stdout&.close rescue nil
     @node_stderr&.close rescue nil
+    @node_stdin = nil
+    @node_stdout = nil
+    @node_stderr = nil
 
     # Force kill the process if still running
     if pid
@@ -398,6 +406,14 @@ class Runner
       end
     end
 
+    # If we couldn't get video via quit (channel poisoned, BROWSER_ERROR,
+    # or any other failure), try to recover from temp directory.
+    # Playwright streams video data to disk during recording, so a
+    # partial file likely exists even without a graceful context.close().
+    if !@video_data && @temp_dir && Dir.exist?(@temp_dir)
+      recover_video_from_temp_dir
+    end
+
     @node_wait_thread = nil
   end
 
@@ -498,18 +514,24 @@ class Runner
       elements = response.dig('result', 'elements') || []
       { result: "Selected '#{display_value}' in element #{element_id}", screenshot_base64: screenshot_base64, elements: elements }
 
-    when 'keypress'
+    when 'type'
       # Substitute credential placeholders at the last moment before browser execution
       # Supports inline placeholders: <<USERNAME>>@example.com, <<USER>>:<<PASS>>, etc.
       keys = action['keys']
       display_keys = keys # For logging (shows placeholders, not actual values)
-      # Skip credential substitution for special key combos like ctrl+a
-      keys = substitute_credentials(keys) if contains_credential_placeholder?(keys) && !special_key_combo?(keys)
+      keys = substitute_credentials(keys) if contains_credential_placeholder?(keys)
 
-      response = send_to_node('keypress', { keys: keys })
+      response = send_to_node('type', { keys: keys })
       screenshot_base64 = response.dig('result', 'screenshot_base64')
       elements = response.dig('result', 'elements') || []
-      { result: "Pressed keys: #{display_keys}", screenshot_base64: screenshot_base64, elements: elements }
+      { result: "Typed: #{display_keys}", screenshot_base64: screenshot_base64, elements: elements }
+
+    when 'hotkey'
+      keys = action['keys']
+      response = send_to_node('hotkey', { keys: keys })
+      screenshot_base64 = response.dig('result', 'screenshot_base64')
+      elements = response.dig('result', 'elements') || []
+      { result: "Hotkey: #{keys}", screenshot_base64: screenshot_base64, elements: elements }
 
     when 'wait'
       response = send_to_node('wait', { ms: action['ms'] })
@@ -587,8 +609,8 @@ class Runner
     log "Uploading video (#{@video_data.bytesize} bytes)..."
     @step_client.upload_video(scenario_id: @scenario_id, video_data: @video_data)
     log "Video uploaded successfully."
-  rescue StepClient::StepError => e
-    log "Warning: Failed to upload video: #{e.message}"
+  rescue StandardError => e
+    log "Warning: Failed to upload video: #{e.class}: #{e.message}"
   end
 
   def cleanup_temp_dir
@@ -611,6 +633,50 @@ class Runner
     # Ignore cleanup errors
   end
 
+  # Recover video file from temp directory when the Node channel was poisoned
+  # and we couldn't send the quit command to get the video path.
+  # Playwright streams video data to disk during recording, so a partial
+  # .webm file likely exists even without a graceful context.close().
+  #
+  # Safety: @temp_dir is scoped to tmp/<server_run_id>/<scenario_id>.
+  # server_run_id has a unique index (see Document::Test::Run), so each run
+  # gets an isolated directory. Stale .webm files from other runs cannot
+  # appear here. On a permanent filesystem (Sidekiq workers), leftover files
+  # would only exist from a previous crash of this exact run ID, which cannot
+  # be re-enqueued with the same ID.
+  def recover_video_from_temp_dir
+    video_files = Dir.glob(File.join(@temp_dir, '*.webm')).sort_by { |f| File.size(f) }.reverse
+    return if video_files.empty?
+
+    video_path = video_files.first
+    file_size = File.size(video_path)
+
+    if file_size > 0
+      if ffmpeg_available?
+        remuxed_path = video_path.sub(/\.webm$/, '_remuxed.webm')
+        if system('ffmpeg', '-i', video_path, '-c', 'copy', remuxed_path, '-y', '-loglevel', 'error')
+          log "Recovered video remuxed: #{remuxed_path}"
+          File.delete(video_path) rescue nil
+          video_path = remuxed_path
+        else
+          log "Warning: ffmpeg remux failed on recovered video, using original"
+        end
+      end
+
+      @video_data = File.binread(video_path)
+      log "Recovered video from temp dir: #{video_path} (#{@video_data.bytesize} bytes)"
+      File.delete(video_path) rescue nil
+    else
+      log "Warning: Video file in temp dir is empty: #{video_path}"
+    end
+  rescue StandardError => e
+    log "Warning: Failed to recover video from temp dir: #{e.message}"
+  end
+
+  def ffmpeg_available?
+    system('ffmpeg', '-version', out: File::NULL, err: File::NULL)
+  end
+
   # --- Safe path handling ---
 
   # Build a safe temp directory path, validating that IDs are safe path segments
@@ -641,7 +707,7 @@ class Runner
       status: status,
       reason: reason
     )
-  rescue StepClient::StepError => e
+  rescue StepClientError => e
     log "Warning: Failed to send verdict to server: #{e.message}"
   end
 
@@ -681,14 +747,6 @@ class Runner
 
   CREDENTIAL_PLACEHOLDER_PATTERN = /<<([A-Z][A-Z0-9_]*)>>/
 
-  # Special key combinations that should not be treated as credential placeholders
-  SPECIAL_KEY_COMBOS = %w[ctrl+a Ctrl+A].freeze
-
-  # Check if the value is a special key combo (e.g., ctrl+a)
-  def special_key_combo?(value)
-    SPECIAL_KEY_COMBOS.include?(value)
-  end
-
   # Check if the value contains any credential placeholders
   def contains_credential_placeholder?(value)
     return false unless value.is_a?(String)

package/lib/step_client.rb

@@ -7,9 +7,12 @@ require "net/http"
 require "uri"
 require "json"
 require "base64"
+require_relative "client_errors"
 
 class StepClient
-  class StepError < StandardError; end
+  # Subclass shared error for API stability (supports subclassing, .class checks, .name)
+  # Runner can rescue either StepClient::StepError or the base StepClientError
+  class StepError < StepClientError; end
 
   attr_reader :server_run_id
 
@@ -59,6 +62,10 @@ class StepClient
 
     response = post("/api/runs/#{@server_run_id}/step", body)
 
+    # TODO: Credentials defaulted to {} may change semantics. Server returns nil when no
+    # credentials are present (after first step). Returning {} instead of nil may cause
+    # downstream code to branch incorrectly (`if credentials` is always truthy). Consider
+    # returning nil when server returns nil, or document this as intentional contract.
     {
       action: response["action"],
       step_number: response["step_number"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specsage/cli",
-  "version": "0.1.12",
+  "version": "0.1.14",
   "description": "SpecSage CLI - AI-powered end-to-end testing automation (Node wrapper for Ruby CLI)",
   "type": "module",
   "bin": {