browser-pilot 0.0.6 → 0.0.7

package/dist/cli.cjs

@@ -3487,13 +3487,18 @@ var INPUT_DEBOUNCE_MS = 300;
 var NAVIGATION_DEBOUNCE_MS = 500;
 function selectBestSelectors(candidates) {
   const qualityOrder = {
-    "stable-attr": 0,
-    id: 1,
-    "css-path": 2
+    "role-name": 0,
+    text: 1,
+    "aria-label": 2,
+    testid: 3,
+    "stable-attr": 4,
+    id: 5,
+    "name-attr": 6,
+    "css-path": 7
   };
   const sorted = [...candidates].sort((a, b) => {
-    const aOrder = qualityOrder[a.quality] ?? 3;
-    const bOrder = qualityOrder[b.quality] ?? 3;
+    const aOrder = qualityOrder[a.quality] ?? 8;
+    const bOrder = qualityOrder[b.quality] ?? 8;
     return aOrder - bOrder;
   });
   const seen = /* @__PURE__ */ new Set();
@@ -3506,6 +3511,67 @@ function selectBestSelectors(candidates) {
   }
   return result;
 }
+function generateAnnotation(event) {
+  const { kind, element, url } = event;
+  const name = element?.accessibleName || element?.text || element?.ariaLabel;
+  const role = element?.computedRole || element?.role || element?.tag || "";
+  switch (kind) {
+    case "click":
+      if (name && role) {
+        return `Clicked '${name}' ${role}`;
+      } else if (name) {
+        return `Clicked '${name}'`;
+      } else if (role) {
+        return `Clicked ${role}`;
+      }
+      return "Clicked element";
+    case "dblclick":
+      if (name && role) {
+        return `Double-clicked '${name}' ${role}`;
+      }
+      return "Double-clicked element";
+    case "input":
+      if (name) {
+        return `Filled '${name}' with value`;
+      }
+      return "Filled input with value";
+    case "change":
+      if (element?.type === "checkbox" || element?.type === "radio") {
+        const action = event.checked ? "Checked" : "Unchecked";
+        if (name) {
+          return `${action} '${name}' ${element.type}`;
+        }
+        return `${action} ${element.type}`;
+      }
+      if (element?.tag === "select") {
+        if (name) {
+          return `Selected option in '${name}'`;
+        }
+        return "Selected option";
+      }
+      if (name) {
+        return `Changed '${name}'`;
+      }
+      return "Changed element";
+    case "submit":
+      if (name) {
+        return `Submitted '${name}' form`;
+      }
+      return "Submitted form";
+    case "keydown":
+      if (event.key === "Enter") {
+        return "Pressed Enter";
+      }
+      return `Pressed ${event.key}`;
+    case "navigation":
+      return `Navigated to ${url}`;
+    default:
+      if (name) {
+        return `${kind} on '${name}'`;
+      }
+      return `${kind} on element`;
+  }
+}
 function debounceInputEvents(events) {
   const result = [];
   for (let i = 0; i < events.length; i++) {
@@ -3584,22 +3650,37 @@ function insertNavigationSteps(events, startUrl) {
   }
   return result;
 }
+function buildElementMeta(event) {
+  const el = event.element;
+  if (!el) return void 0;
+  return {
+    role: el.computedRole || el.role,
+    name: el.accessibleName || el.text || el.ariaLabel,
+    tag: el.tag
+  };
+}
 function eventToStep(event) {
   const selectors = selectBestSelectors(event.selectors);
+  const elementMeta = buildElementMeta(event);
+  const annotation = generateAnnotation(event);
   switch (event.kind) {
     case "click":
     case "dblclick":
       if (selectors.length === 0) return null;
       return {
         action: "click",
-        selector: selectors.length === 1 ? selectors[0] : selectors
+        selector: selectors.length === 1 ? selectors[0] : selectors,
+        element: elementMeta,
+        annotation
       };
     case "input":
       if (selectors.length === 0) return null;
       return {
         action: "fill",
         selector: selectors.length === 1 ? selectors[0] : selectors,
-        value: event.value ?? ""
+        value: event.value ?? "",
+        element: elementMeta,
+        annotation
       };
     case "change": {
       if (selectors.length === 0) return null;
@@ -3610,19 +3691,25 @@ function eventToStep(event) {
         return {
           action: "select",
           selector: selectors.length === 1 ? selectors[0] : selectors,
-          value: event.value ?? ""
+          value: event.value ?? "",
+          element: elementMeta,
+          annotation
         };
       }
       if (type === "checkbox" || type === "radio") {
         return {
           action: event.checked ? "check" : "uncheck",
-          selector: selectors.length === 1 ? selectors[0] : selectors
+          selector: selectors.length === 1 ? selectors[0] : selectors,
+          element: elementMeta,
+          annotation
         };
       }
       return {
         action: "fill",
         selector: selectors.length === 1 ? selectors[0] : selectors,
-        value: event.value ?? ""
+        value: event.value ?? "",
+        element: elementMeta,
+        annotation
       };
     }
     case "keydown":
@@ -3631,7 +3718,9 @@ function eventToStep(event) {
         return {
           action: "submit",
           selector: selectors.length === 1 ? selectors[0] : selectors,
-          method: "enter"
+          method: "enter",
+          element: elementMeta,
+          annotation
         };
       }
       return null;
@@ -3639,12 +3728,15 @@ function eventToStep(event) {
       if (selectors.length === 0) return null;
       return {
         action: "submit",
-        selector: selectors.length === 1 ? selectors[0] : selectors
+        selector: selectors.length === 1 ? selectors[0] : selectors,
+        element: elementMeta,
+        annotation
       };
     case "navigation":
       return {
         action: "goto",
-        url: event.url
+        url: event.url,
+        annotation
       };
     default:
       return null;
@@ -3787,19 +3879,50 @@ var RECORDER_SCRIPT = `(function() {
   function getSelectorCandidates(el) {
     const candidates = [];
 
-    // 1. Stable attributes (highest quality)
+    // Get semantic info for role-based selectors
+    const role = getRole(el);
+    const name = getAccessibleName(el);
+
+    // 1. Role + name selector (highest priority for semantic elements)
+    if (role && name) {
+      const escapedName = name.replace(/'/g, "\\\\'");
+      candidates.push({
+        selector: "role=" + role + "[name='" + escapedName + "']",
+        quality: 'role-name'
+      });
+    }
+
+    // 2. Text-based selector (for buttons, links, menuitems)
+    if (name && ['button', 'link', 'menuitem'].includes(role)) {
+      candidates.push({
+        selector: "text=" + name,
+        quality: 'text'
+      });
+    }
+
+    // 3. aria-label attribute selector
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel) {
+      const escaped = ariaLabel.replace(/"/g, '\\\\"');
+      candidates.push({
+        selector: '[aria-label="' + escaped + '"]',
+        quality: 'aria-label'
+      });
+    }
+
+    // 4. Stable attributes (testid, name)
     const stableAttr = getStableAttrSelector(el);
     if (stableAttr) {
       candidates.push({ selector: stableAttr, quality: 'stable-attr' });
     }
 
-    // 2. ID selector
+    // 5. ID selector
     const idSel = getIdSelector(el);
     if (idSel) {
       candidates.push({ selector: idSel, quality: 'id' });
     }
 
-    // 3. CSS path (fallback)
+    // 6. CSS path (fallback)
     const cssPath = buildCssPath(el);
     if (cssPath) {
       candidates.push({ selector: cssPath, quality: 'css-path' });
@@ -3808,6 +3931,128 @@ var RECORDER_SCRIPT = `(function() {
     return candidates;
   }
 
+  // Compute accessible name per W3C AccName spec
+  // Priority: aria-labelledby > aria-label > label > title > content > alt > placeholder
+  function getAccessibleName(el) {
+    if (!el || el.nodeType !== 1) return null;
+
+    // 1. aria-labelledby
+    const labelledBy = el.getAttribute('aria-labelledby');
+    if (labelledBy) {
+      const labels = labelledBy.split(/\\s+/)
+        .map(function(id) {
+          const ref = document.getElementById(id);
+          return ref ? ref.textContent : null;
+        })
+        .filter(Boolean);
+      if (labels.length) return labels.join(' ').trim().slice(0, 100);
+    }
+
+    // 2. aria-label
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel) return ariaLabel.trim().slice(0, 100);
+
+    // 3. Native <label> for form elements
+    if (el.labels && el.labels.length) {
+      const labelTexts = Array.from(el.labels)
+        .map(function(l) { return l.textContent; })
+        .filter(Boolean);
+      if (labelTexts.length) return labelTexts.join(' ').trim().slice(0, 100);
+    }
+
+    // 4. title attribute
+    const title = el.getAttribute('title');
+    if (title) return title.trim().slice(0, 100);
+
+    // 5. Content for buttons, links, summary
+    const tag = el.tagName.toLowerCase();
+    const role = el.getAttribute('role');
+    if (['button', 'a', 'summary'].includes(tag) || role === 'button' || role === 'link' || role === 'menuitem') {
+      const text = (el.textContent || '').trim();
+      if (text) return text.slice(0, 100);
+    }
+
+    // 6. alt for images
+    if (tag === 'img') {
+      const alt = el.getAttribute('alt');
+      if (alt) return alt.trim().slice(0, 100);
+    }
+
+    // 7. placeholder for inputs
+    if (['input', 'textarea'].includes(tag)) {
+      const placeholder = el.getAttribute('placeholder');
+      if (placeholder) return placeholder.trim().slice(0, 100);
+    }
+
+    return null;
+  }
+
+  // Get explicit ARIA role or implicit role from HTML tag
+  function getRole(el) {
+    if (!el || el.nodeType !== 1) return null;
+
+    // 1. Explicit role attribute
+    const explicitRole = el.getAttribute('role');
+    if (explicitRole) return explicitRole;
+
+    // 2. Implicit role from tag/type
+    const tag = el.tagName.toLowerCase();
+    const type = (el.getAttribute('type') || '').toLowerCase();
+
+    // Input types to roles
+    if (tag === 'input') {
+      var inputRoles = {
+        'button': 'button',
+        'submit': 'button',
+        'reset': 'button',
+        'image': 'button',
+        'checkbox': 'checkbox',
+        'radio': 'radio',
+        'range': 'slider',
+        'search': 'searchbox'
+      };
+      if (inputRoles[type]) return inputRoles[type];
+      // text, email, tel, url, number, password all map to textbox
+      return 'textbox';
+    }
+
+    // Other tags with implicit roles
+    var tagRoles = {
+      'button': 'button',
+      'select': 'combobox',
+      'textarea': 'textbox',
+      'nav': 'navigation',
+      'main': 'main',
+      'header': 'banner',
+      'footer': 'contentinfo',
+      'aside': 'complementary',
+      'article': 'article',
+      'ul': 'list',
+      'ol': 'list',
+      'li': 'listitem',
+      'table': 'table',
+      'tr': 'row',
+      'td': 'cell',
+      'th': 'columnheader',
+      'form': 'form',
+      'img': 'img',
+      'dialog': 'dialog',
+      'menu': 'menu',
+      'summary': 'button'
+    };
+    if (tagRoles[tag]) return tagRoles[tag];
+
+    // Anchor with href is a link
+    if (tag === 'a' && el.hasAttribute('href')) return 'link';
+
+    // Section with aria-label or aria-labelledby is a region
+    if (tag === 'section' && (el.hasAttribute('aria-label') || el.hasAttribute('aria-labelledby'))) {
+      return 'region';
+    }
+
+    return null;
+  }
+
   // Get element summary for debugging
   function getElementSummary(el) {
     if (!el || el.nodeType !== 1) return null;
@@ -3820,7 +4065,9 @@ var RECORDER_SCRIPT = `(function() {
       role: el.getAttribute('role') || null,
       ariaLabel: el.getAttribute('aria-label') || null,
       testid: el.getAttribute('data-testid') || null,
-      text: text || null
+      text: text || null,
+      accessibleName: getAccessibleName(el),
+      computedRole: getRole(el)
     };
   }
 

package/dist/cli.mjs

@@ -681,13 +681,18 @@ var INPUT_DEBOUNCE_MS = 300;
 var NAVIGATION_DEBOUNCE_MS = 500;
 function selectBestSelectors(candidates) {
   const qualityOrder = {
-    "stable-attr": 0,
-    id: 1,
-    "css-path": 2
+    "role-name": 0,
+    text: 1,
+    "aria-label": 2,
+    testid: 3,
+    "stable-attr": 4,
+    id: 5,
+    "name-attr": 6,
+    "css-path": 7
   };
   const sorted = [...candidates].sort((a, b) => {
-    const aOrder = qualityOrder[a.quality] ?? 3;
-    const bOrder = qualityOrder[b.quality] ?? 3;
+    const aOrder = qualityOrder[a.quality] ?? 8;
+    const bOrder = qualityOrder[b.quality] ?? 8;
     return aOrder - bOrder;
   });
   const seen = /* @__PURE__ */ new Set();
@@ -700,6 +705,67 @@ function selectBestSelectors(candidates) {
   }
   return result;
 }
+function generateAnnotation(event) {
+  const { kind, element, url } = event;
+  const name = element?.accessibleName || element?.text || element?.ariaLabel;
+  const role = element?.computedRole || element?.role || element?.tag || "";
+  switch (kind) {
+    case "click":
+      if (name && role) {
+        return `Clicked '${name}' ${role}`;
+      } else if (name) {
+        return `Clicked '${name}'`;
+      } else if (role) {
+        return `Clicked ${role}`;
+      }
+      return "Clicked element";
+    case "dblclick":
+      if (name && role) {
+        return `Double-clicked '${name}' ${role}`;
+      }
+      return "Double-clicked element";
+    case "input":
+      if (name) {
+        return `Filled '${name}' with value`;
+      }
+      return "Filled input with value";
+    case "change":
+      if (element?.type === "checkbox" || element?.type === "radio") {
+        const action = event.checked ? "Checked" : "Unchecked";
+        if (name) {
+          return `${action} '${name}' ${element.type}`;
+        }
+        return `${action} ${element.type}`;
+      }
+      if (element?.tag === "select") {
+        if (name) {
+          return `Selected option in '${name}'`;
+        }
+        return "Selected option";
+      }
+      if (name) {
+        return `Changed '${name}'`;
+      }
+      return "Changed element";
+    case "submit":
+      if (name) {
+        return `Submitted '${name}' form`;
+      }
+      return "Submitted form";
+    case "keydown":
+      if (event.key === "Enter") {
+        return "Pressed Enter";
+      }
+      return `Pressed ${event.key}`;
+    case "navigation":
+      return `Navigated to ${url}`;
+    default:
+      if (name) {
+        return `${kind} on '${name}'`;
+      }
+      return `${kind} on element`;
+  }
+}
 function debounceInputEvents(events) {
   const result = [];
   for (let i = 0; i < events.length; i++) {
@@ -778,22 +844,37 @@ function insertNavigationSteps(events, startUrl) {
   }
   return result;
 }
+function buildElementMeta(event) {
+  const el = event.element;
+  if (!el) return void 0;
+  return {
+    role: el.computedRole || el.role,
+    name: el.accessibleName || el.text || el.ariaLabel,
+    tag: el.tag
+  };
+}
 function eventToStep(event) {
   const selectors = selectBestSelectors(event.selectors);
+  const elementMeta = buildElementMeta(event);
+  const annotation = generateAnnotation(event);
   switch (event.kind) {
     case "click":
     case "dblclick":
       if (selectors.length === 0) return null;
       return {
         action: "click",
-        selector: selectors.length === 1 ? selectors[0] : selectors
+        selector: selectors.length === 1 ? selectors[0] : selectors,
+        element: elementMeta,
+        annotation
       };
     case "input":
       if (selectors.length === 0) return null;
       return {
         action: "fill",
         selector: selectors.length === 1 ? selectors[0] : selectors,
-        value: event.value ?? ""
+        value: event.value ?? "",
+        element: elementMeta,
+        annotation
       };
     case "change": {
       if (selectors.length === 0) return null;
@@ -804,19 +885,25 @@ function eventToStep(event) {
         return {
           action: "select",
           selector: selectors.length === 1 ? selectors[0] : selectors,
-          value: event.value ?? ""
+          value: event.value ?? "",
+          element: elementMeta,
+          annotation
         };
       }
       if (type === "checkbox" || type === "radio") {
         return {
           action: event.checked ? "check" : "uncheck",
-          selector: selectors.length === 1 ? selectors[0] : selectors
+          selector: selectors.length === 1 ? selectors[0] : selectors,
+          element: elementMeta,
+          annotation
         };
       }
       return {
         action: "fill",
         selector: selectors.length === 1 ? selectors[0] : selectors,
-        value: event.value ?? ""
+        value: event.value ?? "",
+        element: elementMeta,
+        annotation
       };
     }
     case "keydown":
@@ -825,7 +912,9 @@ function eventToStep(event) {
         return {
           action: "submit",
           selector: selectors.length === 1 ? selectors[0] : selectors,
-          method: "enter"
+          method: "enter",
+          element: elementMeta,
+          annotation
         };
       }
       return null;
@@ -833,12 +922,15 @@ function eventToStep(event) {
       if (selectors.length === 0) return null;
       return {
         action: "submit",
-        selector: selectors.length === 1 ? selectors[0] : selectors
+        selector: selectors.length === 1 ? selectors[0] : selectors,
+        element: elementMeta,
+        annotation
       };
     case "navigation":
       return {
         action: "goto",
-        url: event.url
+        url: event.url,
+        annotation
       };
     default:
       return null;
@@ -981,19 +1073,50 @@ var RECORDER_SCRIPT = `(function() {
   function getSelectorCandidates(el) {
     const candidates = [];
 
-    // 1. Stable attributes (highest quality)
+    // Get semantic info for role-based selectors
+    const role = getRole(el);
+    const name = getAccessibleName(el);
+
+    // 1. Role + name selector (highest priority for semantic elements)
+    if (role && name) {
+      const escapedName = name.replace(/'/g, "\\\\'");
+      candidates.push({
+        selector: "role=" + role + "[name='" + escapedName + "']",
+        quality: 'role-name'
+      });
+    }
+
+    // 2. Text-based selector (for buttons, links, menuitems)
+    if (name && ['button', 'link', 'menuitem'].includes(role)) {
+      candidates.push({
+        selector: "text=" + name,
+        quality: 'text'
+      });
+    }
+
+    // 3. aria-label attribute selector
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel) {
+      const escaped = ariaLabel.replace(/"/g, '\\\\"');
+      candidates.push({
+        selector: '[aria-label="' + escaped + '"]',
+        quality: 'aria-label'
+      });
+    }
+
+    // 4. Stable attributes (testid, name)
     const stableAttr = getStableAttrSelector(el);
     if (stableAttr) {
       candidates.push({ selector: stableAttr, quality: 'stable-attr' });
     }
 
-    // 2. ID selector
+    // 5. ID selector
     const idSel = getIdSelector(el);
     if (idSel) {
       candidates.push({ selector: idSel, quality: 'id' });
     }
 
-    // 3. CSS path (fallback)
+    // 6. CSS path (fallback)
     const cssPath = buildCssPath(el);
     if (cssPath) {
       candidates.push({ selector: cssPath, quality: 'css-path' });
@@ -1002,6 +1125,128 @@ var RECORDER_SCRIPT = `(function() {
     return candidates;
   }
 
+  // Compute accessible name per W3C AccName spec
+  // Priority: aria-labelledby > aria-label > label > title > content > alt > placeholder
+  function getAccessibleName(el) {
+    if (!el || el.nodeType !== 1) return null;
+
+    // 1. aria-labelledby
+    const labelledBy = el.getAttribute('aria-labelledby');
+    if (labelledBy) {
+      const labels = labelledBy.split(/\\s+/)
+        .map(function(id) {
+          const ref = document.getElementById(id);
+          return ref ? ref.textContent : null;
+        })
+        .filter(Boolean);
+      if (labels.length) return labels.join(' ').trim().slice(0, 100);
+    }
+
+    // 2. aria-label
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel) return ariaLabel.trim().slice(0, 100);
+
+    // 3. Native <label> for form elements
+    if (el.labels && el.labels.length) {
+      const labelTexts = Array.from(el.labels)
+        .map(function(l) { return l.textContent; })
+        .filter(Boolean);
+      if (labelTexts.length) return labelTexts.join(' ').trim().slice(0, 100);
+    }
+
+    // 4. title attribute
+    const title = el.getAttribute('title');
+    if (title) return title.trim().slice(0, 100);
+
+    // 5. Content for buttons, links, summary
+    const tag = el.tagName.toLowerCase();
+    const role = el.getAttribute('role');
+    if (['button', 'a', 'summary'].includes(tag) || role === 'button' || role === 'link' || role === 'menuitem') {
+      const text = (el.textContent || '').trim();
+      if (text) return text.slice(0, 100);
+    }
+
+    // 6. alt for images
+    if (tag === 'img') {
+      const alt = el.getAttribute('alt');
+      if (alt) return alt.trim().slice(0, 100);
+    }
+
+    // 7. placeholder for inputs
+    if (['input', 'textarea'].includes(tag)) {
+      const placeholder = el.getAttribute('placeholder');
+      if (placeholder) return placeholder.trim().slice(0, 100);
+    }
+
+    return null;
+  }
+
+  // Get explicit ARIA role or implicit role from HTML tag
+  function getRole(el) {
+    if (!el || el.nodeType !== 1) return null;
+
+    // 1. Explicit role attribute
+    const explicitRole = el.getAttribute('role');
+    if (explicitRole) return explicitRole;
+
+    // 2. Implicit role from tag/type
+    const tag = el.tagName.toLowerCase();
+    const type = (el.getAttribute('type') || '').toLowerCase();
+
+    // Input types to roles
+    if (tag === 'input') {
+      var inputRoles = {
+        'button': 'button',
+        'submit': 'button',
+        'reset': 'button',
+        'image': 'button',
+        'checkbox': 'checkbox',
+        'radio': 'radio',
+        'range': 'slider',
+        'search': 'searchbox'
+      };
+      if (inputRoles[type]) return inputRoles[type];
+      // text, email, tel, url, number, password all map to textbox
+      return 'textbox';
+    }
+
+    // Other tags with implicit roles
+    var tagRoles = {
+      'button': 'button',
+      'select': 'combobox',
+      'textarea': 'textbox',
+      'nav': 'navigation',
+      'main': 'main',
+      'header': 'banner',
+      'footer': 'contentinfo',
+      'aside': 'complementary',
+      'article': 'article',
+      'ul': 'list',
+      'ol': 'list',
+      'li': 'listitem',
+      'table': 'table',
+      'tr': 'row',
+      'td': 'cell',
+      'th': 'columnheader',
+      'form': 'form',
+      'img': 'img',
+      'dialog': 'dialog',
+      'menu': 'menu',
+      'summary': 'button'
+    };
+    if (tagRoles[tag]) return tagRoles[tag];
+
+    // Anchor with href is a link
+    if (tag === 'a' && el.hasAttribute('href')) return 'link';
+
+    // Section with aria-label or aria-labelledby is a region
+    if (tag === 'section' && (el.hasAttribute('aria-label') || el.hasAttribute('aria-labelledby'))) {
+      return 'region';
+    }
+
+    return null;
+  }
+
   // Get element summary for debugging
   function getElementSummary(el) {
     if (!el || el.nodeType !== 1) return null;
@@ -1014,7 +1259,9 @@ var RECORDER_SCRIPT = `(function() {
       role: el.getAttribute('role') || null,
       ariaLabel: el.getAttribute('aria-label') || null,
       testid: el.getAttribute('data-testid') || null,
-      text: text || null
+      text: text || null,
+      accessibleName: getAccessibleName(el),
+      computedRole: getRole(el)
     };
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "browser-pilot",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Lightweight CDP-based browser automation for Node.js, Bun, and Cloudflare Workers",
   "type": "module",
   "main": "./dist/index.cjs",