@everydaydevopsio/pilot 0.5.2 → 0.6.1

package/src/browser/inspect/snapshot.ts

@@ -0,0 +1,348 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from './element-ref.js';
+import type { ElementInfo, ElementBounds, SnapshotResult } from './types.js';
+import { getLogger } from '../../util/logger.js';
+
+/**
+ * Roles that are always included in snapshots when they have
+ * a name or are interactive.
+ */
+const INTERACTIVE_ROLES = new Set([
+  'button',
+  'link',
+  'textbox',
+  'searchbox',
+  'checkbox',
+  'radio',
+  'combobox',
+  'listbox',
+  'option',
+  'menuitem',
+  'menuitemcheckbox',
+  'menuitemradio',
+  'tab',
+  'switch',
+  'slider',
+  'spinbutton',
+  'treeitem'
+]);
+
+/**
+ * Roles included in snapshots when they have a name.
+ */
+const SEMANTIC_ROLES = new Set([
+  'heading',
+  'img',
+  'alert',
+  'dialog',
+  'alertdialog',
+  'navigation',
+  'main',
+  'complementary',
+  'contentinfo',
+  'banner',
+  'form',
+  'region',
+  'table',
+  'cell',
+  'row',
+  'columnheader',
+  'rowheader'
+]);
+
+interface AXNode {
+  nodeId: string;
+  ignored: boolean;
+  role?: { type: string; value?: string };
+  name?: { type: string; value?: string };
+  description?: { type: string; value?: string };
+  value?: { type: string; value?: unknown };
+  properties?: Array<{
+    name: string;
+    value: { type: string; value?: unknown };
+  }>;
+  backendDOMNodeId?: number;
+  childIds?: string[];
+  parentId?: string;
+}
+
+function getAXProperty(node: AXNode, propName: string): unknown | undefined {
+  const prop = node.properties?.find((p) => p.name === propName);
+  return prop?.value?.value;
+}
+
+function isNodeVisible(node: AXNode): boolean {
+  const hidden = getAXProperty(node, 'hidden');
+  if (hidden === true) return false;
+  return true;
+}
+
+function shouldIncludeNode(node: AXNode): boolean {
+  if (node.ignored) return false;
+
+  const role = node.role?.value;
+  if (!role || role === 'none' || role === 'presentation') return false;
+
+  // Generic containers without names are noise
+  if (role === 'generic' || role === 'GenericContainer') {
+    return false;
+  }
+
+  const name = node.name?.value;
+
+  // Interactive roles are always included
+  if (INTERACTIVE_ROLES.has(role)) return true;
+
+  // Semantic roles are included when they have a name
+  if (SEMANTIC_ROLES.has(role) && name) return true;
+
+  // Static text nodes with content
+  if (role === 'StaticText' || role === 'text') return false;
+
+  // Include any other node that has an explicit name (ARIA-labeled)
+  if (name) return true;
+
+  return false;
+}
+
+async function getBoxModelSafe(
+  client: Client,
+  backendNodeId: number
+): Promise<ElementBounds | undefined> {
+  try {
+    const { model } = await client.DOM.getBoxModel({ backendNodeId });
+    // content quad: [x1,y1, x2,y2, x3,y3, x4,y4]
+    const content = model.content;
+    const x = Math.min(content[0], content[2], content[4], content[6]);
+    const y = Math.min(content[1], content[3], content[5], content[7]);
+    const maxX = Math.max(content[0], content[2], content[4], content[6]);
+    const maxY = Math.max(content[1], content[3], content[5], content[7]);
+    return {
+      x: Math.round(x),
+      y: Math.round(y),
+      width: Math.round(maxX - x),
+      height: Math.round(maxY - y)
+    };
+  } catch {
+    // Element may not have a box model (e.g. display:none, zero-size)
+    return undefined;
+  }
+}
+
+async function getNodeTag(
+  client: Client,
+  backendNodeId: number
+): Promise<{ tag: string; type?: string }> {
+  try {
+    const { node } = await client.DOM.describeNode({
+      backendNodeId,
+      depth: 0
+    });
+    const tag = node.nodeName?.toLowerCase() ?? '';
+    // Extract input type from attributes
+    let type: string | undefined;
+    if (tag === 'input' && node.attributes) {
+      for (let i = 0; i < node.attributes.length; i += 2) {
+        if (node.attributes[i] === 'type') {
+          type = node.attributes[i + 1];
+          break;
+        }
+      }
+    }
+    return { tag, type };
+  } catch {
+    return { tag: '' };
+  }
+}
+
+export async function buildSnapshot(
+  client: Client,
+  refMap: ElementRefMap
+): Promise<SnapshotResult> {
+  const logger = getLogger();
+
+  // Clear refs for the new snapshot without incrementing generation.
+  // Generation only advances on navigation (via invalidate()).
+  refMap.resetRefs();
+
+  // Enable required domains
+  await Promise.all([client.Accessibility.enable(), client.DOM.enable({})]);
+
+  // Get the full accessibility tree
+  const { nodes } = await client.Accessibility.getFullAXTree({});
+
+  // Get page info
+  const { result: urlResult } = await client.Runtime.evaluate({
+    expression: 'document.URL',
+    returnByValue: true
+  });
+  const { result: titleResult } = await client.Runtime.evaluate({
+    expression: 'document.title',
+    returnByValue: true
+  });
+
+  const url = String(urlResult.value ?? '');
+  const title = String(titleResult.value ?? '');
+
+  // Filter to meaningful nodes
+  const candidates = (nodes as AXNode[]).filter(shouldIncludeNode);
+
+  logger.debug(
+    { totalNodes: nodes.length, candidates: candidates.length },
+    'Building snapshot'
+  );
+
+  // Build element info for each candidate.
+  // Assign refs first, then fetch DOM details in parallel per node.
+  const validCandidates = candidates.filter((n) => n.backendDOMNodeId);
+  const refs = validCandidates.map((axNode) => {
+    const ref = refMap.nextRef();
+    refMap.set(ref, axNode.backendDOMNodeId!);
+    return ref;
+  });
+
+  const domDetails = await Promise.all(
+    validCandidates.map(async (axNode) => {
+      const id = axNode.backendDOMNodeId!;
+      const [tagInfo, bounds] = await Promise.all([
+        getNodeTag(client, id),
+        getBoxModelSafe(client, id)
+      ]);
+      return { tagInfo, bounds };
+    })
+  );
+
+  const elements: ElementInfo[] = [];
+
+  for (let i = 0; i < validCandidates.length; i++) {
+    const axNode = validCandidates[i];
+    const ref = refs[i];
+    const { tagInfo, bounds } = domDetails[i];
+
+    const role = axNode.role?.value ?? '';
+    const name = axNode.name?.value ?? '';
+
+    const { tag, type } = tagInfo;
+
+    // Extract properties
+    const checked = getAXProperty(axNode, 'checked');
+    const disabled = getAXProperty(axNode, 'disabled');
+    const required = getAXProperty(axNode, 'required');
+    const selected = getAXProperty(axNode, 'selected');
+    const expanded = getAXProperty(axNode, 'expanded');
+    const level = getAXProperty(axNode, 'level');
+
+    // Get value for form controls
+    let value: string | undefined;
+    if (axNode.value?.value !== undefined && axNode.value?.value !== '') {
+      value = String(axNode.value.value);
+    }
+
+    const visible = isNodeVisible(axNode) && bounds !== undefined;
+    const description = axNode.description?.value || undefined;
+
+    const element: ElementInfo = {
+      ref,
+      role,
+      name,
+      tag,
+      visible,
+      ...(type !== undefined && { type }),
+      ...(value !== undefined && { value }),
+      ...(checked !== undefined && {
+        checked: checked === true || checked === 'true'
+      }),
+      ...(disabled === true && { disabled: true }),
+      ...(required === true && { required: true }),
+      ...(selected === true && { selected: true }),
+      ...(expanded !== undefined && { expanded: expanded === true }),
+      ...(typeof level === 'number' && { level }),
+      ...(bounds && { bounds }),
+      ...(description && { description })
+    };
+
+    elements.push(element);
+  }
+
+  return {
+    url,
+    title,
+    timestamp: Date.now(),
+    elements
+  };
+}
+
+export function formatSnapshot(snapshot: SnapshotResult): string {
+  const lines: string[] = [];
+  lines.push(`Page: ${snapshot.title || snapshot.url}`);
+  lines.push(`URL: ${snapshot.url}`);
+  lines.push('');
+
+  if (snapshot.elements.length === 0) {
+    lines.push('No interactive elements found.');
+    return lines.join('\n');
+  }
+
+  lines.push(`Elements (${snapshot.elements.length}):`);
+  lines.push('');
+
+  for (const el of snapshot.elements) {
+    const parts: string[] = [`[${el.ref}]`, el.role];
+
+    if (el.name) {
+      parts.push(`"${el.name}"`);
+    }
+
+    const attrs: string[] = [];
+    if (el.tag && el.tag !== el.role) attrs.push(el.tag);
+    if (el.type) attrs.push(`type=${el.type}`);
+    if (el.value !== undefined) attrs.push(`value="${el.value}"`);
+    if (el.checked !== undefined)
+      attrs.push(el.checked ? 'checked' : 'unchecked');
+    if (el.disabled) attrs.push('disabled');
+    if (el.required) attrs.push('required');
+    if (el.selected) attrs.push('selected');
+    if (el.expanded !== undefined)
+      attrs.push(el.expanded ? 'expanded' : 'collapsed');
+    if (el.level !== undefined) attrs.push(`level=${el.level}`);
+    if (!el.visible) attrs.push('hidden');
+
+    if (attrs.length > 0) {
+      parts.push(`(${attrs.join(', ')})`);
+    }
+
+    lines.push('  ' + parts.join(' '));
+  }
+
+  return lines.join('\n');
+}
+
+export function findElements(
+  snapshot: SnapshotResult,
+  query: {
+    role?: string;
+    name?: string;
+    text?: string;
+  }
+): ElementInfo[] {
+  return snapshot.elements.filter((el) => {
+    if (query.role && el.role.toLowerCase() !== query.role.toLowerCase()) {
+      return false;
+    }
+    if (query.name) {
+      const nameMatch = el.name
+        .toLowerCase()
+        .includes(query.name.toLowerCase());
+      if (!nameMatch) return false;
+    }
+    if (query.text) {
+      const textLower = query.text.toLowerCase();
+      const nameMatch = el.name.toLowerCase().includes(textLower);
+      const valueMatch = el.value?.toLowerCase().includes(textLower) ?? false;
+      const descMatch =
+        el.description?.toLowerCase().includes(textLower) ?? false;
+      if (!nameMatch && !valueMatch && !descMatch) return false;
+    }
+    return true;
+  });
+}

package/src/browser/inspect/styles.ts

@@ -0,0 +1,258 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from './element-ref.js';
+
+/**
+ * Default diagnostic properties returned when no filter is specified.
+ * Covers the most common layout/visibility debugging needs.
+ */
+const DEFAULT_PROPERTIES = new Set([
+  'display',
+  'visibility',
+  'opacity',
+  'position',
+  'width',
+  'height',
+  'min-width',
+  'min-height',
+  'max-width',
+  'max-height',
+  'margin-top',
+  'margin-right',
+  'margin-bottom',
+  'margin-left',
+  'padding-top',
+  'padding-right',
+  'padding-bottom',
+  'padding-left',
+  'overflow',
+  'overflow-x',
+  'overflow-y',
+  'z-index',
+  'transform',
+  'color',
+  'background-color',
+  'font-size',
+  'font-weight',
+  'text-align',
+  'flex-direction',
+  'flex-wrap',
+  'justify-content',
+  'align-items',
+  'align-self',
+  'flex-grow',
+  'flex-shrink',
+  'gap',
+  'grid-template-columns',
+  'grid-template-rows',
+  'border',
+  'border-radius',
+  'box-sizing',
+  'pointer-events',
+  'cursor'
+]);
+
+export interface BoxModelInfo {
+  content: { x: number; y: number; width: number; height: number };
+  padding: { top: number; right: number; bottom: number; left: number };
+  border: { top: number; right: number; bottom: number; left: number };
+  margin: { top: number; right: number; bottom: number; left: number };
+  width: number;
+  height: number;
+}
+
+export interface MatchedRule {
+  selector: string;
+  source: string;
+  properties: Record<string, string>;
+}
+
+export interface StylesResult {
+  ref: string;
+  computed: Record<string, string>;
+  matched: MatchedRule[];
+  boxModel?: BoxModelInfo;
+}
+
+let cssEnabledFor: WeakRef<Client> | null = null;
+
+async function ensureCssEnabled(client: Client): Promise<void> {
+  if (cssEnabledFor?.deref() === client) return;
+  await client.CSS.enable();
+  cssEnabledFor = new WeakRef(client);
+}
+
+// Reset CSS enabled state (for reconnection)
+export function resetCssState(): void {
+  cssEnabledFor = null;
+}
+
+function parseQuad(quad: number[]): {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+} {
+  const x = Math.min(quad[0], quad[2], quad[4], quad[6]);
+  const y = Math.min(quad[1], quad[3], quad[5], quad[7]);
+  const maxX = Math.max(quad[0], quad[2], quad[4], quad[6]);
+  const maxY = Math.max(quad[1], quad[3], quad[5], quad[7]);
+  return {
+    x: Math.round(x),
+    y: Math.round(y),
+    width: Math.round(maxX - x),
+    height: Math.round(maxY - y)
+  };
+}
+
+function computeBoxEdges(
+  outer: number[],
+  inner: number[]
+): { top: number; right: number; bottom: number; left: number } {
+  const outerRect = parseQuad(outer);
+  const innerRect = parseQuad(inner);
+  return {
+    top: Math.round(innerRect.y - outerRect.y),
+    right: Math.round(
+      outerRect.x + outerRect.width - (innerRect.x + innerRect.width)
+    ),
+    bottom: Math.round(
+      outerRect.y + outerRect.height - (innerRect.y + innerRect.height)
+    ),
+    left: Math.round(innerRect.x - outerRect.x)
+  };
+}
+
+export async function getStyles(
+  client: Client,
+  refMap: ElementRefMap,
+  ref: string,
+  properties?: string[]
+): Promise<StylesResult> {
+  const backendNodeId = refMap.resolve(ref);
+
+  await ensureCssEnabled(client);
+  await client.DOM.enable({});
+
+  // We need a nodeId for CSS APIs — push the backend node to the frontend
+  const { nodeIds } = await client.DOM.pushNodesByBackendIdsToFrontend({
+    backendNodeIds: [backendNodeId]
+  });
+  const nodeId = nodeIds[0];
+  if (!nodeId) {
+    throw new Error(`Could not resolve ref "${ref}" to a DOM node`);
+  }
+
+  // Get computed styles
+  const { computedStyle } = await client.CSS.getComputedStyleForNode({
+    nodeId
+  });
+
+  const filterSet = properties
+    ? new Set(properties.map((p) => p.toLowerCase()))
+    : DEFAULT_PROPERTIES;
+
+  const computed: Record<string, string> = {};
+  for (const prop of computedStyle) {
+    if (filterSet.has(prop.name)) {
+      computed[prop.name] = prop.value;
+    }
+  }
+
+  // Get matched CSS rules
+  const matched: MatchedRule[] = [];
+  try {
+    const { matchedCSSRules } = await client.CSS.getMatchedStylesForNode({
+      nodeId
+    });
+    if (matchedCSSRules) {
+      for (const match of matchedCSSRules) {
+        const rule = match.rule;
+        const selector = rule.selectorList?.text ?? '';
+        const source = rule.origin ?? 'regular';
+        const ruleProps: Record<string, string> = {};
+        for (const prop of rule.style.cssProperties) {
+          if (filterSet.has(prop.name) && prop.value) {
+            ruleProps[prop.name] = prop.value;
+          }
+        }
+        if (Object.keys(ruleProps).length > 0) {
+          matched.push({ selector, source, properties: ruleProps });
+        }
+      }
+    }
+  } catch {
+    // getMatchedStylesForNode can fail for some node types
+  }
+
+  // Get box model
+  let boxModel: BoxModelInfo | undefined;
+  try {
+    const { model } = await client.DOM.getBoxModel({ backendNodeId });
+    boxModel = {
+      content: parseQuad(model.content),
+      padding: computeBoxEdges(model.padding, model.content),
+      border: computeBoxEdges(model.border, model.padding),
+      margin: computeBoxEdges(model.margin, model.border),
+      width: model.width,
+      height: model.height
+    };
+  } catch {
+    // Element may not have a box model (display:none)
+  }
+
+  return { ref, computed, matched, boxModel };
+}
+
+export function formatStyles(result: StylesResult): string {
+  const lines: string[] = [`Styles for ${result.ref}:\n`];
+
+  // Computed styles (sorted for stable output)
+  lines.push('Computed:');
+  const entries = Object.entries(result.computed).sort(([a], [b]) =>
+    a.localeCompare(b)
+  );
+  if (entries.length === 0) {
+    lines.push('  (none)');
+  } else {
+    for (const [name, value] of entries) {
+      lines.push(`  ${name}: ${value}`);
+    }
+  }
+
+  // Box model
+  if (result.boxModel) {
+    const bm = result.boxModel;
+    lines.push('');
+    lines.push('Box Model:');
+    lines.push(
+      `  content: ${bm.content.width}x${bm.content.height} at (${bm.content.x}, ${bm.content.y})`
+    );
+    lines.push(
+      `  padding: ${bm.padding.top} ${bm.padding.right} ${bm.padding.bottom} ${bm.padding.left}`
+    );
+    lines.push(
+      `  border: ${bm.border.top} ${bm.border.right} ${bm.border.bottom} ${bm.border.left}`
+    );
+    lines.push(
+      `  margin: ${bm.margin.top} ${bm.margin.right} ${bm.margin.bottom} ${bm.margin.left}`
+    );
+    lines.push(`  total: ${bm.width}x${bm.height}`);
+  }
+
+  // Matched rules
+  if (result.matched.length > 0) {
+    lines.push('');
+    lines.push('Matched Rules:');
+    for (const rule of result.matched) {
+      lines.push(`  ${rule.selector} (${rule.source}):`);
+      const sortedProps = Object.entries(rule.properties).sort(([a], [b]) =>
+        a.localeCompare(b)
+      );
+      for (const [name, value] of sortedProps) {
+        lines.push(`    ${name}: ${value}`);
+      }
+    }
+  }
+
+  return lines.join('\n');
+}

package/src/browser/inspect/types.ts

@@ -0,0 +1,37 @@
+export interface ElementBounds {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface ElementInfo {
+  ref: string;
+  role: string;
+  name: string;
+  tag: string;
+  type?: string;
+  value?: string;
+  checked?: boolean;
+  disabled?: boolean;
+  required?: boolean;
+  selected?: boolean;
+  expanded?: boolean;
+  level?: number;
+  visible: boolean;
+  bounds?: ElementBounds;
+  description?: string;
+}
+
+export interface SnapshotResult {
+  url: string;
+  title: string;
+  timestamp: number;
+  elements: ElementInfo[];
+}
+
+export interface StaleElementError {
+  code: 'STALE_ELEMENT_REFERENCE';
+  ref: string;
+  message: string;
+}

package/src/browser/interaction/dialogs.ts

@@ -0,0 +1,54 @@
+import type { Client } from 'chrome-remote-interface';
+
+export interface PendingDialog {
+  type: string;
+  message: string;
+  defaultPrompt?: string;
+  timestamp: number;
+}
+
+export class DialogQueue {
+  private pending: PendingDialog[] = [];
+
+  push(dialog: PendingDialog): void {
+    this.pending.push(dialog);
+  }
+
+  list(): PendingDialog[] {
+    return [...this.pending];
+  }
+
+  shift(): PendingDialog | undefined {
+    return this.pending.shift();
+  }
+
+  size(): number {
+    return this.pending.length;
+  }
+
+  clear(): void {
+    this.pending = [];
+  }
+}
+
+export function attachDialogHandler(client: Client, queue: DialogQueue): void {
+  client.Page.javascriptDialogOpening((params) => {
+    queue.push({
+      type: params.type,
+      message: params.message,
+      defaultPrompt: params.defaultPrompt ?? undefined,
+      timestamp: Date.now()
+    });
+  });
+}
+
+export async function handleDialog(
+  client: Client,
+  accept: boolean,
+  promptText?: string
+): Promise<void> {
+  await client.Page.handleJavaScriptDialog({
+    accept,
+    promptText
+  });
+}