@everydaydevopsio/pilot 0.5.2 → 0.6.0

package/src/commands/press_key.ts

@@ -0,0 +1,134 @@
+import type { Client } from 'chrome-remote-interface';
+
+/**
+ * Map of named keys to CDP key event properties.
+ */
+const KEY_MAP: Record<
+  string,
+  { key: string; code: string; keyCode?: number; text?: string }
+> = {
+  enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
+  tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
+  escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
+  backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
+  delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
+  space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
+  arrowup: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
+  arrowdown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
+  arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
+  arrowright: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
+  home: { key: 'Home', code: 'Home', keyCode: 36 },
+  end: { key: 'End', code: 'End', keyCode: 35 },
+  pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
+  pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
+  f1: { key: 'F1', code: 'F1', keyCode: 112 },
+  f2: { key: 'F2', code: 'F2', keyCode: 113 },
+  f3: { key: 'F3', code: 'F3', keyCode: 114 },
+  f4: { key: 'F4', code: 'F4', keyCode: 115 },
+  f5: { key: 'F5', code: 'F5', keyCode: 116 },
+  f6: { key: 'F6', code: 'F6', keyCode: 117 },
+  f7: { key: 'F7', code: 'F7', keyCode: 118 },
+  f8: { key: 'F8', code: 'F8', keyCode: 119 },
+  f9: { key: 'F9', code: 'F9', keyCode: 120 },
+  f10: { key: 'F10', code: 'F10', keyCode: 121 },
+  f11: { key: 'F11', code: 'F11', keyCode: 122 },
+  f12: { key: 'F12', code: 'F12', keyCode: 123 }
+};
+
+// CDP modifier bit flags
+const MODIFIER_FLAGS: Record<string, number> = {
+  alt: 1,
+  control: 2,
+  ctrl: 2,
+  meta: 4,
+  cmd: 4,
+  command: 4,
+  shift: 8
+};
+
+export interface PressKeyParams {
+  key: string;
+}
+
+export interface PressKeyResult {
+  key: string;
+  modifiers: string[];
+}
+
+/**
+ * Parse a key string like "Control+a", "Meta+Shift+Enter", or "Tab".
+ */
+function parseKeyCombo(input: string): {
+  modifiers: number;
+  modifierNames: string[];
+  keyInfo: { key: string; code: string; keyCode?: number; text?: string };
+} {
+  const parts = input.split('+');
+  let modifiers = 0;
+  const modifierNames: string[] = [];
+
+  // Last part is the actual key, everything before is a modifier
+  const keyPart = parts.pop()!;
+
+  for (const mod of parts) {
+    const flag = MODIFIER_FLAGS[mod.toLowerCase()];
+    if (flag) {
+      modifiers |= flag;
+      modifierNames.push(mod);
+    }
+  }
+
+  // Look up the key in our map
+  const mapped = KEY_MAP[keyPart.toLowerCase()];
+  if (mapped) {
+    return { modifiers, modifierNames, keyInfo: mapped };
+  }
+
+  // Single character key
+  if (keyPart.length === 1) {
+    const charCode = keyPart.charCodeAt(0);
+    return {
+      modifiers,
+      modifierNames,
+      keyInfo: {
+        key: keyPart,
+        code: `Key${keyPart.toUpperCase()}`,
+        keyCode: charCode,
+        text: modifiers === 0 ? keyPart : undefined
+      }
+    };
+  }
+
+  // Unknown named key — pass through
+  return {
+    modifiers,
+    modifierNames,
+    keyInfo: { key: keyPart, code: keyPart }
+  };
+}
+
+export async function executePressKey(
+  client: Client,
+  params: PressKeyParams
+): Promise<PressKeyResult> {
+  const { modifiers, modifierNames, keyInfo } = parseKeyCombo(params.key);
+
+  await client.Input.dispatchKeyEvent({
+    type: 'keyDown',
+    key: keyInfo.key,
+    code: keyInfo.code,
+    windowsVirtualKeyCode: keyInfo.keyCode,
+    modifiers,
+    ...(keyInfo.text && { text: keyInfo.text })
+  });
+
+  await client.Input.dispatchKeyEvent({
+    type: 'keyUp',
+    key: keyInfo.key,
+    code: keyInfo.code,
+    windowsVirtualKeyCode: keyInfo.keyCode,
+    modifiers
+  });
+
+  return { key: keyInfo.key, modifiers: modifierNames };
+}

package/src/commands/scroll.ts

@@ -0,0 +1,79 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+
+export interface ScrollParams {
+  ref?: string;
+  x?: number;
+  y?: number;
+  direction?: 'up' | 'down' | 'left' | 'right';
+  amount?: number;
+}
+
+export interface ScrollResult {
+  scrollX: number;
+  scrollY: number;
+}
+
+/**
+ * Scroll the page or an element.
+ * - If ref is given, scrolls the element into view.
+ * - If direction/amount is given, scrolls the page.
+ * - If x/y is given, scrolls the page to absolute coordinates.
+ */
+export async function executeScroll(
+  client: Client,
+  refMap: ElementRefMap,
+  params: ScrollParams
+): Promise<ScrollResult> {
+  if (params.ref) {
+    // Scroll element into view
+    const backendNodeId = refMap.resolve(params.ref);
+    await client.DOM.scrollIntoViewIfNeeded({ backendNodeId });
+
+    const { result } = await client.Runtime.evaluate({
+      expression: `JSON.stringify({ scrollX: window.scrollX, scrollY: window.scrollY })`,
+      returnByValue: true
+    });
+    const pos = JSON.parse(String(result.value));
+    return { scrollX: pos.scrollX, scrollY: pos.scrollY };
+  }
+
+  if (params.direction) {
+    const amount = params.amount ?? 300;
+    const deltaX =
+      params.direction === 'left'
+        ? -amount
+        : params.direction === 'right'
+          ? amount
+          : 0;
+    const deltaY =
+      params.direction === 'up'
+        ? -amount
+        : params.direction === 'down'
+          ? amount
+          : 0;
+
+    await client.Input.dispatchMouseEvent({
+      type: 'mouseWheel',
+      x: 0,
+      y: 0,
+      deltaX,
+      deltaY
+    });
+
+    // Small delay for scroll to take effect
+    await new Promise((r) => setTimeout(r, 100));
+  } else if (params.x !== undefined || params.y !== undefined) {
+    await client.Runtime.evaluate({
+      expression: `window.scrollTo(${params.x ?? 0}, ${params.y ?? 0})`,
+      awaitPromise: false
+    });
+  }
+
+  const { result } = await client.Runtime.evaluate({
+    expression: `JSON.stringify({ scrollX: window.scrollX, scrollY: window.scrollY })`,
+    returnByValue: true
+  });
+  const pos = JSON.parse(String(result.value));
+  return { scrollX: pos.scrollX, scrollY: pos.scrollY };
+}

package/src/commands/select.ts

@@ -0,0 +1,78 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
+
+export interface SelectParams {
+  ref: string;
+  value?: string;
+  label?: string;
+  index?: number;
+}
+
+export interface SelectResult {
+  ref: string;
+  selectedValue: string;
+  selectedLabel: string;
+}
+
+/**
+ * Select an option in a <select> element by value, label, or index.
+ */
+export async function executeSelect(
+  client: Client,
+  refMap: ElementRefMap,
+  params: SelectParams
+): Promise<SelectResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  await focusRef(client, resolved.backendNodeId);
+
+  const { result, exceptionDetails } = await client.Runtime.callFunctionOn({
+    objectId: resolved.objectId,
+    functionDeclaration: `function(byValue, byLabel, byIndex) {
+      if (this.tagName.toLowerCase() !== 'select') {
+        throw new Error('Element is not a <select>');
+      }
+      var found = false;
+      for (var i = 0; i < this.options.length; i++) {
+        var opt = this.options[i];
+        if (byValue !== null && opt.value === byValue) {
+          this.selectedIndex = i; found = true; break;
+        }
+        if (byLabel !== null && opt.textContent.trim() === byLabel) {
+          this.selectedIndex = i; found = true; break;
+        }
+        if (byIndex !== null && i === byIndex) {
+          this.selectedIndex = i; found = true; break;
+        }
+      }
+      if (!found) throw new Error('Option not found');
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+      var sel = this.options[this.selectedIndex];
+      return { value: sel.value, label: sel.textContent.trim() };
+    }`,
+    arguments: [
+      { value: params.value ?? null },
+      { value: params.label ?? null },
+      { value: params.index ?? null }
+    ],
+    returnByValue: true,
+    awaitPromise: false
+  });
+
+  if (exceptionDetails) {
+    throw new Error(
+      exceptionDetails.exception?.description ??
+        exceptionDetails.text ??
+        'Failed to select option'
+    );
+  }
+
+  const selected = result.value as { value: string; label: string };
+  return {
+    ref: params.ref,
+    selectedValue: selected.value,
+    selectedLabel: selected.label
+  };
+}

package/src/commands/snapshot.ts

@@ -0,0 +1,18 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { buildSnapshot, formatSnapshot } from '../browser/inspect/snapshot.js';
+import type { SnapshotResult } from '../browser/inspect/types.js';
+
+export interface SnapshotCommandResult {
+  text: string;
+  snapshot: SnapshotResult;
+}
+
+export async function executeSnapshot(
+  client: Client,
+  refMap: ElementRefMap
+): Promise<SnapshotCommandResult> {
+  const snapshot = await buildSnapshot(client, refMap);
+  const text = formatSnapshot(snapshot);
+  return { text, snapshot };
+}

package/src/commands/styles.ts

@@ -0,0 +1,24 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { getStyles, formatStyles } from '../browser/inspect/styles.js';
+import type { StylesResult } from '../browser/inspect/styles.js';
+
+export interface StylesParams {
+  ref: string;
+  properties?: string[];
+}
+
+export interface StylesCommandResult {
+  text: string;
+  styles: StylesResult;
+}
+
+export async function executeStyles(
+  client: Client,
+  refMap: ElementRefMap,
+  params: StylesParams
+): Promise<StylesCommandResult> {
+  const styles = await getStyles(client, refMap, params.ref, params.properties);
+  const text = formatStyles(styles);
+  return { text, styles };
+}

package/src/commands/type.ts

@@ -1,8 +1,11 @@
 import { z } from 'zod';
 import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
 
 export const TypeParamsSchema = z.object({
   text: z.string(),
+  ref: z.string().optional(),
   selector: z.string().optional(),
   clearFirst: z.boolean().default(false),
   delayMs: z.number().int().min(0).default(0)
@@ -30,9 +33,16 @@ async function focusSelector(client: Client, selector: string): Promise<void> {
 
 export async function executeType(
   client: Client,
-  params: TypeParams
+  params: TypeParams,
+  refMap?: ElementRefMap
 ): Promise<TypeResult> {
-  if (params.selector) {
+  if (params.ref) {
+    if (!refMap) {
+      throw new Error('Element ref map is required when using ref parameter');
+    }
+    const resolved = await resolveRef(client, refMap, params.ref);
+    await focusRef(client, resolved.backendNodeId);
+  } else if (params.selector) {
     await focusSelector(client, params.selector);
   }
 

package/src/commands/upload.ts

@@ -0,0 +1,36 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import {
+  validateUploadFiles,
+  uploadFiles
+} from '../browser/interaction/uploads.js';
+
+export interface UploadParams {
+  ref: string;
+  files: string[];
+}
+
+export interface UploadResult {
+  ref: string;
+  files: string[];
+}
+
+export async function executeUpload(
+  client: Client,
+  refMap: ElementRefMap,
+  params: UploadParams
+): Promise<UploadResult> {
+  if (params.files.length === 0) {
+    throw new Error('At least one file is required');
+  }
+
+  const backendNodeId = refMap.resolve(params.ref);
+
+  // Validate files before uploading
+  validateUploadFiles(params.files);
+
+  // Upload to the file input element
+  await uploadFiles(client, backendNodeId, params.files);
+
+  return { ref: params.ref, files: params.files };
+}

package/src/mcp/console-buffer.ts

@@ -1,11 +1,21 @@
 export type ConsoleLevel = 'log' | 'warn' | 'error' | 'info' | 'debug';
 
+export interface StackFrame {
+  url: string;
+  functionName: string;
+  lineNumber: number;
+  columnNumber: number;
+}
+
 export interface ConsoleMessage {
   level: ConsoleLevel;
   text: string;
   url: string;
   lineNumber: number;
+  columnNumber?: number;
   timestamp: number;
+  stackFrames?: StackFrame[];
+  isException?: boolean;
 }
 
 export interface ConsoleFilter {

package/src/mcp/server.ts

@@ -1,9 +1,29 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { BrowserManager } from '../browser.js';
+import { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resetCssState } from '../browser/inspect/styles.js';
+import { DialogQueue } from '../browser/interaction/dialogs.js';
+import { attachDialogHandler } from '../browser/interaction/dialogs.js';
+import {
+  DownloadTracker,
+  enableDownloads,
+  attachDownloadHandler
+} from '../browser/interaction/downloads.js';
+import { NetworkBuffer } from '../browser/network/network-buffer.js';
+import { attachNetworkMonitor } from '../browser/network/network-monitor.js';
 import { loadConfig } from '../util/config.js';
 import { ConsoleBuffer, type ConsoleMessage } from './console-buffer.js';
 import { registerBrowserTools } from './tools/browser.js';
 import { registerErrorTools } from './tools/errors.js';
+import { registerSnapshotTools } from './tools/snapshot.js';
+import { registerInteractionTools } from './tools/interaction.js';
+import { registerNetworkTools } from './tools/network.js';
+import { registerRuntimeTools } from './tools/runtime.js';
+import { registerStylesTools } from './tools/styles.js';
+import { registerFilesTools } from './tools/files.js';
+import { createTraceState } from '../browser/performance/trace.js';
+import { registerPerformanceTools } from './tools/performance.js';
+import { registerConnectTools } from './tools/connect.js';
 
 export interface McpConfig {
   bufferSize: number;
@@ -14,6 +34,11 @@ export interface McpConfig {
 export interface BrowserContext {
   manager: BrowserManager | null;
   consoleBuffer: ConsoleBuffer;
+  elementRefMap: ElementRefMap;
+  networkBuffer: NetworkBuffer;
+  dialogQueue: DialogQueue;
+  downloadTracker: DownloadTracker;
+  traceState: ReturnType<typeof createTraceState>;
   baseConfig: McpConfig;
 }
 
@@ -27,18 +52,57 @@ export async function createMcpServer(config: McpConfig): Promise<{
   });
 
   const consoleBuffer = new ConsoleBuffer(config.bufferSize);
+  const elementRefMap = new ElementRefMap();
+  const networkBuffer = new NetworkBuffer();
+  const dialogQueue = new DialogQueue();
+  const downloadTracker = new DownloadTracker();
+  const traceState = createTraceState();
 
   const context: BrowserContext = {
     manager: null,
     consoleBuffer,
+    elementRefMap,
+    networkBuffer,
+    dialogQueue,
+    downloadTracker,
+    traceState,
     baseConfig: config
   };
 
-  function attachConsoleBuffer(manager: BrowserManager): void {
+  function attachEventHandlers(manager: BrowserManager): void {
+    let monitoredClient: unknown = null;
+
+    function ensureMonitors(): void {
+      const client = manager.getClient();
+      if (client && client !== monitoredClient) {
+        attachNetworkMonitor(client, networkBuffer);
+        attachDialogHandler(client, dialogQueue);
+        attachDownloadHandler(client, downloadTracker);
+        void enableDownloads(client).catch(() => {
+          // Download behavior setup may fail in some environments
+        });
+        monitoredClient = client;
+      }
+    }
+
     manager.setEventCallback((event, data) => {
       if (event === 'console_message') {
         consoleBuffer.push(data as ConsoleMessage);
       }
+      if (event === 'page_navigated') {
+        elementRefMap.invalidate();
+      }
+      if (event === 'browser_connected') {
+        ensureMonitors();
+      }
+      if (event === 'browser_disconnected') {
+        monitoredClient = null;
+        resetCssState();
+        dialogQueue.clear();
+        downloadTracker.clear();
+      }
+      // Reattach after tab switch (new client, no connect event)
+      ensureMonitors();
     });
   }
 
@@ -48,12 +112,20 @@ export async function createMcpServer(config: McpConfig): Promise<{
       cdpHost: config.cdpHost
     });
     const manager = new BrowserManager(browserConfig);
-    attachConsoleBuffer(manager);
+    attachEventHandlers(manager);
     return manager;
   }
 
   registerBrowserTools(server, context, makeBrowserManager);
   registerErrorTools(server, consoleBuffer);
+  registerSnapshotTools(server, context);
+  registerInteractionTools(server, context);
+  registerNetworkTools(server, context);
+  registerRuntimeTools(server, consoleBuffer);
+  registerStylesTools(server, context);
+  registerFilesTools(server, context);
+  registerPerformanceTools(server, context);
+  registerConnectTools(server, context, makeBrowserManager);
 
   return {
     server,