e2e-pilot 0.0.69

package/src/network-capture.ts

@@ -0,0 +1,140 @@
+import type { Page, BrowserContext, Response } from 'playwright-core'
+import fs from 'node:fs'
+import path from 'node:path'
+
+export interface NetworkRequest {
+  ts: number
+  pageIndex: number
+  pageTitle: string
+  method: string
+  url: string
+  status: number
+  reqBody: string | Record<string, unknown> | null
+  resBody: string | Record<string, unknown> | null
+}
+
+function tryParseJson(text: string): string | Record<string, unknown> {
+  try {
+    const parsed = JSON.parse(text)
+    if (typeof parsed === 'object' && parsed !== null) {
+      return parsed
+    }
+    return text
+  } catch {
+    return text
+  }
+}
+
+const MAX_REQUEST_BODY_SIZE = 1024 * 1024  // 1mb
+
+async function captureNetworkRequest({
+  response,
+  pageIndex,
+  pageTitle,
+}: {
+  response: Response
+  pageIndex: number
+  pageTitle: string
+}): Promise<NetworkRequest | null> {
+  const request = response.request()
+  const resourceType = request.resourceType()
+
+  // Only capture XHR/fetch (API calls)
+  if (resourceType !== 'xhr' && resourceType !== 'fetch') {
+    return null
+  }
+
+  let reqBody: string | Record<string, unknown> | null = null
+  let resBody: string | Record<string, unknown> | null = null
+
+  try {
+    const postData = request.postData()
+    if (postData) {
+      const truncated =
+        postData.length > MAX_REQUEST_BODY_SIZE ? postData.slice(0, MAX_REQUEST_BODY_SIZE) + '...' : postData
+      reqBody = tryParseJson(truncated)
+    }
+  } catch {
+    // Ignore errors getting request body
+  }
+
+  try {
+    const body = await response.text()
+    if (body) {
+      const truncated = body.length > MAX_REQUEST_BODY_SIZE ? body.slice(0, MAX_REQUEST_BODY_SIZE) + '...' : body
+      resBody = tryParseJson(truncated)
+    }
+  } catch {
+    // Ignore errors getting response body
+  }
+
+  return {
+    ts: Date.now(),
+    pageIndex,
+    pageTitle,
+    method: request.method(),
+    url: request.url(),
+    status: response.status(),
+    reqBody,
+    resBody,
+  }
+}
+
+export interface NetworkCapture {
+  requests: NetworkRequest[]
+  cleanup: () => void
+}
+
+export function setupNetworkCapture({ context }: { context: BrowserContext }): NetworkCapture {
+  const requests: NetworkRequest[] = []
+  const allPages = context.pages()
+
+  const createHandler = (page: Page, pageIndex: number) => {
+    return async (response: Response) => {
+      let pageTitle = ''
+      try {
+        pageTitle = await page.title()
+      } catch {
+        // Page might be closed or navigating
+      }
+      const captured = await captureNetworkRequest({ response, pageIndex, pageTitle })
+      if (captured) {
+        requests.push(captured)
+      }
+    }
+  }
+
+  const handlers = new Map<Page, (response: Response) => Promise<void>>()
+
+  for (let i = 0; i < allPages.length; i++) {
+    const page = allPages[i]
+    const handler = createHandler(page, i)
+    handlers.set(page, handler)
+    page.on('response', handler)
+  }
+
+  const cleanup = () => {
+    for (const [page, handler] of handlers) {
+      page.off('response', handler)
+    }
+    handlers.clear()
+  }
+
+  return { requests, cleanup }
+}
+
+export function saveNetworkRequests({
+  requests,
+  snapshotDir,
+}: {
+  requests: NetworkRequest[]
+  snapshotDir: string
+}): string | null {
+  if (requests.length === 0) {
+    return null
+  }
+  const networksPath = path.join(snapshotDir, 'networks.jsonl')
+  const jsonlContent = requests.map((req) => JSON.stringify(req)).join('\n')
+  fs.writeFileSync(networksPath, jsonlContent, 'utf-8')
+  return networksPath
+}

package/src/prompt.bak.md

@@ -0,0 +1,323 @@
+# e2e-pilot execute
+
+Control user's Chrome browser via playwright code snippets. Prefer single-line code with semicolons between statements. If you get "Extension not running" error, tell user to click the E2E Pilot extension icon on the tab they want to control.
+
+You can collaborate with the user - they can help with captchas, difficult elements, or reproducing bugs.
+
+## context variables
+
+- `state` - object persisted between calls, use to store data/pages (e.g., `state.myPage = await context.newPage()`)
+- `page` - default page the user activated, use this unless working with multiple pages
+- `context` - browser context, access all pages via `context.pages()`
+- `require` - load Node.js modules like fs
+- Node.js globals: `setTimeout`, `setInterval`, `fetch`, `URL`, `Buffer`, `crypto`, etc.
+
+## rules
+
+- **Multiple calls**: use multiple execute calls for complex logic - helps understand intermediate state and isolate which action failed
+- **Never close**: never call `browser.close()` or `context.close()`. Only close pages you created or if user asks
+- **No bringToFront**: never call unless user asks - it's disruptive and unnecessary, you can interact with background pages
+- **Check state after actions**: always verify page state after clicking/submitting (see next section)
+- **Clean up listeners**: call `page.removeAllListeners()` at end of message to prevent leaks
+- **CDP sessions**: use `getCDPSession({ page })` not `page.context().newCDPSession()` - the latter doesn't work through e2e-pilot relay
+- **Wait for load**: use `page.waitForLoadState('load')` not `page.waitForEvent('load')` - waitForEvent times out if already loaded
+- **Avoid timeouts**: prefer proper waits over `page.waitForTimeout()` - there are better ways to wait for elements
+
+## checking page state
+
+After any action (click, submit, navigate), verify what happened:
+
+```js
+console.log('url:', page.url()); console.log(await accessibilitySnapshot({ page }).then(x => x.split('\n').slice(0, 30).join('\n')));
+```
+
+For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout.
+
+If nothing changed, try `await page.waitForLoadState('networkidle', {timeout: 3000})` or you may have clicked the wrong element.
+
+## accessibility snapshots
+
+```js
+await accessibilitySnapshot({ page, search?, contextLines?, showDiffSinceLastCall? })
+```
+
+- `search` - string/regex to filter results (returns first 10 matches with context)
+- `contextLines` - lines of context around matches (default: 10)
+- `showDiffSinceLastCall` - returns diff since last snapshot (useful after actions)
+
+Example output:
+
+```md
+- banner [ref=e3]:
+    - link "Home" [ref=e5] [cursor=pointer]:
+        - /url: /
+    - navigation [ref=e12]:
+        - link "Docs" [ref=e13] [cursor=pointer]:
+            - /url: /docs
+```
+
+Use `aria-ref` to interact - **no quotes around the ref value**:
+
+```js
+await page.locator('aria-ref=e13').click()
+```
+
+Search for specific elements:
+
+```js
+const snapshot = await accessibilitySnapshot({ page, search: /button|submit/i })
+```
+
+## choosing between snapshot methods
+
+Both `accessibilitySnapshot` and `screenshotWithAccessibilityLabels` use the same `aria-ref` system, so you can combine them effectively.
+
+**Use `accessibilitySnapshot` when:**
+- Page has simple, semantic structure (articles, forms, lists)
+- You need to search for specific text or patterns
+- Token usage matters (text is smaller than images)
+- You need to process the output programmatically
+
+**Use `screenshotWithAccessibilityLabels` when:**
+- Page has complex visual layout (grids, galleries, dashboards, maps)
+- Spatial position matters (e.g., "first image", "top-left button")
+- DOM order doesn't match visual order
+- You need to understand the visual hierarchy
+
+**Combining both:** Use screenshot first to understand layout and identify target elements visually, then use `accessibilitySnapshot({ search: /pattern/ })` for efficient searching in subsequent calls.
+
+## selector best practices
+
+**For unknown websites**: use `accessibilitySnapshot()` with `aria-ref` - it shows what's actually interactive.
+
+**For development** (when you have source code access), prefer stable selectors in this order:
+
+1. **Best**: `[data-testid="submit"]` - explicit test attributes, never change accidentally
+2. **Good**: `getByRole('button', { name: 'Save' })` - accessible, semantic
+3. **Good**: `getByText('Sign in')`, `getByLabel('Email')` - readable, user-facing
+4. **OK**: `input[name="email"]`, `button[type="submit"]` - semantic HTML
+5. **Avoid**: `.btn-primary`, `#submit` - classes/IDs change frequently
+6. **Last resort**: `div.container > form > button` - fragile, breaks easily
+
+Combine locators for precision:
+
+```js
+page.locator('tr').filter({ hasText: 'John' }).locator('button').click()
+page.locator('button').nth(2).click()
+```
+
+If a locator matches multiple elements, Playwright throws "strict mode violation". Use `.first()`, `.last()`, or `.nth(n)`:
+
+```js
+await page.locator('button').first().click()  // first match
+await page.locator('.item').last().click()    // last match
+await page.locator('li').nth(3).click()       // 4th item (0-indexed)
+```
+
+## working with pages
+
+Find a specific page:
+
+```js
+const pages = context.pages().filter(x => x.url().includes('localhost'));
+if (pages.length !== 1) throw new Error(`Expected 1 page, found ${pages.length}`);
+state.targetPage = pages[0];
+```
+
+Create new page:
+
+```js
+state.newPage = await context.newPage();
+await state.newPage.goto('https://example.com');
+```
+
+## common patterns
+
+**Popups** - capture before triggering:
+
+```js
+const [popup] = await Promise.all([page.waitForEvent('popup'), page.click('a[target=_blank]')]);
+await popup.waitForLoadState(); console.log('Popup URL:', popup.url());
+```
+
+**Downloads** - capture and save:
+
+```js
+const [download] = await Promise.all([page.waitForEvent('download'), page.click('button.download')]);
+await download.saveAs(`/tmp/${download.suggestedFilename()}`);
+```
+
+**iFrames** - use frameLocator:
+
+```js
+const frame = page.frameLocator('#my-iframe');
+await frame.locator('button').click();
+```
+
+**Dialogs** - handle alerts/confirms/prompts:
+
+```js
+page.on('dialog', async dialog => { console.log(dialog.message()); await dialog.accept(); });
+await page.click('button.trigger-alert');
+```
+
+## utility functions
+
+**getLatestLogs** - retrieve captured browser console logs (up to 5000 per page, cleared on navigation):
+
+```js
+await getLatestLogs({ page?, count?, search? })
+// Examples:
+const errors = await getLatestLogs({ search: /error/i, count: 50 })
+const pageLogs = await getLatestLogs({ page })
+```
+
+For custom log collection across runs, store in state: `state.logs = []; page.on('console', m => state.logs.push(m.text()))`
+
+**waitForPageLoad** - smart load detection that ignores analytics/ads:
+
+```js
+await waitForPageLoad({ page, timeout?, pollInterval?, minWait? })
+// Returns: { success, readyState, pendingRequests, waitTimeMs, timedOut }
+```
+
+**getCDPSession** - send raw CDP commands:
+
+```js
+const cdp = await getCDPSession({ page });
+const metrics = await cdp.send('Page.getLayoutMetrics');
+```
+
+**getLocatorStringForElement** - get stable selector from ephemeral aria-ref:
+
+```js
+const selector = await getLocatorStringForElement(page.locator('aria-ref=e14'));
+// => "getByRole('button', { name: 'Save' })"
+```
+
+**getReactSource** - get React component source location (dev mode only):
+
+```js
+const source = await getReactSource({ locator: page.locator('aria-ref=e5') });
+// => { fileName, lineNumber, columnNumber, componentName }
+```
+
+**getStylesForLocator** - inspect CSS styles applied to an element, like browser DevTools "Styles" panel. Useful for debugging styling issues, finding where a CSS property is defined (file:line), and checking inherited styles. Returns selector, source location, and declarations for each matching rule. ALWAYS read `https://playwriter.dev/resources/styles-api.md` first.
+
+```js
+const styles = await getStylesForLocator({ locator: page.locator('.btn'), cdp: await getCDPSession({ page }) });
+console.log(formatStylesAsText(styles));
+```
+
+**createDebugger** - set breakpoints, step through code, inspect variables at runtime. Useful for debugging issues that only reproduce in browser, understanding code flow, and inspecting state at specific points. Can pause on exceptions, evaluate expressions in scope, and blackbox framework code. ALWAYS read `https://playwriter.dev/resources/debugger-api.md` first.
+
+```js
+const cdp = await getCDPSession({ page }); const dbg = createDebugger({ cdp }); await dbg.enable();
+const scripts = await dbg.listScripts({ search: 'app' });
+await dbg.setBreakpoint({ file: scripts[0].url, line: 42 });
+// when paused: dbg.inspectLocalVariables(), dbg.stepOver(), dbg.resume()
+```
+
+**createEditor** - view and live-edit page scripts and CSS at runtime. Edits are in-memory (persist until reload). Useful for testing quick fixes, searching page scripts with grep, and toggling debug flags. ALWAYS read `https://playwriter.dev/resources/editor-api.md` first.
+
+```js
+const cdp = await getCDPSession({ page }); const editor = createEditor({ cdp }); await editor.enable();
+const matches = await editor.grep({ regex: /console\.log/ });
+await editor.edit({ url: matches[0].url, oldString: 'DEBUG = false', newString: 'DEBUG = true' });
+```
+
+**screenshotWithAccessibilityLabels** - take a screenshot with Vimium-style visual labels overlaid on interactive elements. Shows labels, captures screenshot, then removes labels. The image and accessibility snapshot are automatically included in the response. Can be called multiple times to capture multiple screenshots. Use a timeout of **20 seconds** for complex pages.
+
+Prefer this for pages with grids, image galleries, maps, or complex visual layouts where spatial position matters. For simple text-heavy pages, `accessibilitySnapshot` with search is faster and uses fewer tokens.
+
+```js
+await screenshotWithAccessibilityLabels({ page });
+// Image and accessibility snapshot are automatically included in response
+// Use aria-ref from snapshot to interact with elements
+await page.locator('aria-ref=e5').click();
+
+// Can take multiple screenshots in one execution
+await screenshotWithAccessibilityLabels({ page });
+await page.click('button');
+await screenshotWithAccessibilityLabels({ page });
+// Both images are included in the response
+```
+
+Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs.
+
+## pinned elements
+
+Users can right-click → "Copy E2E Pilot Element Reference" to store elements in `globalThis.e2ePilotPinnedElem1` (increments for each pin). The reference is copied to clipboard:
+
+```js
+const el = await page.evaluateHandle(() => globalThis.e2ePilotPinnedElem1);
+await el.click();
+```
+
+## page.evaluate
+
+Code inside `page.evaluate()` runs in the browser - use plain JavaScript only, no TypeScript syntax. Return values and log outside (console.log inside evaluate runs in browser, not visible):
+
+```js
+const title = await page.evaluate(() => document.title);
+console.log('Title:', title);
+
+const info = await page.evaluate(() => ({
+    url: location.href,
+    buttons: document.querySelectorAll('button').length,
+}));
+console.log(info);
+```
+
+## loading files
+
+Fill inputs with file content:
+
+```js
+const fs = require('node:fs'); const content = fs.readFileSync('./data.txt', 'utf-8'); await page.locator('textarea').fill(content);
+```
+
+## network interception
+
+For scraping or reverse-engineering APIs, intercept network requests instead of scrolling DOM. Store in `state` to analyze across calls:
+
+```js
+state.requests = []; state.responses = [];
+page.on('request', req => { if (req.url().includes('/api/')) state.requests.push({ url: req.url(), method: req.method(), headers: req.headers() }); });
+page.on('response', async res => { if (res.url().includes('/api/')) { try { state.responses.push({ url: res.url(), status: res.status(), body: await res.json() }); } catch {} } });
+```
+
+Then trigger actions (scroll, click, navigate) and analyze captured data:
+
+```js
+console.log('Captured', state.responses.length, 'API calls');
+state.responses.forEach(r => console.log(r.status, r.url.slice(0, 80)));
+```
+
+Inspect a specific response to understand schema:
+
+```js
+const resp = state.responses.find(r => r.url.includes('users'));
+console.log(JSON.stringify(resp.body, null, 2).slice(0, 2000));
+```
+
+Replay API directly (useful for pagination):
+
+```js
+const { url, headers } = state.requests.find(r => r.url.includes('feed'));
+const data = await page.evaluate(async ({ url, headers }) => { const res = await fetch(url, { headers }); return res.json(); }, { url, headers });
+console.log(data);
+```
+
+Clean up listeners when done: `page.removeAllListeners('request'); page.removeAllListeners('response');`
+
+## capabilities
+
+Examples of what e2e-pilot can do:
+- Monitor console logs while user reproduces a bug
+- Intercept network requests to reverse-engineer APIs and build SDKs
+- Scrape data by replaying paginated API calls instead of scrolling DOM
+- Get accessibility snapshot to find elements, then automate interactions
+- Use visual screenshots to understand complex layouts like image grids, dashboards, or maps
+- Debug issues by collecting logs and controlling the page simultaneously
+- Handle popups, downloads, iframes, and dialog boxes

package/src/prompt.md

@@ -0,0 +1,7 @@
+Control user's Chrome browser via playwright code snippets. If you get "Extension not running" error, tell user to click the E2E Pilot extension icon on the tab they want to control.
+
+Each `execute` call saves accessibility snapshots to `.e2e-pilot/snapshots/`. Read these files to:
+
+- Understand page structure before acting
+- Find element locators (look for `aria-ref` values)
+- Debug when actions fail

package/src/protocol.ts

@@ -0,0 +1,63 @@
+import { CDPEventFor, ProtocolMapping } from './cdp-types.js'
+
+export const VERSION = 1
+
+type ForwardCDPCommand =
+  {
+    [K in keyof ProtocolMapping.Commands]: {
+      id: number
+      method: 'forwardCDPCommand'
+      params: {
+        method: K
+        sessionId?: string
+        params?: ProtocolMapping.Commands[K]['paramsType'][0]
+      }
+    }
+  }[keyof ProtocolMapping.Commands]
+
+export type ExtensionCommandMessage = ForwardCDPCommand
+
+export type ExtensionResponseMessage = {
+  id: number
+  method?: undefined
+  result?: any
+  error?: string
+}
+
+/**
+ * This produces a discriminated union for narrowing, similar to ForwardCDPCommand,
+ * but for forwarded CDP events. Uses CDPEvent to maintain proper type extraction.
+ */
+export type ExtensionEventMessage =
+  {
+    [K in keyof ProtocolMapping.Events]: {
+      id?: undefined
+      method: 'forwardCDPEvent'
+      params: {
+        method: CDPEventFor<K>['method']
+        sessionId?: string
+        params?: CDPEventFor<K>['params']
+      }
+    }
+  }[keyof ProtocolMapping.Events]
+
+export type ExtensionLogMessage = {
+  id?: undefined
+  method: 'log'
+  params: {
+    level: 'log' | 'debug' | 'info' | 'warn' | 'error'
+    args: string[]
+  }
+}
+
+export type ExtensionPongMessage = {
+  id?: undefined
+  method: 'pong'
+}
+
+export type ServerPingMessage = {
+  method: 'ping'
+  id?: undefined
+}
+
+export type ExtensionMessage = ExtensionResponseMessage | ExtensionEventMessage | ExtensionLogMessage | ExtensionPongMessage

package/src/react-source.ts

@@ -0,0 +1,94 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { Page, Locator, ElementHandle } from 'playwright-core'
+import type { ICDPSession, CDPSession } from './cdp-session.js'
+
+export interface ReactSourceLocation {
+  fileName: string | null
+  lineNumber: number | null
+  columnNumber: number | null
+  componentName: string | null
+}
+
+let bippyCode: string | null = null
+
+function getBippyCode(): string {
+  if (bippyCode) {
+    return bippyCode
+  }
+  const currentDir = path.dirname(fileURLToPath(import.meta.url))
+  const bippyPath = path.join(currentDir, '..', 'dist', 'bippy.js')
+  bippyCode = fs.readFileSync(bippyPath, 'utf-8')
+  return bippyCode
+}
+
+export async function getReactSource({
+  locator,
+  cdp: cdpSession,
+}: {
+  locator: Locator | ElementHandle
+  cdp: ICDPSession
+}): Promise<ReactSourceLocation | null> {
+  // Cast to CDPSession for internal type safety - at runtime both are compatible
+  const cdp = cdpSession as CDPSession
+  const page: Page = 'page' in locator && typeof locator.page === 'function' ? locator.page() : (locator as any)._page
+
+  if (!page) {
+    throw new Error('Could not get page from locator')
+  }
+
+  const hasBippy = await page.evaluate(() => !!(globalThis as any).__bippy)
+
+  if (!hasBippy) {
+    const code = getBippyCode()
+    await cdp.send('Runtime.evaluate', { expression: code })
+  }
+
+  const result = await (locator as any).evaluate(async (el: any) => {
+    const bippy = (globalThis as any).__bippy
+    if (!bippy) {
+      throw new Error('bippy not loaded')
+    }
+
+    const fiber = bippy.getFiberFromHostInstance(el)
+    if (!fiber) {
+      return { _notFound: 'fiber' as const }
+    }
+
+    const source = await bippy.getSource(fiber)
+    if (source) {
+      return {
+        fileName: source.fileName ? bippy.normalizeFileName(source.fileName) : null,
+        lineNumber: source.lineNumber ?? null,
+        columnNumber: source.columnNumber ?? null,
+        componentName: source.functionName ?? bippy.getDisplayName(fiber.type) ?? null,
+      }
+    }
+
+    const ownerStack = await bippy.getOwnerStack(fiber)
+    for (const frame of ownerStack) {
+      if (frame.fileName && bippy.isSourceFile(frame.fileName)) {
+        return {
+          fileName: bippy.normalizeFileName(frame.fileName),
+          lineNumber: frame.lineNumber ?? null,
+          columnNumber: frame.columnNumber ?? null,
+          componentName: frame.functionName ?? null,
+        }
+      }
+    }
+
+    return { _notFound: 'source' as const }
+  })
+
+  if (result && '_notFound' in result) {
+    if (result._notFound === 'fiber') {
+      console.warn('[getReactSource] no fiber found - is this a React element?')
+    } else {
+      console.warn('[getReactSource] no source location found - is this a React dev build?')
+    }
+    return null
+  }
+
+  return result
+}