jev-cdp 0.1.2 → 0.1.4

package/README.md

@@ -27,8 +27,8 @@ This is an early experimental port. See [NOTICE.md](NOTICE.md) for source attrib
 Run the published CLI without adding it to a project. Bun must be installed for either command:
 
 ```bash
-bunx jev-cdp@0.1.2 help run
-npx -y jev-cdp@0.1.2 help run
+bunx jev-cdp@0.1.3 help run
+npx -y jev-cdp@0.1.3 help run
 ```
 
 Chrome with a CDP endpoint and `TYPESAFE_API_KEY` are required for browser runs. FFmpeg is required for `--recording`.
@@ -96,10 +96,12 @@ bun run run -- run \
   --final-state
 ```
 
-`--recording` captures Chrome's compositor screencast stream for the full goal and renders an H.264 MP4. Because Chrome's native pointer is not part of that stream, the adapter draws a high-contrast cursor that starts at the viewport center, glides to each target, and pulses on clicks. `--interaction-pauses` adds a deterministic delay in milliseconds after moving to a click target and before pressing the mouse; Jev does not choose or observe this delay. `--screenshot` saves the final viewport after the goal stops; when recording is also enabled, it reuses the final screencast frame.
+`--recording` captures Chrome's compositor screencast stream for the full goal and renders an H.264 MP4. Because Chrome's native pointer is not part of that stream, the adapter draws a high-contrast cursor that starts at the viewport center, glides to each target, and pulses on clicks. `--interaction-pauses` adds a deterministic delay after opening or loading a page, switching to an attached tab, or changing the URL within a page. Jev chooses during that delay, and the adapter waits only for any time left before acting. The same flag also pauses after moving to a click target and before pressing the mouse. Jev does not choose or observe these delays. `--screenshot` saves the final viewport after the goal stops; when recording is also enabled, it reuses the final screencast frame.
 
 `--final-state` adds an AI-oriented semantic snapshot to the final JSON on standard output. It includes the final URL, title, visible text, viewport, scroll state, actionable elements, accessible labels, and control state such as `pressed`, `checked`, `selected`, and `expanded`. Progress remains on standard error, so a coding agent can parse standard output as one JSON object and choose the next bounded goal without another browser observation.
 
+The snapshot includes visible controls in direct child iframes, including cross-origin frames. Clicking a link there uses its frame coordinates and checks that the observed control is still current. If that click opens a new tab, Jev switches to it, returns its target ID for the next goal, and keeps the recording and 1120×780 viewport consistent across the switch.
+
 ## Supply known field values without another LLM
 
 For portable QA scenarios, let the planner provide exact test data instead of invoking the Luna fallback. Match a field by its observed accessible label:
@@ -181,7 +183,7 @@ The context is disposed after evidence capture by default. Combine it with `--ke
 | `--tab TARGET_ID` | — | create a new tab | Attach to one exact existing Chrome page target |
 | `tabs` | — | — | Print target IDs, titles, and URLs for open page tabs |
 | `--recording PATH.mp4` | — | off | Record the complete goal with an animated cursor |
-| `--interaction-pauses MS` | — | `0` | Wait after moving to a click target, before mousedown |
+| `--interaction-pauses MS` | — | `0` | Pause after page loads and before clicks, overlapping page pauses with Jev decisions |
 | `--screenshot PATH.jpg` | — | off | Save the final browser viewport |
 | `--final-state` | — | off | Include the final semantic page state in stdout JSON |
 | `--field-value LABEL=VALUE` | — | Luna fallback | Type caller-provided test data into the exactly labeled field |

package/dist/cli.js

@@ -3,7 +3,7 @@
 // package.json
 var package_default = {
   name: "jev-cdp",
-  version: "0.1.2",
+  version: "0.1.4",
   description: "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   type: "module",
   license: "MIT",
@@ -257,6 +257,24 @@ var snapshot_default = `// Ported from browser-use/jev-ultrafast at commit 452c1
 
 // src/browser.ts
 var MARKER = `(() => { const state=${snapshot_default}; return state?.marker ?? null; })()`;
+var RECORDING_CURSOR_INIT = `(() => {
+  if (window !== window.top) return;
+  const mount = () => {
+    if (!document.documentElement) return false;
+    if (document.getElementById('__jev-recording-cursor')) return true;
+    const cursor = document.createElement('div');
+    cursor.id = '__jev-recording-cursor';
+    cursor.setAttribute('aria-hidden', 'true');
+    cursor.innerHTML = '<svg width="28" height="34" viewBox="0 0 28 34" xmlns="http://www.w3.org/2000/svg"><path d="M2 2v25l7-7 5 11 5-2-5-11h10z" fill="white" stroke="#111827" stroke-width="2.5" stroke-linejoin="round"/></svg>';
+    Object.assign(cursor.style, {position:'fixed',left:'50vw',top:'50vh',width:'28px',height:'34px',zIndex:'2147483647',pointerEvents:'none',filter:'drop-shadow(0 2px 2px rgba(0,0,0,.35))',transition:'left 180ms cubic-bezier(.2,.8,.2,1), top 180ms cubic-bezier(.2,.8,.2,1)',transform:'translate(-3px,-3px)'});
+    document.documentElement.append(cursor);
+    return true;
+  };
+  if (!mount()) {
+    const observer = new MutationObserver(() => { if (mount()) observer.disconnect(); });
+    observer.observe(document, { childList: true });
+  }
+})()`;
 
 class StalePageError extends Error {
 }
@@ -294,8 +312,20 @@ class Browser {
   #recordingSequence = 0;
   #recordingWrites = Promise.resolve();
   #stopRecordingEvents;
+  #stopPageEvents = [];
+  #mainFrameId;
+  #pageLoadPromise = null;
+  #resolvePageLoad;
+  #pauseUntil = 0;
+  #frameSessions = new Map;
+  #frameContexts = new Map;
+  #knownTargets = new Set;
+  #openedTabs = [];
+  #cdpUrl;
+  #switchOnPopup = false;
   constructor(cdp, sessionId, targetId, ownsTarget, browserContextId, options) {
     this.#cdp = cdp;
+    this.#cdpUrl = options.cdpUrl;
     this.#sessionId = sessionId;
     this.#targetId = targetId;
     this.#ownsTarget = ownsTarget;
@@ -353,7 +383,20 @@ class Browser {
       flatten: true
     });
     const browser = new Browser(cdp, attached.sessionId, targetId, ownsTarget, browserContextId, options);
+    browser.#knownTargets = new Set((await listChromeTargets(options.cdpUrl)).map((target) => target.id));
     try {
+      browser.#stopPageEvents.push(cdp.on("Target.attachedToTarget", browser.#sessionId, (params) => {
+        const info = params.targetInfo;
+        if (info?.type === "iframe" && info.targetId && typeof params.sessionId === "string")
+          browser.#frameSessions.set(info.targetId, params.sessionId);
+      }));
+      browser.#stopPageEvents.push(cdp.on("Page.frameNavigated", browser.#sessionId, (params) => {
+        const frame = params.frame;
+        if (frame?.id)
+          browser.#frameContexts.delete(frame.id);
+      }));
+      await browser.call("Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: false, flatten: true });
+      await browser.call("Page.enable");
       await browser.call("Emulation.setDeviceMetricsOverride", {
         width: 1120,
         height: 780,
@@ -361,9 +404,11 @@ class Browser {
         mobile: false
       });
       await browser.call("Emulation.setFocusEmulationEnabled", { enabled: true });
+      await browser.watchPageLoads();
       if (options.url)
         await browser.call("Page.navigate", { url: options.url });
       await browser.waitForReady();
+      browser.#pauseUntil = performance.now() + browser.#interactionPauses;
       await browser.startRecording();
       return browser;
     } catch (error) {
@@ -377,23 +422,83 @@ class Browser {
   get targetId() {
     return this.#targetId;
   }
+  async watchPageLoads() {
+    if (!this.#interactionPauses)
+      return;
+    await this.call("Page.enable");
+    const tree = await this.call("Page.getFrameTree");
+    this.#mainFrameId = tree.frameTree.frame.id;
+    const loading = (params) => {
+      if (params.frameId !== this.#mainFrameId || this.#pageLoadPromise)
+        return;
+      this.#pageLoadPromise = new Promise((resolve2) => {
+        this.#resolvePageLoad = resolve2;
+      });
+    };
+    const loaded = () => {
+      if (!this.#pageLoadPromise)
+        return;
+      this.#pauseUntil = performance.now() + this.#interactionPauses;
+      this.#resolvePageLoad?.();
+      this.#resolvePageLoad = undefined;
+      this.#pageLoadPromise = null;
+    };
+    this.#stopPageEvents.push(this.#cdp.on("Page.frameStartedLoading", this.#sessionId, loading), this.#cdp.on("Page.frameNavigated", this.#sessionId, (params) => {
+      const frame = params.frame;
+      if (frame && frame.id === this.#mainFrameId && !frame.parentId)
+        loading({ frameId: frame.id });
+    }), this.#cdp.on("Page.loadEventFired", this.#sessionId, loaded), this.#cdp.on("Page.frameStoppedLoading", this.#sessionId, (params) => {
+      if (params.frameId === this.#mainFrameId)
+        loaded();
+    }), this.#cdp.on("Page.navigatedWithinDocument", this.#sessionId, (params) => {
+      if (params.frameId === this.#mainFrameId) {
+        this.#pauseUntil = performance.now() + this.#interactionPauses;
+        this.resetRecordingCursor().catch(() => {
+          return;
+        });
+      }
+    }));
+  }
+  async waitForInteractionPause() {
+    if (!this.#interactionPauses)
+      return;
+    while (true) {
+      const pageLoad = this.#pageLoadPromise;
+      if (pageLoad) {
+        await Promise.race([
+          pageLoad,
+          Bun.sleep(15000).then(() => {
+            throw new Error("Page did not finish loading within 15 seconds");
+          })
+        ]);
+        continue;
+      }
+      const remaining = this.#pauseUntil - performance.now();
+      if (remaining <= 0)
+        return;
+      await Bun.sleep(remaining);
+    }
+  }
   async startRecording() {
     if (!this.#recordingPath)
       return;
     if (!Bun.which("ffmpeg"))
       throw new Error("--recording requires ffmpeg on PATH");
-    this.#recordingDirectory = await mkdtemp(join(tmpdir(), "jev-cdp-recording-"));
+    const firstSegment = !this.#recordingDirectory;
+    if (firstSegment)
+      this.#recordingDirectory = await mkdtemp(join(tmpdir(), "jev-cdp-recording-"));
     await this.call("Page.enable");
+    await this.call("Page.addScriptToEvaluateOnNewDocument", { source: RECORDING_CURSOR_INIT });
     const viewport = await this.evaluate("({width: innerWidth, height: innerHeight})");
     if (!viewport)
       throw new Error("Could not read the recording viewport");
     await this.animateCursor(viewport.width / 2, viewport.height / 2);
     const initial = await this.call("Page.captureScreenshot", { format: "jpeg", quality: 70 });
-    const initialPath = join(this.#recordingDirectory, "000000.jpg");
+    const initialPath = join(this.#recordingDirectory, `${String(this.#recordingSequence++).padStart(6, "0")}.jpg`);
     await Bun.write(initialPath, Buffer.from(initial.data, "base64"));
-    this.#recordingFrames.push({ path: initialPath, elapsedMs: 0 });
-    this.#recordingSequence = 1;
-    this.#recordingStartedAt = performance.now();
+    if (firstSegment)
+      this.#recordingStartedAt = performance.now();
+    this.#recordingFrames.push({ path: initialPath, elapsedMs: firstSegment ? 0 : performance.now() - this.#recordingStartedAt });
     this.#stopRecordingEvents = this.#cdp.on("Page.screencastFrame", this.#sessionId, (params) => {
       const frame = params;
       this.call("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => {
@@ -485,25 +590,37 @@ class Browser {
   async animateCursor(x, y, click = false) {
     if (!this.#recordingPath)
       return;
+    await this.evaluate(RECORDING_CURSOR_INIT);
     await this.evaluate(`(point => {
-      let cursor=document.getElementById('__jev-recording-cursor');
-      if (!cursor) {
-        cursor=document.createElement('div');
-        cursor.id='__jev-recording-cursor'; cursor.setAttribute('aria-hidden','true');
-        cursor.innerHTML='<svg width="28" height="34" viewBox="0 0 28 34" xmlns="http://www.w3.org/2000/svg"><path d="M2 2v25l7-7 5 11 5-2-5-11h10z" fill="white" stroke="#111827" stroke-width="2.5" stroke-linejoin="round"/></svg>';
-        Object.assign(cursor.style,{position:'fixed',left:'50vw',top:'50vh',width:'28px',height:'34px',zIndex:'2147483647',pointerEvents:'none',filter:'drop-shadow(0 2px 2px rgba(0,0,0,.35))',transition:'left 180ms cubic-bezier(.2,.8,.2,1), top 180ms cubic-bezier(.2,.8,.2,1)',transform:'translate(-3px,-3px)'});
-        document.documentElement.append(cursor);
-      }
+      const cursor=document.getElementById('__jev-recording-cursor');
+      if (!cursor) return;
       cursor.style.left=point.x+'px'; cursor.style.top=point.y+'px';
       if (point.click) cursor.animate([{transform:'translate(-3px,-3px) scale(1)'},{transform:'translate(-3px,-3px) scale(.72)'},{transform:'translate(-3px,-3px) scale(1)'}],{duration:260,easing:'ease-out'});
     })(${JSON.stringify({ x, y, click })})`);
     await Bun.sleep(click ? 80 : 200);
   }
+  async resetRecordingCursor() {
+    if (!this.#recordingPath)
+      return;
+    await this.evaluate(`${RECORDING_CURSOR_INIT}; (() => {
+      const cursor = document.getElementById('__jev-recording-cursor');
+      if (!cursor) return;
+      cursor.style.transition = 'none';
+      cursor.style.left = '50vw';
+      cursor.style.top = '50vh';
+      requestAnimationFrame(() => { cursor.style.transition = 'left 180ms cubic-bezier(.2,.8,.2,1), top 180ms cubic-bezier(.2,.8,.2,1)'; });
+    })()`);
+  }
   async waitForReady() {
     const deadline = Date.now() + 15000;
     while (Date.now() < deadline) {
-      if (await this.evaluate("document.readyState") === "complete")
-        return;
+      try {
+        if (!this.#pageLoadPromise && await this.evaluate("document.readyState") === "complete")
+          return;
+      } catch (error) {
+        if (!(error instanceof StalePageError))
+          throw error;
+      }
       await Bun.sleep(20);
     }
     throw new Error("Page did not finish loading within 15 seconds");
@@ -514,6 +631,64 @@ class Browser {
       throw new StalePageError("Document changed during evaluation");
     return response.result?.value;
   }
+  async evaluateFrame(frameId, expression) {
+    const session = this.#frameSessions.get(frameId);
+    let contextId = this.#frameContexts.get(frameId);
+    if (!session && !contextId) {
+      const context = await this.call("Page.createIsolatedWorld", { frameId });
+      contextId = context.executionContextId;
+      this.#frameContexts.set(frameId, contextId);
+    }
+    const response = await this.#cdp.command("Runtime.evaluate", { expression, returnByValue: true, ...contextId ? { contextId } : {} }, session ?? this.#sessionId);
+    if (response.exceptionDetails)
+      throw new StalePageError("Frame changed during evaluation");
+    return response.result?.value;
+  }
+  async frameOffset(frameId) {
+    const owner = await this.call("DOM.getFrameOwner", { frameId });
+    const box = await this.call("DOM.getBoxModel", { backendNodeId: owner.backendNodeId });
+    return { x: box.model.content[0], y: box.model.content[1] };
+  }
+  async childFrames() {
+    const tree = await this.call("Page.getFrameTree");
+    return (tree.frameTree.childFrames ?? []).map((child) => child.frame.id);
+  }
+  async discoverTabs() {
+    if (!this.#switchOnPopup)
+      return;
+    for (const target of await listChromeTargets(this.#cdpUrl)) {
+      if (target.type !== "page" || this.#knownTargets.has(target.id))
+        continue;
+      this.#knownTargets.add(target.id);
+      const attached = await this.#cdp.command("Target.attachToTarget", { targetId: target.id, flatten: true });
+      try {
+        await this.#cdp.command("Emulation.setDeviceMetricsOverride", {
+          width: 1120,
+          height: 780,
+          deviceScaleFactor: 1,
+          mobile: false
+        }, attached.sessionId);
+      } finally {
+        if (this.#switchOnPopup) {
+          if (this.#recordingPath) {
+            await this.call("Page.stopScreencast").catch(() => {
+              return;
+            });
+            this.#stopRecordingEvents?.();
+            this.#stopRecordingEvents = undefined;
+            await this.#recordingWrites;
+          }
+          this.#sessionId = attached.sessionId;
+          this.#targetId = target.id;
+          this.#switchOnPopup = false;
+          await this.startRecording();
+        } else {
+          await this.#cdp.command("Target.detachFromTarget", { sessionId: attached.sessionId });
+        }
+      }
+      this.#openedTabs.push({ id: target.id, url: target.url, title: target.title });
+    }
+  }
   async settleAfterInput() {
     const action = this.#afterInput;
     this.#afterInput = null;
@@ -549,6 +724,7 @@ class Browser {
   }
   async observe(screenshot = this.#screenshots) {
     await this.settleAfterInput();
+    await this.discoverTabs();
     let info;
     for (let attempt = 0;attempt < 10; attempt++) {
       try {
@@ -563,6 +739,36 @@ class Browser {
     }
     if (!info)
       throw new StalePageError("Document is navigating");
+    if (this.#openedTabs.length) {
+      const targets = await listChromeTargets(this.#cdpUrl);
+      this.#openedTabs = this.#openedTabs.map((tab) => {
+        const current = targets.find((target) => target.id === tab.id);
+        return current ? { ...tab, url: current.url, title: current.title } : tab;
+      });
+      info.text = `${info.text}
+${this.#openedTabs.map((tab) => `Opened new tab: ${tab.title} ${tab.url} (target ${tab.id})`).join(`
+`)}`;
+    }
+    for (const frameId of await this.childFrames()) {
+      try {
+        const offset = await this.frameOffset(frameId);
+        const child = await this.evaluateFrame(frameId, snapshot_default);
+        if (!child)
+          continue;
+        info.text = `${info.text}
+${child.text}`.slice(0, 6000);
+        for (const action of child.actions) {
+          if (!action.node || !action.rect)
+            continue;
+          const rect = { ...action.rect, x: action.rect.x + offset.x, y: action.rect.y + offset.y };
+          if (rect.x < 0 || rect.y < 0 || rect.x >= info.w || rect.y >= info.h)
+            continue;
+          info.actions.push({ ...action, frameId, rect, id: `e${info.actions.length + 1}` });
+          info.guards[`${frameId}:${action.node}`] = child.guards[String(action.node)];
+        }
+        info.marker = [info.marker, frameId, child.marker];
+      } catch {}
+    }
     const page = { ...info, fingerprint: fingerprint(info) };
     if (screenshot) {
       const capture = await this.call("Page.captureScreenshot", { format: "jpeg", quality: 72 });
@@ -571,6 +777,12 @@ class Browser {
     return page;
   }
   async fresh(page, action) {
+    if (action?.frameId && typeof action.node === "number") {
+      const current = await this.evaluateFrame(action.frameId, `(() => {
+        const c=window.__jevFast; return c ? c.guard(c.nodes.get(${action.node})) : null;
+      })()`);
+      return stableStringify(current) === stableStringify(page.guards[`${action.frameId}:${action.node}`]);
+    }
     if (action && (action.kind === "click" || action.kind === "select")) {
       if (typeof action.node !== "number")
         return false;
@@ -603,7 +815,11 @@ class Browser {
     }
     if (typeof action.node !== "number")
       throw new Error("Invalid observed node");
-    const target = await this.evaluate(`(action => {
+    const target = await (action.frameId ? this.evaluateFrame(action.frameId, `(action => {
+      const e=window.__jevFast?.nodes.get(action.node);
+      if (!e?.isConnected || !e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return null;
+      const r=e.getBoundingClientRect(); return {x:r.x+r.width/2,y:r.y+r.height/2};
+    })(${JSON.stringify(action)})`) : this.evaluate(`(action => {
       const e=window.__jevFast?.nodes.get(action.node);
       if (!e?.isConnected || e.matches(':disabled') || e.closest('[aria-disabled="true"],[inert]') ||
           !e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return null;
@@ -619,12 +835,17 @@ class Browser {
         e.dispatchEvent(new Event('change',{bubbles:true}));
       }
       return {x,y};
-    })(${JSON.stringify(action)})`);
+    })(${JSON.stringify(action)})`));
     if (!target) {
       if (action.kind === "select")
         throw new Error("Dropdown execution was not confirmed");
       throw new StalePageError("Target changed or is covered");
     }
+    if (action.frameId) {
+      const offset = await this.frameOffset(action.frameId);
+      target.x += offset.x;
+      target.y += offset.y;
+    }
     await this.animateCursor(target.x, target.y);
     if (action.kind !== "select") {
       if (action.kind === "click" && this.#interactionPauses > 0) {
@@ -665,11 +886,18 @@ class Browser {
       }
     }
     this.#afterInput = action;
+    if (action.frameId && action.kind === "click") {
+      this.#switchOnPopup = true;
+      await Bun.sleep(650);
+    }
   }
   async close() {
     if (this.#closed)
       return;
     this.#closed = true;
+    for (const stop of this.#stopPageEvents)
+      stop();
+    this.#stopPageEvents = [];
     let failure;
     try {
       if (this.#recordingPath)
@@ -1150,6 +1378,7 @@ class Agent {
         this.#textCalls.push({ ...helper, field: action.label, value: text });
       }
     }
+    await this.#browser.waitForInteractionPause();
     await this.#browser.act(action, page, text ?? undefined);
     this.#pendingText = null;
     const entry = {
@@ -1279,7 +1508,8 @@ Browser behavior:
                                  [env: JEV_BROWSER_VISIBLE=1]
   --keep-open                    Leave a runner-created tab or context open.
                                  [env: JEV_BROWSER_KEEP_OPEN=1]
-  --interaction-pauses <ms>      Pause after moving to a click target, before mousedown.
+  --interaction-pauses <ms>      Pause after page loads and before clicks. Jev decisions
+                                 run during page pauses, so only remaining time is waited.
 
 Known field values:
   --field-value <label=value>    Type an exact non-secret value when that accessible

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-cdp",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   "type": "module",
   "license": "MIT",