jev-cdp 0.1.3 → 0.1.4

package/README.md

@@ -100,6 +100,8 @@ bun run run -- run \
 
 `--final-state` adds an AI-oriented semantic snapshot to the final JSON on standard output. It includes the final URL, title, visible text, viewport, scroll state, actionable elements, accessible labels, and control state such as `pressed`, `checked`, `selected`, and `expanded`. Progress remains on standard error, so a coding agent can parse standard output as one JSON object and choose the next bounded goal without another browser observation.
 
+The snapshot includes visible controls in direct child iframes, including cross-origin frames. Clicking a link there uses its frame coordinates and checks that the observed control is still current. If that click opens a new tab, Jev switches to it, returns its target ID for the next goal, and keeps the recording and 1120×780 viewport consistent across the switch.
+
 ## Supply known field values without another LLM
 
 For portable QA scenarios, let the planner provide exact test data instead of invoking the Luna fallback. Match a field by its observed accessible label:

package/dist/cli.js

@@ -3,7 +3,7 @@
 // package.json
 var package_default = {
   name: "jev-cdp",
-  version: "0.1.3",
+  version: "0.1.4",
   description: "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   type: "module",
   license: "MIT",
@@ -317,8 +317,15 @@ class Browser {
   #pageLoadPromise = null;
   #resolvePageLoad;
   #pauseUntil = 0;
+  #frameSessions = new Map;
+  #frameContexts = new Map;
+  #knownTargets = new Set;
+  #openedTabs = [];
+  #cdpUrl;
+  #switchOnPopup = false;
   constructor(cdp, sessionId, targetId, ownsTarget, browserContextId, options) {
     this.#cdp = cdp;
+    this.#cdpUrl = options.cdpUrl;
     this.#sessionId = sessionId;
     this.#targetId = targetId;
     this.#ownsTarget = ownsTarget;
@@ -376,7 +383,20 @@ class Browser {
       flatten: true
     });
     const browser = new Browser(cdp, attached.sessionId, targetId, ownsTarget, browserContextId, options);
+    browser.#knownTargets = new Set((await listChromeTargets(options.cdpUrl)).map((target) => target.id));
     try {
+      browser.#stopPageEvents.push(cdp.on("Target.attachedToTarget", browser.#sessionId, (params) => {
+        const info = params.targetInfo;
+        if (info?.type === "iframe" && info.targetId && typeof params.sessionId === "string")
+          browser.#frameSessions.set(info.targetId, params.sessionId);
+      }));
+      browser.#stopPageEvents.push(cdp.on("Page.frameNavigated", browser.#sessionId, (params) => {
+        const frame = params.frame;
+        if (frame?.id)
+          browser.#frameContexts.delete(frame.id);
+      }));
+      await browser.call("Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: false, flatten: true });
+      await browser.call("Page.enable");
       await browser.call("Emulation.setDeviceMetricsOverride", {
         width: 1120,
         height: 780,
@@ -464,7 +484,9 @@ class Browser {
       return;
     if (!Bun.which("ffmpeg"))
       throw new Error("--recording requires ffmpeg on PATH");
-    this.#recordingDirectory = await mkdtemp(join(tmpdir(), "jev-cdp-recording-"));
+    const firstSegment = !this.#recordingDirectory;
+    if (firstSegment)
+      this.#recordingDirectory = await mkdtemp(join(tmpdir(), "jev-cdp-recording-"));
     await this.call("Page.enable");
     await this.call("Page.addScriptToEvaluateOnNewDocument", { source: RECORDING_CURSOR_INIT });
     const viewport = await this.evaluate("({width: innerWidth, height: innerHeight})");
@@ -472,11 +494,11 @@ class Browser {
       throw new Error("Could not read the recording viewport");
     await this.animateCursor(viewport.width / 2, viewport.height / 2);
     const initial = await this.call("Page.captureScreenshot", { format: "jpeg", quality: 70 });
-    const initialPath = join(this.#recordingDirectory, "000000.jpg");
+    const initialPath = join(this.#recordingDirectory, `${String(this.#recordingSequence++).padStart(6, "0")}.jpg`);
     await Bun.write(initialPath, Buffer.from(initial.data, "base64"));
-    this.#recordingFrames.push({ path: initialPath, elapsedMs: 0 });
-    this.#recordingSequence = 1;
-    this.#recordingStartedAt = performance.now();
+    if (firstSegment)
+      this.#recordingStartedAt = performance.now();
+    this.#recordingFrames.push({ path: initialPath, elapsedMs: firstSegment ? 0 : performance.now() - this.#recordingStartedAt });
     this.#stopRecordingEvents = this.#cdp.on("Page.screencastFrame", this.#sessionId, (params) => {
       const frame = params;
       this.call("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => {
@@ -609,6 +631,64 @@ class Browser {
       throw new StalePageError("Document changed during evaluation");
     return response.result?.value;
   }
+  async evaluateFrame(frameId, expression) {
+    const session = this.#frameSessions.get(frameId);
+    let contextId = this.#frameContexts.get(frameId);
+    if (!session && !contextId) {
+      const context = await this.call("Page.createIsolatedWorld", { frameId });
+      contextId = context.executionContextId;
+      this.#frameContexts.set(frameId, contextId);
+    }
+    const response = await this.#cdp.command("Runtime.evaluate", { expression, returnByValue: true, ...contextId ? { contextId } : {} }, session ?? this.#sessionId);
+    if (response.exceptionDetails)
+      throw new StalePageError("Frame changed during evaluation");
+    return response.result?.value;
+  }
+  async frameOffset(frameId) {
+    const owner = await this.call("DOM.getFrameOwner", { frameId });
+    const box = await this.call("DOM.getBoxModel", { backendNodeId: owner.backendNodeId });
+    return { x: box.model.content[0], y: box.model.content[1] };
+  }
+  async childFrames() {
+    const tree = await this.call("Page.getFrameTree");
+    return (tree.frameTree.childFrames ?? []).map((child) => child.frame.id);
+  }
+  async discoverTabs() {
+    if (!this.#switchOnPopup)
+      return;
+    for (const target of await listChromeTargets(this.#cdpUrl)) {
+      if (target.type !== "page" || this.#knownTargets.has(target.id))
+        continue;
+      this.#knownTargets.add(target.id);
+      const attached = await this.#cdp.command("Target.attachToTarget", { targetId: target.id, flatten: true });
+      try {
+        await this.#cdp.command("Emulation.setDeviceMetricsOverride", {
+          width: 1120,
+          height: 780,
+          deviceScaleFactor: 1,
+          mobile: false
+        }, attached.sessionId);
+      } finally {
+        if (this.#switchOnPopup) {
+          if (this.#recordingPath) {
+            await this.call("Page.stopScreencast").catch(() => {
+              return;
+            });
+            this.#stopRecordingEvents?.();
+            this.#stopRecordingEvents = undefined;
+            await this.#recordingWrites;
+          }
+          this.#sessionId = attached.sessionId;
+          this.#targetId = target.id;
+          this.#switchOnPopup = false;
+          await this.startRecording();
+        } else {
+          await this.#cdp.command("Target.detachFromTarget", { sessionId: attached.sessionId });
+        }
+      }
+      this.#openedTabs.push({ id: target.id, url: target.url, title: target.title });
+    }
+  }
   async settleAfterInput() {
     const action = this.#afterInput;
     this.#afterInput = null;
@@ -644,6 +724,7 @@ class Browser {
   }
   async observe(screenshot = this.#screenshots) {
     await this.settleAfterInput();
+    await this.discoverTabs();
     let info;
     for (let attempt = 0;attempt < 10; attempt++) {
       try {
@@ -658,6 +739,36 @@ class Browser {
     }
     if (!info)
       throw new StalePageError("Document is navigating");
+    if (this.#openedTabs.length) {
+      const targets = await listChromeTargets(this.#cdpUrl);
+      this.#openedTabs = this.#openedTabs.map((tab) => {
+        const current = targets.find((target) => target.id === tab.id);
+        return current ? { ...tab, url: current.url, title: current.title } : tab;
+      });
+      info.text = `${info.text}
+${this.#openedTabs.map((tab) => `Opened new tab: ${tab.title} ${tab.url} (target ${tab.id})`).join(`
+`)}`;
+    }
+    for (const frameId of await this.childFrames()) {
+      try {
+        const offset = await this.frameOffset(frameId);
+        const child = await this.evaluateFrame(frameId, snapshot_default);
+        if (!child)
+          continue;
+        info.text = `${info.text}
+${child.text}`.slice(0, 6000);
+        for (const action of child.actions) {
+          if (!action.node || !action.rect)
+            continue;
+          const rect = { ...action.rect, x: action.rect.x + offset.x, y: action.rect.y + offset.y };
+          if (rect.x < 0 || rect.y < 0 || rect.x >= info.w || rect.y >= info.h)
+            continue;
+          info.actions.push({ ...action, frameId, rect, id: `e${info.actions.length + 1}` });
+          info.guards[`${frameId}:${action.node}`] = child.guards[String(action.node)];
+        }
+        info.marker = [info.marker, frameId, child.marker];
+      } catch {}
+    }
     const page = { ...info, fingerprint: fingerprint(info) };
     if (screenshot) {
       const capture = await this.call("Page.captureScreenshot", { format: "jpeg", quality: 72 });
@@ -666,6 +777,12 @@ class Browser {
     return page;
   }
   async fresh(page, action) {
+    if (action?.frameId && typeof action.node === "number") {
+      const current = await this.evaluateFrame(action.frameId, `(() => {
+        const c=window.__jevFast; return c ? c.guard(c.nodes.get(${action.node})) : null;
+      })()`);
+      return stableStringify(current) === stableStringify(page.guards[`${action.frameId}:${action.node}`]);
+    }
     if (action && (action.kind === "click" || action.kind === "select")) {
       if (typeof action.node !== "number")
         return false;
@@ -698,7 +815,11 @@ class Browser {
     }
     if (typeof action.node !== "number")
       throw new Error("Invalid observed node");
-    const target = await this.evaluate(`(action => {
+    const target = await (action.frameId ? this.evaluateFrame(action.frameId, `(action => {
+      const e=window.__jevFast?.nodes.get(action.node);
+      if (!e?.isConnected || !e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return null;
+      const r=e.getBoundingClientRect(); return {x:r.x+r.width/2,y:r.y+r.height/2};
+    })(${JSON.stringify(action)})`) : this.evaluate(`(action => {
       const e=window.__jevFast?.nodes.get(action.node);
       if (!e?.isConnected || e.matches(':disabled') || e.closest('[aria-disabled="true"],[inert]') ||
           !e.checkVisibility({checkOpacity:true,checkVisibilityCSS:true})) return null;
@@ -714,12 +835,17 @@ class Browser {
         e.dispatchEvent(new Event('change',{bubbles:true}));
       }
       return {x,y};
-    })(${JSON.stringify(action)})`);
+    })(${JSON.stringify(action)})`));
     if (!target) {
       if (action.kind === "select")
         throw new Error("Dropdown execution was not confirmed");
       throw new StalePageError("Target changed or is covered");
     }
+    if (action.frameId) {
+      const offset = await this.frameOffset(action.frameId);
+      target.x += offset.x;
+      target.y += offset.y;
+    }
     await this.animateCursor(target.x, target.y);
     if (action.kind !== "select") {
       if (action.kind === "click" && this.#interactionPauses > 0) {
@@ -760,6 +886,10 @@ class Browser {
       }
     }
     this.#afterInput = action;
+    if (action.frameId && action.kind === "click") {
+      this.#switchOnPopup = true;
+      await Bun.sleep(650);
+    }
   }
   async close() {
     if (this.#closed)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-cdp",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   "type": "module",
   "license": "MIT",