jev-cdp 0.1.9 → 0.1.10

package/README.md

@@ -27,8 +27,8 @@ This is an early experimental port. See [NOTICE.md](NOTICE.md) for source attrib
 Run the published CLI without adding it to a project. Bun must be installed for either command:
 
 ```bash
-bunx jev-cdp@0.1.8 help run
-npx -y jev-cdp@0.1.8 help run
+bunx jev-cdp@0.1.10 help run
+npx -y jev-cdp@0.1.10 help run
 ```
 
 Chrome with a CDP endpoint and `TYPESAFE_API_KEY` are required for browser runs. FFmpeg with H.264 encoding (`libx264`) is required for `--recording`. Jev selects `-fps_mode vfr` when FFmpeg supports it and falls back to `-vsync vfr` for older builds. Run `jev-cdp doctor` to check the installed encoder with a short MP4 encode and decode.

package/dist/cli.js

@@ -3,7 +3,7 @@
 // package.json
 var package_default = {
   name: "jev-cdp",
-  version: "0.1.9",
+  version: "0.1.10",
   description: "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   type: "module",
   license: "MIT",
@@ -348,16 +348,70 @@ var snapshot_default = `// Ported from browser-use/jev-ultrafast at commit 452c1
     }
     return null;
   };
+  const scrollContainers = () =>
+    [...document.querySelectorAll('*')].filter(e =>
+      e !== document.scrollingElement &&
+      visible(e) &&
+      /^(auto|scroll)$/.test(getComputedStyle(e).overflowY) &&
+      e.clientHeight > 0 &&
+      e.scrollHeight > e.clientHeight + 2
+    );
+  cache.scrollPoint = (e, delta) => {
+    if (!e?.isConnected || !visible(e)) return null;
+    if (delta > 0
+      ? e.scrollTop + e.clientHeight >= e.scrollHeight - 2
+      : e.scrollTop <= 0) return null;
+    const r = e.getBoundingClientRect();
+    let left = Math.max(0, r.left);
+    let right = Math.min(innerWidth, r.right);
+    let top = Math.max(0, r.top);
+    let bottom = Math.min(innerHeight, r.bottom);
+    for (let p = e.parentElement; p; p = p.parentElement) {
+      if (p === document.body || p === document.documentElement) continue;
+      const style = getComputedStyle(p);
+      const bounds = p.getBoundingClientRect();
+      if (/^(auto|scroll|hidden|clip)$/.test(style.overflowX)) {
+        left = Math.max(left, bounds.left);
+        right = Math.min(right, bounds.right);
+      }
+      if (/^(auto|scroll|hidden|clip)$/.test(style.overflowY)) {
+        top = Math.max(top, bounds.top);
+        bottom = Math.min(bottom, bounds.bottom);
+      }
+    }
+    if (right - left < 4 || bottom - top < 4) return null;
+    for (const fraction of [0.5, 0.9, 0.1]) {
+      const point = {
+        x: left + (right - left) * fraction,
+        y: top + (bottom - top) / 2,
+      };
+      let hit = document.elementFromPoint(point.x, point.y);
+      if (!hit || !e.contains(hit)) continue;
+      while (hit && hit !== e) {
+        if (
+          /^(auto|scroll)$/.test(getComputedStyle(hit).overflowY) &&
+          hit.scrollHeight > hit.clientHeight + 2
+        ) break;
+        hit = hit.parentElement;
+      }
+      if (hit === e) return point;
+    }
+    return null;
+  };
   cache.pageKey=()=>[performance.timeOrigin,location.href,scrollX,scrollY,innerWidth,innerHeight,
     [...document.querySelectorAll('input,textarea,select')].filter(safe)
-      .map(e=>[identity(e),stateValue(e),e.checked,e.selectedIndex,e.disabled,e.readOnly])];
+      .map(e=>[identity(e),stateValue(e),e.checked,e.selectedIndex,e.disabled,e.readOnly]),
+    scrollContainers().map(e=>[
+      identity(e),e.scrollTop,e.scrollLeft,e.clientHeight,e.scrollHeight
+    ])];
   cache.guard=e=>{
     if (!e?.isConnected || !visible(e)) return null;
     const scope=e.closest('form,dialog,[role="dialog"],article,li,tr,[role="row"]') || e.parentElement;
     return [identity(e),role(e),name(e),sensitive(e)?Boolean(e.value):e.value??null,e.checked??null,e.selectedIndex??null,
       e.readOnly??null,e.matches(':disabled'),e.getAttribute('aria-disabled'),
       e.getAttribute('aria-expanded'),e.getAttribute('aria-checked'),e.getAttribute('aria-selected'),
-      e.getAttribute('href'),scope?.innerText?.slice(0,6000)||''];
+      e.getAttribute('href'),scope?.innerText?.slice(0,6000)||'',
+      e.scrollTop,e.scrollLeft,e.clientHeight,e.scrollHeight];
   };
   const actions=[];
   for (const e of document.querySelectorAll(selector)) {
@@ -409,12 +463,35 @@ var snapshot_default = `// Ported from browser-use/jev-ultrafast at commit 452c1
       }
     }
   }
+  for (const e of scrollContainers()) {
+    const label =
+      e.getAttribute('aria-label') ||
+      e.closest('form,dialog,[role="dialog"]')
+        ?.querySelector('h1,h2,h3')?.textContent ||
+      'panel';
+    for (const delta of [560, -560]) {
+      const point = cache.scrollPoint(e, delta);
+      if (!point) continue;
+      actions.push({
+        kind: 'scroll',
+        node: identity(e),
+        label: 'Scroll ' + (delta > 0 ? 'down' : 'up') +
+          ' in ' + label.trim().slice(0, 80),
+        delta,
+        point,
+        rect: { x: point.x - 1, y: point.y - 1, w: 2, h: 2 },
+        clickable: true,
+        coveredBy: null,
+        scrollTop: e.scrollTop,
+      });
+    }
+  }
   const text=words.join('\\n').slice(0,6000), height=document.documentElement.scrollHeight;
   const page_key=cache.pageKey(), guards={};
   for (const a of actions) if (!(a.node in guards)) guards[a.node]=cache.guard(cache.nodes.get(a.node));
   const semantics=actions.map(({rect,...action})=>action);
   const marker=[performance.timeOrigin,location.href,scrollX,scrollY,innerWidth,innerHeight,
-    document.title,text,semantics,page_key[6]];
+    document.title,text,semantics,page_key[6],page_key[7]];
   const omitted_actions=Math.max(0,actions.length-250);
   actions.splice(250);
   actions.forEach((a,i)=>a.id='e'+(i+1));
@@ -723,8 +800,8 @@ class Browser {
       this.signalChange();
       if (params.frameId !== this.#mainFrameId || this.#pageLoadPromise)
         return;
-      this.#pageLoadPromise = new Promise((resolve2) => {
-        this.#resolvePageLoad = resolve2;
+      this.#pageLoadPromise = new Promise((resolve) => {
+        this.#resolvePageLoad = resolve;
       });
     };
     const loaded = () => {
@@ -769,11 +846,11 @@ class Browser {
       listener();
   }
   async waitForChange(ms) {
-    await new Promise((resolve2) => {
+    await new Promise((resolve) => {
       const done = () => {
         clearTimeout(timer);
         this.#changeListeners.delete(done);
-        resolve2();
+        resolve();
       };
       const timer = setTimeout(done, Math.min(ms, 150));
       this.#changeListeners.add(done);
@@ -782,11 +859,11 @@ class Browser {
   async waitForNewTarget(ms) {
     if (this.#newTargets.size)
       return;
-    await new Promise((resolve2) => {
+    await new Promise((resolve) => {
       const done = () => {
         clearTimeout(timer);
         this.#popupListeners.delete(done);
-        resolve2();
+        resolve();
       };
       const timer = setTimeout(done, ms);
       this.#popupListeners.add(done);
@@ -1255,9 +1332,9 @@ class Browser {
     if (waited)
       this.#waitSpentMs += performance.now() - waitStarted;
     if (this.#openedTabs.length) {
-      const targets2 = await listChromeTargets(this.#cdpUrl);
+      const targets = await listChromeTargets(this.#cdpUrl);
       this.#openedTabs = this.#openedTabs.map((tab) => {
-        const current = targets2.find((target) => target.id === tab.id);
+        const current = targets.find((target) => target.id === tab.id);
         return current ? { ...tab, url: current.url, title: current.title } : tab;
       });
       info.text = `${info.text}
@@ -1265,7 +1342,7 @@ ${this.#openedTabs.map((tab) => `Opened new tab: ${tab.title} ${tab.url} (target
 `)}`;
     }
     const frames = await this.frameStates();
-    for (const frame of frames.filter((frame2) => frame2.parentId !== null)) {
+    for (const frame of frames.filter((frame) => frame.parentId !== null)) {
       const frameId = frame.id;
       try {
         const offset = await this.frameScreenOffset(frameId, frames);
@@ -1417,15 +1494,36 @@ ${child.text}`.slice(0, 6000);
       return { element: null, performedAt: performance.now() };
     }
     if (action.kind === "scroll") {
-      await this.animateCursor(550, 650);
+      let point = { x: 550, y: 650 };
+      if (typeof action.node === "number") {
+        const expression = `window.__jevFast?.scrollPoint(
+          window.__jevFast.nodes.get(${action.node}),
+          ${action.delta ?? 0}
+        )`;
+        const target = action.frameId ? await this.evaluateFrame(action.frameId, expression) : await this.evaluate(expression);
+        if (!target)
+          throw new StalePageError("Scroll target changed or is covered");
+        point = target;
+        if (action.frameId) {
+          const offset = await this.frameScreenOffset(action.frameId);
+          point.x += offset.x;
+          point.y += offset.y;
+          const covered = await this.evaluate(`(() => {
+            const e = document.elementFromPoint(${point.x}, ${point.y});
+            return !e || e.tagName !== "IFRAME";
+          })()`);
+          if (covered)
+            throw new StalePageError("Scroll target is covered");
+        }
+      }
+      await this.animateCursor(point.x, point.y);
       await this.call("Input.dispatchMouseEvent", {
         type: "mouseWheel",
-        x: 550,
-        y: 650,
+        ...point,
         deltaX: 0,
         deltaY: action.delta ?? 0
       });
-      return { element: null, performedAt: performance.now() };
+      return { element: null, performedAt: performance.now(), scrollPoint: point };
     }
     if (typeof action.node !== "number")
       throw new Error("Invalid observed node");
@@ -1667,10 +1765,10 @@ function actionSpace(actions) {
       continue;
     const identity = `${action.frameId ?? "main"}:${action.node}`;
     if (!indices.has(identity)) {
-      const index2 = String(elements.length + 1);
-      indices.set(identity, index2);
-      const element2 = {
-        index: index2,
+      const index = String(elements.length + 1);
+      indices.set(identity, index);
+      const element = {
+        index,
         label: action.label.split(" \u2192 ")[0] ?? action.label,
         operations: [],
         frameId: action.frameId ?? null,
@@ -1683,13 +1781,13 @@ function actionSpace(actions) {
       };
       for (const key of ["role", "value", "checked", "selected", "expanded", "pressed", "sensitive"]) {
         if (action[key] !== undefined)
-          element2[key] = action[key];
+          element[key] = action[key];
       }
       if (action.kind === "select") {
-        element2.value = action.current_value ?? "";
-        element2.options = [];
+        element.value = action.current_value ?? "";
+        element.options = [];
       }
-      elements.push(element2);
+      elements.push(element);
     }
     const index = indices.get(identity);
     if (action.clickable === false)
@@ -1718,17 +1816,17 @@ async function choose(page, goal, history, providedFields = []) {
     SELECT: "Select an observed dropdown value."
   };
   const operations = {};
-  for (const operation2 of Object.keys(targets))
-    operations[operation2] = labels[operation2];
-  for (const [operation2, action] of Object.entries(controls))
-    operations[operation2] = action.label;
+  for (const operation of Object.keys(targets))
+    operations[operation] = labels[operation];
+  for (const [operation, action] of Object.entries(controls))
+    operations[operation] = action.label;
   operations.DONE = "Every requirement is visibly satisfied.";
   operations.BLOCKED = "No supported operation can progress.";
   const questions = {
     operation: { type: "choice", criteria: operations, instructions: { goal, rules: NEXT_ACTION } }
   };
-  for (const [operation2, candidates] of Object.entries(targets)) {
-    questions[`${operation2.toLowerCase()}_target`] = {
+  for (const [operation, candidates] of Object.entries(targets)) {
+    questions[`${operation.toLowerCase()}_target`] = {
       type: "choice",
       criteria: Object.fromEntries(Object.entries(candidates).map(([index, action]) => [index, {
         element: `[${index}] ${action.label}`,
@@ -1742,7 +1840,7 @@ async function choose(page, goal, history, providedFields = []) {
         current_value: action.current_value ?? action.value ?? "",
         ...Object.fromEntries(["role", "checked", "selected", "expanded", "pressed", "sensitive"].filter((name) => action[name] !== undefined).map((name) => [name, action[name]]))
       }])),
-      instructions: { goal, operation: operation2, rules: [NEXT_ACTION, TARGET] }
+      instructions: { goal, operation, rules: [NEXT_ACTION, TARGET] }
     };
   }
   const body = {
@@ -2082,7 +2180,7 @@ class Agent {
     }
     await this.#browser.waitForInteractionPause();
     const fromTargetId = this.#browser.targetId;
-    const { element, performedAt } = await this.#browser.act(action, page, text ?? undefined);
+    const { element, performedAt, scrollPoint } = await this.#browser.act(action, page, text ?? undefined);
     this.#pendingText = null;
     const entry = {
       step: this.#history.length + 1,
@@ -2106,6 +2204,7 @@ class Agent {
       element,
       value: action.kind === "select" ? action.value ?? null : null,
       delta_y: action.kind === "scroll" ? action.delta ?? 0 : null,
+      scroll_point: scrollPoint ?? null,
       redacted: action.kind === "fill" && (Boolean(action.sensitive) || this.#sensitiveFieldLabels.has(action.label)),
       usage: decision.usage,
       executed_ms: Math.round(performedAt - this.#startedAt),
@@ -2450,7 +2549,7 @@ function actionEvent(entry, maxSteps) {
       element: entry.element,
       ...entry.kind === "fill" ? { text: entry.text, redacted: entry.redacted } : {},
       ...entry.kind === "select" ? { optionValue: entry.value } : {},
-      ...entry.kind === "scroll" ? { deltaY: entry.delta_y, point: { x: 550, y: 650 } } : {},
+      ...entry.kind === "scroll" ? { deltaY: entry.delta_y, point: entry.scroll_point ?? { x: 550, y: 650 } } : {},
       ...entry.kind === "wait" ? { durationMs: 100 } : {}
     }
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-cdp",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "description": "A small Jev-powered bridge to Chrome through the Chrome DevTools Protocol.",
   "type": "module",
   "license": "MIT",