omnius 1.0.207 → 1.0.208

package/dist/index.js

@@ -10326,15 +10326,19 @@ function pngDimensions(buffer2) {
   }
   return null;
 }
-async function describeFocusedEditable(pageHandle) {
-  const active = await pageHandle.evaluate(`(() => {
+async function describeFocusedEditableInContext(context2, frameMeta) {
+  const active = await context2.evaluate(`(() => {
     const el = document.activeElement;
     if (!el) return null;
     const rect = el.getBoundingClientRect();
     const role = (el.getAttribute("role") || "").toLowerCase();
-    const contentEditable = String(el.getAttribute("contenteditable") || "").toLowerCase();
-    const isEditable = el.matches("input, textarea")
-      || contentEditable === "" || contentEditable === "true"
+    const contentEditableAttr = el.getAttribute("contenteditable");
+    const contentEditable = contentEditableAttr !== null
+      && (contentEditableAttr === "" || String(contentEditableAttr).toLowerCase() === "true");
+    const disabled = !!el.disabled;
+    const readOnly = !!el.readOnly;
+    const isEditable = (el.matches("input, textarea") && !disabled && !readOnly)
+      || contentEditable || el.isContentEditable === true
       || ["textbox", "searchbox", "combobox"].includes(role);
     return {
       tag: String(el.tagName || "").toLowerCase(),
@@ -10346,10 +10350,35 @@ async function describeFocusedEditable(pageHandle) {
       placeholder: el.getAttribute("placeholder") || "",
       text: String(el.textContent || "").trim().slice(0, 120),
       isEditable,
+      disabled,
+      readOnly,
       rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
     };
   })()`);
-  return active && typeof active === "object" ? active : null;
+  if (!active || typeof active !== "object")
+    return null;
+  return frameMeta ? { ...active, frame: frameMeta } : active;
+}
+async function describeFocusedEditable(pageHandle) {
+  const main2 = await describeFocusedEditableInContext(pageHandle, { kind: "main", url: pageHandle.url?.() ?? "" }).catch(() => null);
+  if (main2?.["isEditable"])
+    return main2;
+  const frames = typeof pageHandle.frames === "function" ? pageHandle.frames() : [];
+  const mainFrame = typeof pageHandle.mainFrame === "function" ? pageHandle.mainFrame() : null;
+  for (let i2 = 0; i2 < frames.length; i2++) {
+    const frame = frames[i2];
+    if (!frame || frame === mainFrame)
+      continue;
+    const active = await describeFocusedEditableInContext(frame, {
+      kind: "frame",
+      index: i2,
+      url: typeof frame.url === "function" ? frame.url() : "",
+      name: typeof frame.name === "function" ? frame.name() : ""
+    }).catch(() => null);
+    if (active?.["isEditable"])
+      return active;
+  }
+  return main2;
 }
 async function clickAndFillBrowserTarget(pageHandle, target, text, typingDelay) {
   const viewport = pageHandle.viewportSize?.() ?? { width: 1280, height: 720 };
@@ -10464,8 +10493,8 @@ ${input.text.slice(0, 2e4)}`.toLowerCase();
   }
   return { kind: "none", confidence: 0, evidence: [] };
 }
-async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY, forceCandidate, includeOffscreen = false, scrollIntoView = false) {
-  const candidate = await pageHandle.evaluate(`(() => {
+async function findBrowserVisualCandidateInContext(context2, target, visualX, visualY, forceCandidate, includeOffscreen = false, scrollIntoView = false) {
+  const candidate = await context2.evaluate(`(() => {
     const target = ${JSON.stringify(target)};
     const visualX = ${JSON.stringify(visualX)};
     const visualY = ${JSON.stringify(visualY)};
@@ -10496,12 +10525,55 @@ async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY,
       if (window.CSS && typeof window.CSS.escape === "function") return window.CSS.escape(id);
       return String(id).replace(/["\\\\]/g, "\\\\$&");
     };
+    const hasVisibleStyle = (el) => {
+      for (let cur = el; cur && cur.nodeType === 1; cur = cur.parentElement) {
+        const style = getComputedStyle(cur);
+        if (style.display === "none" || style.visibility === "hidden" || style.visibility === "collapse") return false;
+        const opacity = Number(style.opacity);
+        if (Number.isFinite(opacity) && opacity <= 0.02) return false;
+      }
+      return true;
+    };
+    const hasUsableBox = (el) => {
+      const rect = el.getBoundingClientRect();
+      return rect.width > 1 && rect.height > 1;
+    };
+    const inViewport = (rect) => !(rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth);
+    const isRendered = (el) => !!el && hasUsableBox(el) && hasVisibleStyle(el);
     const associatedControl = (el) => {
       if (/^label$/i.test(el.tagName || "") && el.getAttribute("for")) {
         return document.getElementById(el.getAttribute("for"));
       }
+      if (/^label$/i.test(el.tagName || "")) {
+        if (el.control) return el.control;
+        const nested = el.querySelector("input, textarea, select, [contenteditable='true'], [role='textbox']");
+        if (nested) return nested;
+        const labelRect = el.getBoundingClientRect();
+        let scope = el.parentElement;
+        for (let depth = 0; depth < 5 && scope; depth++, scope = scope.parentElement) {
+          const controls = Array.from(scope.querySelectorAll("input, textarea, select, [contenteditable='true'], [role='textbox']"))
+            .filter(control => {
+              const rect = control.getBoundingClientRect();
+              return rect.width > 1 && rect.height > 1 && rect.top >= labelRect.top - 12 && Math.abs(rect.left - labelRect.left) < 260;
+            })
+            .sort((a, b) => {
+              const ar = a.getBoundingClientRect();
+              const br = b.getBoundingClientRect();
+              return (Math.abs(ar.top - labelRect.bottom) - Math.abs(br.top - labelRect.bottom))
+                || (Math.abs(ar.left - labelRect.left) - Math.abs(br.left - labelRect.left));
+            });
+          if (controls[0]) return controls[0];
+        }
+      }
       return el;
     };
+    const clickTargetFor = (el) => {
+      const control = associatedControl(el) || el;
+      if (control === el) return el;
+      if (isRendered(control)) return control;
+      if (/^label$/i.test(el.tagName || "") && isRendered(el)) return el;
+      return control;
+    };
     const associatedLabelText = (el) => {
       const control = associatedControl(el);
       const id = control && control.id ? control.id : el.id;
@@ -10544,20 +10616,25 @@ async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY,
       el.id || "",
       el.className || "",
       el.getAttribute("type") || "",
+      el.getAttribute("autocomplete") || "",
     ].join(" ").toLowerCase();
     const infoFor = (el, score) => {
-      const rect = el.getBoundingClientRect();
-      const visible = !(rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth);
+      const control = associatedControl(el) || el;
+      const clickTarget = clickTargetFor(el);
+      const rect = clickTarget.getBoundingClientRect();
+      const visible = inViewport(rect) && hasVisibleStyle(clickTarget);
       return {
-        tag: String(el.tagName || "").toLowerCase(),
-        id: el.id || "",
-        className: String(el.className || "").slice(0, 160),
-        role: el.getAttribute("role") || "",
-        ariaLabel: el.getAttribute("aria-label") || "",
-        name: el.getAttribute("name") || "",
-        type: el.getAttribute("type") || "",
-        placeholder: el.getAttribute("placeholder") || "",
-        text: String(el.innerText || el.textContent || el.getAttribute("value") || "").trim().slice(0, 240),
+        tag: String(control.tagName || el.tagName || "").toLowerCase(),
+        id: control.id || el.id || "",
+        className: String(control.className || el.className || "").slice(0, 160),
+        role: control.getAttribute("role") || el.getAttribute("role") || "",
+        ariaLabel: control.getAttribute("aria-label") || el.getAttribute("aria-label") || "",
+        name: control.getAttribute("name") || el.getAttribute("name") || "",
+        type: control.getAttribute("type") || el.getAttribute("type") || "",
+        autocomplete: control.getAttribute("autocomplete") || el.getAttribute("autocomplete") || "",
+        placeholder: control.getAttribute("placeholder") || el.getAttribute("placeholder") || "",
+        text: String(el.innerText || el.textContent || control.innerText || control.textContent || control.getAttribute("value") || "").trim().slice(0, 240),
+        clickTag: String(clickTarget.tagName || "").toLowerCase(),
         rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
         center: { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 },
         visible,
@@ -10579,9 +10656,11 @@ async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY,
     let best = null;
     let bestElement = null;
     for (const el of Array.from(document.querySelectorAll(selectors))) {
-      const rect = el.getBoundingClientRect();
-      if (rect.width <= 1 || rect.height <= 1) continue;
-      const visible = !(rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth);
+      if (!isRendered(el)) continue;
+      const clickTarget = clickTargetFor(el);
+      if (!isRendered(clickTarget)) continue;
+      const rect = clickTarget.getBoundingClientRect();
+      const visible = inViewport(rect);
       if (!includeOffscreen && !visible) continue;
       const hay = textFor(el);
       const tokenHits = tokens.filter(t => hay.includes(t)).length;
@@ -10592,7 +10671,7 @@ async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY,
       if (score <= (forceCandidate ? 8 : 0)) continue;
       if (!best || score > best.score) {
         best = infoFor(el, score);
-        bestElement = el;
+        bestElement = clickTarget;
       }
     }
     if (bestElement && scrollIntoView && best && !best.visible) {
@@ -10604,6 +10683,82 @@ async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY,
   })()`);
   return candidate && typeof candidate === "object" ? candidate : null;
 }
+function offsetBrowserCandidate(candidate, offset, viewport, frameMeta) {
+  const rect = candidate["rect"];
+  const center = candidate["center"];
+  const x = Number(rect?.x) + offset.x;
+  const y = Number(rect?.y) + offset.y;
+  const width = Number(rect?.width);
+  const height = Number(rect?.height);
+  const cx = Number(center?.x) + offset.x;
+  const cy = Number(center?.y) + offset.y;
+  const globalRect = {
+    x,
+    y,
+    width,
+    height
+  };
+  const visible = Number.isFinite(x) && Number.isFinite(y) && Number.isFinite(width) && Number.isFinite(height) && !(y + height < 0 || x + width < 0 || y > viewport.height || x > viewport.width);
+  return {
+    ...candidate,
+    rect: globalRect,
+    center: { x: cx, y: cy },
+    visible: candidate["visible"] === true && visible,
+    ...frameMeta ? {
+      frame: frameMeta,
+      frameLocalRect: rect,
+      frameLocalCenter: center
+    } : {}
+  };
+}
+async function findBrowserVisualCandidate(pageHandle, target, visualX, visualY, forceCandidate, includeOffscreen = false, scrollIntoView = false) {
+  const viewport = pageHandle.viewportSize?.() ?? { width: 1280, height: 720 };
+  const candidates = [];
+  const top = await findBrowserVisualCandidateInContext(pageHandle, target, visualX, visualY, forceCandidate, includeOffscreen, scrollIntoView).catch(() => null);
+  if (top)
+    candidates.push(offsetBrowserCandidate(top, { x: 0, y: 0 }, viewport));
+  const frames = typeof pageHandle.frames === "function" ? pageHandle.frames() : [];
+  const mainFrame = typeof pageHandle.mainFrame === "function" ? pageHandle.mainFrame() : null;
+  for (let i2 = 0; i2 < frames.length; i2++) {
+    const frame = frames[i2];
+    if (!frame || frame === mainFrame)
+      continue;
+    const elementHandle = typeof frame.frameElement === "function" ? await frame.frameElement().catch(() => null) : null;
+    if (!elementHandle)
+      continue;
+    let box = await elementHandle.boundingBox().catch(() => null);
+    if (!box || box.width <= 1 || box.height <= 1)
+      continue;
+    const frameVisible = !(box.y + box.height < 0 || box.x + box.width < 0 || box.y > viewport.height || box.x > viewport.width);
+    if (!includeOffscreen && !frameVisible)
+      continue;
+    const localX = Math.max(0, Math.min(box.width, visualX - box.x));
+    const localY = Math.max(0, Math.min(box.height, visualY - box.y));
+    let candidate = await findBrowserVisualCandidateInContext(frame, target, localX, localY, forceCandidate, includeOffscreen, scrollIntoView).catch(() => null);
+    if (!candidate)
+      continue;
+    if (scrollIntoView && (!candidate["visible"] || !frameVisible)) {
+      if (typeof elementHandle.scrollIntoViewIfNeeded === "function") {
+        await elementHandle.scrollIntoViewIfNeeded().catch(() => void 0);
+      }
+      box = await elementHandle.boundingBox().catch(() => box);
+      candidate = {
+        ...candidate,
+        scrolledIntoView: true
+      };
+    }
+    if (!box)
+      continue;
+    candidates.push(offsetBrowserCandidate(candidate, { x: box.x, y: box.y }, viewport, {
+      kind: "frame",
+      index: i2,
+      url: typeof frame.url === "function" ? frame.url() : "",
+      name: typeof frame.name === "function" ? frame.name() : "",
+      rect: { x: box.x, y: box.y, width: box.width, height: box.height }
+    }));
+  }
+  return candidates.filter((candidate) => includeOffscreen || candidate["visible"] === true).sort((a2, b) => Number(b["score"] ?? 0) - Number(a2["score"] ?? 0))[0] ?? null;
+}
 function ok(output, start2) {
   return { success: true, output, durationMs: Date.now() - start2 };
 }
@@ -10682,7 +10837,7 @@ var init_playwright_browser = __esm({
               "clear_diagnostics",
               "close"
             ],
-            description: "Action to perform:\n- navigate: go to a URL\n- click: click element by selector\n- fill: clear input and type text by selector, or by natural-language target when selector is absent\n- type: type text character by character into a selector, or into the currently focused element after visual_click\n- press: press a key (Enter, Tab, Escape, etc.)\n- screenshot: capture the headless browser page, not the desktop; use value to choose the output file path\n- observe_bundle: capture URL/title/viewport, DOM summary, a11y, diagnostics, screenshot, and gate assessment\n- visual_click: browser screenshot -> Moondream point -> elementFromPoint -> human-like Playwright mouse click -> post-action screenshot\n- evaluate: run JavaScript in page context\n- content: get page text content (readable, stripped)\n- dom: get raw page HTML (truncated)\n- dom_summary: compact interactive DOM summary with selectors\n- innerText: get innerText of a specific element\n- select: select dropdown option by value\n- check/uncheck: toggle checkbox\n- hover: hover over element\n- wait: wait for a selector to appear\n- waitForNavigation: wait for page navigation to complete\n- waitForSelector: wait for element matching selector\n- title: get page title\n- url: get current URL\n- getAttribute: get element attribute value\n- innerHTML: get element's innerHTML\n- textContent: get element's textContent\n- goBack/goForward/reload: browser navigation\n- pdf: save page as PDF\n- close: close browser session"
+            description: "Action to perform:\n- navigate: go to a URL\n- click: click element by selector\n- fill: clear input and type text by selector, or by natural-language target when selector is absent\n- type: type text character by character into a selector, or into the currently focused element after visual_click\n- press: press a key (Enter, Tab, Escape, etc.)\n- screenshot: capture the headless browser page, not the desktop; use value to choose the output file path\n- observe_bundle: capture URL/title/viewport, DOM summary, a11y, diagnostics, screenshot, and gate assessment\n- visual_click: browser screenshot -> Moondream point -> elementFromPoint -> human-like Playwright mouse click -> post-action screenshot\n- evaluate: run JavaScript in page context\n- content: get page text content (readable, stripped)\n- dom: get raw page HTML (truncated)\n- dom_summary: compact interactive DOM summary with selectors\n- innerText: get innerText of a specific element\n- select: select dropdown option by value\n- check/uncheck: toggle checkbox\n- hover: hover over element\n- wait: wait for a selector to appear, or sleep for timeout ms when no selector is provided\n- waitForNavigation: wait for page navigation to complete\n- waitForSelector: wait for element matching selector\n- title: get page title\n- url: get current URL\n- getAttribute: get element attribute value\n- innerHTML: get element's innerHTML\n- textContent: get element's textContent\n- goBack/goForward/reload: browser navigation\n- pdf: save page as PDF\n- close: close browser session"
           },
           url: {
             type: "string",
@@ -10843,30 +10998,14 @@ var init_playwright_browser = __esm({
                 await page.type(selector, text, { timeout: timeout2, delay: typingDelay });
                 return ok(`Typed "${text}" into ${selector}`, start2);
               }
-              const active = await page.evaluate(`(() => {
-            const el = document.activeElement;
-            if (!el) return null;
-            const rect = el.getBoundingClientRect();
-            return {
-              tag: String(el.tagName || "").toLowerCase(),
-              id: el.id || "",
-              name: el.getAttribute("name") || "",
-              role: el.getAttribute("role") || "",
-              ariaLabel: el.getAttribute("aria-label") || "",
-              type: el.getAttribute("type") || "",
-              placeholder: el.getAttribute("placeholder") || "",
-              text: String(el.textContent || "").trim().slice(0, 120),
-              isEditable: el.matches("input, textarea, [contenteditable=''], [contenteditable='true']")
-                || ["textbox", "searchbox", "combobox"].includes((el.getAttribute("role") || "").toLowerCase()),
-              rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
-            };
-          })()`);
+              const active = await describeFocusedEditable(page);
               if (!active || typeof active !== "object" || active.isEditable !== true) {
                 return fail("No editable focused element is active; use visual_click on a form field or pass a selector to type.", start2);
               }
               await page.keyboard.type(text, { delay: typingDelay });
               const label = active && typeof active === "object" ? `<${active.tag || "element"}>${active.id ? `#${active.id}` : ""}` : "focused element";
-              return ok(`Typed "${text}" into ${label}`, start2);
+              const frame = active["frame"];
+              return ok(`Typed "${text}" into ${label}${frame?.kind === "frame" ? ` in frame ${frame.index}` : ""}`, start2);
             }
             case "press": {
               const key = text || "Enter";
@@ -10907,7 +11046,14 @@ var init_playwright_browser = __esm({
               return ok(`Hovered: ${resolvedSelector}${resolvedSelector !== selector ? ` (from ${selector})` : ""}`, start2);
             }
             // ── Waiting ──
-            case "wait":
+            case "wait": {
+              if (selector) {
+                await page.waitForSelector(selector, { timeout: timeout2 });
+                return ok(`Element appeared: ${selector}`, start2);
+              }
+              await page.waitForTimeout(timeout2);
+              return ok(`Waited ${timeout2}ms`, start2);
+            }
             case "waitForSelector": {
               if (!selector)
                 return fail("selector is required", start2);
@@ -11360,8 +11506,13 @@ ${JSON.stringify(data, null, 2)}`, start2);
           })()`);
               let clickSource = point.source || pointResult?.source || "vision";
               const candidate = await findBrowserVisualCandidate(page, visualTarget, cssX, cssY, false);
-              if (candidate) {
-                const candidateRecord = candidate;
+              let candidateRecord = candidate;
+              if (!candidateRecord) {
+                candidateRecord = await findBrowserVisualCandidate(page, visualTarget, cssX, cssY, true, true, true);
+                if (candidateRecord)
+                  await page.waitForTimeout(150);
+              }
+              if (candidateRecord) {
                 const center = candidateRecord["center"];
                 const nextX = Number(center?.x);
                 const nextY = Number(center?.y);
@@ -11369,7 +11520,7 @@ ${JSON.stringify(data, null, 2)}`, start2);
                   cssX = Math.max(0, Math.min(viewport.width, nextX));
                   cssY = Math.max(0, Math.min(viewport.height, nextY));
                   elementInfo = candidateRecord;
-                  clickSource = `${clickSource}+dom-candidate`;
+                  clickSource = `${clickSource}+dom-candidate${candidateRecord["scrolledIntoView"] === true ? "+scroll" : ""}`;
                 }
               }
               await page.mouse.move(cssX, cssY, { steps: 12 });

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.207",
+  "version": "1.0.208",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.207",
+      "version": "1.0.208",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.207",
+  "version": "1.0.208",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",