omnius 1.0.205 → 1.0.206

package/dist/index.js

@@ -3987,9 +3987,33 @@ var init_system_deps = __esm({
 // packages/execution/dist/tools/desktop-control.js
 import { execSync as execSync4 } from "node:child_process";
 import { existsSync as existsSync6, statSync as statSync3 } from "node:fs";
+function activateDesktopWindow(windowId) {
+  const attempts = [];
+  const tryAction = (label, command) => {
+    const result = run(command, 5e3);
+    if (result.ok)
+      return label;
+    attempts.push({ label, message: result.message });
+    return null;
+  };
+  if (process.platform === "linux") {
+    if (hasCommand2("xdotool")) {
+      const backend = tryAction("xdotool", `xdotool windowactivate --sync ${quoteShell(windowId)}`);
+      if (backend)
+        return backend;
+    }
+    if (hasCommand2("wmctrl")) {
+      const backend = tryAction("wmctrl", `wmctrl -ia ${quoteShell(windowId)}`);
+      if (backend)
+        return backend;
+    }
+  }
+  throw new Error("No desktop window activation backend succeeded.\n" + formatDesktopAttempts(attempts) + "\n" + desktopAutomationRecoveryMessage());
+}
 function captureDesktopScreenshot(outputPath3) {
   const attempts = [];
   const out = quoteShell(outputPath3);
+  const allowInteractiveWaylandScreenshot = envFlag("OMNIUS_DESKTOP_INTERACTIVE_SCREENSHOT", false);
   const tryCapture = (label, command, timeout2 = 1e4) => {
     const result = run(command, timeout2);
     if (result.ok && existsSync6(outputPath3)) {
@@ -4022,11 +4046,22 @@ $bitmap.Dispose()
       return backend;
   } else if (process.platform === "linux") {
     const desktop = `${process.env["XDG_CURRENT_DESKTOP"] || ""} ${process.env["DESKTOP_SESSION"] || ""}`;
-    if ((/wayland/i.test(process.env["XDG_SESSION_TYPE"] || "") || process.env["WAYLAND_DISPLAY"]) && hasCommand2("gdbus") && hasCommand2("dbus-monitor")) {
+    if ((/wayland/i.test(process.env["XDG_SESSION_TYPE"] || "") || process.env["WAYLAND_DISPLAY"]) && /gnome/i.test(desktop) && hasCommand2("gdbus") && allowInteractiveWaylandScreenshot) {
+      const backend = tryCapture("gnome-shell-interactive-screenshot-dbus", gnomeInteractiveScreenshotCommand(outputPath3), 13e4);
+      if (backend)
+        return backend;
+    }
+    if ((/wayland/i.test(process.env["XDG_SESSION_TYPE"] || "") || process.env["WAYLAND_DISPLAY"]) && hasCommand2("gdbus") && hasCommand2("dbus-monitor") && allowInteractiveWaylandScreenshot) {
       const backend = tryCapture("xdg-desktop-portal-screenshot", portalScreenshotCommand(outputPath3), 13e4);
       if (backend)
         return backend;
     }
+    if ((/wayland/i.test(process.env["XDG_SESSION_TYPE"] || "") || process.env["WAYLAND_DISPLAY"]) && !allowInteractiveWaylandScreenshot) {
+      attempts.push({
+        label: "interactive-wayland-screenshot",
+        message: "skipped by default to avoid unattended GNOME/portal screenshot selection stalls. For desktop app loops, pass window_title to vision_action_loop so Omnius captures the named X11/XWayland window. Set OMNIUS_DESKTOP_INTERACTIVE_SCREENSHOT=1 only when a human will complete the full-screen screenshot prompt."
+      });
+    }
     if (/gnome/i.test(desktop) && hasCommand2("gdbus")) {
       const backend = tryCapture("gnome-shell-screenshot-dbus", `gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot --method org.gnome.Shell.Screenshot.Screenshot false false ${out}`);
       if (backend)
@@ -4079,6 +4114,83 @@ $bitmap.Dispose()
   }
   throw new Error("No desktop screenshot backend succeeded.\n" + formatDesktopAttempts(attempts) + "\n" + desktopAutomationRecoveryMessage());
 }
+function captureDesktopWindowScreenshot(outputPath3, windowTitle) {
+  const attempts = [];
+  const out = quoteShell(outputPath3);
+  const runText = (command, timeout2 = 5e3) => {
+    const result = runCaptureText(command, timeout2);
+    if (result.ok)
+      return result.output.trim();
+    attempts.push({ label: command.split(/\s+/)[0] || command, message: result.message });
+    return null;
+  };
+  if (process.platform !== "linux") {
+    throw new Error("Window screenshot capture is currently implemented for Linux/X11/XWayland windows only.");
+  }
+  if (!hasCommand2("xdotool")) {
+    attempts.push({ label: "xdotool", message: "not found on PATH" });
+  } else if (!hasCommand2("import")) {
+    attempts.push({ label: "import", message: "ImageMagick import not found on PATH" });
+  } else {
+    const windowId = windowTitle ? runText(`xdotool search --name ${quoteShell(windowTitle)} | tail -n1`) : runText("xdotool getactivewindow");
+    if (windowId) {
+      const geometry = runText(`xdotool getwindowgeometry --shell ${quoteShell(windowId)}`);
+      const parsed = parseXdotoolGeometry(geometry || "");
+      if (!parsed) {
+        attempts.push({ label: "xdotool getwindowgeometry", message: `Could not parse geometry for window ${windowId}: ${geometry}` });
+      } else {
+        const result = run(`import -window ${quoteShell(windowId)} ${out}`, 1e4);
+        if (result.ok && existsSync6(outputPath3)) {
+          const inspection = inspectScreenshot(outputPath3);
+          if (inspection.ok) {
+            return {
+              backend: "imagemagick-import-window",
+              windowId,
+              ...parsed
+            };
+          }
+          attempts.push({ label: "import-window", message: inspection.message });
+        } else {
+          attempts.push({ label: "import-window", message: result.message });
+        }
+      }
+    }
+  }
+  throw new Error("No desktop window screenshot backend succeeded.\n" + formatDesktopAttempts(attempts) + "\n" + desktopAutomationRecoveryMessage());
+}
+function gnomeInteractiveScreenshotCommand(outputPath3) {
+  const script = `
+set -eu
+out=${quoteShell(outputPath3)}
+printf '%s\\n' 'Omnius desktop screenshot: choose Full Screen in the GNOME screenshot UI, then press Enter.' >&2
+call_out="$(gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot --method org.gnome.Shell.Screenshot.InteractiveScreenshot 2>&1)" || {
+  printf '%s\\n' "$call_out" >&2
+  exit 1
+}
+uri="$(printf '%s\\n' "$call_out" | sed -n "s/.*'\\(file:[^']*\\)'.*/\\1/p" | tail -n 1)"
+if [ -z "$uri" ]; then
+  printf 'GNOME interactive screenshot did not return a file URI: %s\\n' "$call_out" >&2
+  exit 1
+fi
+
+if command -v gio >/dev/null 2>&1; then
+  gio copy -f "$uri" "$out"
+elif command -v python3 >/dev/null 2>&1; then
+  python3 - "$uri" "$out" <<'PY'
+import shutil, sys, urllib.parse
+uri, out = sys.argv[1], sys.argv[2]
+parsed = urllib.parse.urlparse(uri)
+if parsed.scheme != "file":
+    raise SystemExit(f"Unsupported screenshot URI scheme: {parsed.scheme}")
+shutil.copyfile(urllib.parse.unquote(parsed.path), out)
+PY
+else
+  printf 'Need gio or python3 to copy GNOME screenshot URI %s to %s.\\n' "$uri" "$out" >&2
+  exit 1
+fi
+`.trim();
+  return `bash -lc ${quoteShellLiteral(script)}`;
+}
 function portalScreenshotCommand(outputPath3) {
   const script = `
 set -eu
@@ -4086,6 +4198,7 @@ out=${quoteShell(outputPath3)}
 monitor_file="$(mktemp)"
 cleanup() {
   if [ -n "\${monpid:-}" ]; then kill "$monpid" >/dev/null 2>&1 || true; fi
+  if [ -n "\${keypid:-}" ]; then kill "$keypid" >/dev/null 2>&1 || true; fi
   rm -f "$monitor_file"
 }
 trap cleanup EXIT
@@ -4102,6 +4215,25 @@ fi
 
 dbus-monitor --session "type='signal',interface='org.freedesktop.portal.Request',member='Response',path='$handle'" > "$monitor_file" 2>&1 &
 monpid=$!
+if [ "\${OMNIUS_SCREENSHOT_AUTO_CONFIRM:-1}" != "0" ]; then
+  (
+    sleep 1
+    i=0
+    while [ "$i" -lt 8 ]; do
+      if command -v xdotool >/dev/null 2>&1; then
+        DISPLAY="\${DISPLAY:-:0}" xdotool key Return >/dev/null 2>&1 || true
+      elif command -v ydotool >/dev/null 2>&1; then
+        ydotool key 28:1 28:0 >/dev/null 2>&1 || true
+      elif command -v dotool >/dev/null 2>&1; then
+        printf 'key enter
+' | dotool >/dev/null 2>&1 || true
+      fi
+      i=$((i + 1))
+      sleep 0.5
+    done
+  ) &
+  keypid=$!
+fi
 deadline=$(( $(date +%s) + 120 ))
 while :; do
   if grep -q "member=Response" "$monitor_file"; then break; fi
@@ -4144,7 +4276,7 @@ else
   exit 1
 fi
 `.trim();
-  return `bash -lc ${quoteShell(script)}`;
+  return `bash -lc ${quoteShellLiteral(script)}`;
 }
 function moveDesktopPointer(x, y) {
   const result = performDesktopPointerAction({ x, y, moveOnly: true });
@@ -4167,6 +4299,25 @@ function clickDesktopAt(x, y, button, clickType) {
   lastPointer = { x: Math.round(x), y: Math.round(y) };
   return result.backend;
 }
+function typeDesktopText(text, delayMs = 10) {
+  const result = performDesktopKeyboardAction({
+    kind: "type",
+    text,
+    delayMs
+  });
+  if (!result.ok)
+    throw new Error(result.message);
+  return result.backend;
+}
+function pressDesktopKey(key) {
+  const result = performDesktopKeyboardAction({
+    kind: "key",
+    key
+  });
+  if (!result.ok)
+    throw new Error(result.message);
+  return result.backend;
+}
 function tryRunXdotoolShellFallback(command) {
   if (!/\bxdotool\b|\bxdtool\b/.test(command))
     return null;
@@ -4278,7 +4429,9 @@ function desktopAutomationRecoveryMessage(command) {
     "  Linux pointer control: xdotool/X11, ydotool, dotool, python-xlib",
     "  macOS: cliclick, then System Events",
     "  Windows: PowerShell user32 input",
-    "On Wayland, Omnius requests screenshot permission through xdg-desktop-portal when available. Approve the system screenshot prompt to continue.",
+    "On Wayland, unattended full-desktop screenshots are compositor-restricted and interactive screenshot prompts are skipped by default.",
+    "For desktop app loops on GNOME Wayland, pass window_title to vision_action_loop so Omnius captures the named X11/XWayland window without a full-screen prompt.",
+    "Set OMNIUS_DESKTOP_INTERACTIVE_SCREENSHOT=1 only when a human will complete the full-screen screenshot prompt.",
     "On GNOME Wayland, unattended screenshots may still be denied by compositor policy. Install gnome-screenshot or grant screenshot permission for the session if capture is blocked.",
     "On Wayland pointer control, install and enable ydotool or dotool when xdotool cannot open an X display."
   ].join("\n");
@@ -4382,6 +4535,72 @@ ${options2.moveOnly ? "" : `for _ in range(${clicks}):
     message: "No desktop mouse backend succeeded.\n" + formatDesktopAttempts(attempts) + "\n" + desktopAutomationRecoveryMessage()
   };
 }
+function performDesktopKeyboardAction(options2) {
+  const attempts = [];
+  const tryAction = (label, command) => {
+    const result = run(command, 1e4);
+    if (result.ok)
+      return label;
+    attempts.push({ label, message: result.message });
+    return null;
+  };
+  if (process.platform === "linux") {
+    if (hasCommand2("xdotool")) {
+      const command = options2.kind === "type" ? `xdotool type --clearmodifiers --delay ${Math.max(0, Math.min(500, Math.round(options2.delayMs)))} ${quoteShell(options2.text)}` : `xdotool key --clearmodifiers ${quoteShell(options2.key)}`;
+      const backend = tryAction("xdotool", command);
+      if (backend)
+        return { ok: true, backend };
+    } else {
+      attempts.push({ label: "xdotool", message: "not found on PATH" });
+    }
+    if (hasCommand2("ydotool")) {
+      const command = options2.kind === "type" ? `ydotool type ${quoteShell(options2.text)}` : `ydotool key ${quoteShell(options2.key)}`;
+      const backend = tryAction("ydotool", command);
+      if (backend)
+        return { ok: true, backend };
+    }
+    if (hasCommand2("dotool")) {
+      const commandText = options2.kind === "type" ? `type ${options2.text}
+` : `key ${options2.key}
+`;
+      const backend = tryAction("dotool", `printf ${quoteShell(commandText)} | dotool`);
+      if (backend)
+        return { ok: true, backend };
+    }
+  } else if (process.platform === "darwin") {
+    if (options2.kind === "type") {
+      const script = `tell application "System Events" to keystroke ${quoteAppleScript(options2.text)}`;
+      const backend = tryAction("osascript-system-events", `osascript -e ${quoteShell(script)}`);
+      if (backend)
+        return { ok: true, backend };
+    } else {
+      const script = `tell application "System Events" to key code ${quoteAppleScript(options2.key)}`;
+      const backend = tryAction("osascript-system-events", `osascript -e ${quoteShell(script)}`);
+      if (backend)
+        return { ok: true, backend };
+    }
+  }
+  return {
+    ok: false,
+    message: "No desktop keyboard backend succeeded.\n" + formatDesktopAttempts(attempts) + "\n" + desktopAutomationRecoveryMessage()
+  };
+}
+function parseXdotoolGeometry(raw) {
+  const values = /* @__PURE__ */ new Map();
+  for (const line of raw.split(/\r?\n/)) {
+    const match = line.match(/^([A-Z]+)=(-?\d+)$/);
+    if (match)
+      values.set(match[1], Number(match[2]));
+  }
+  const x = values.get("X");
+  const y = values.get("Y");
+  const width = values.get("WIDTH");
+  const height = values.get("HEIGHT");
+  if ([x, y, width, height].every((value2) => Number.isFinite(value2))) {
+    return { x, y, width, height };
+  }
+  return null;
+}
 function windowsMouseScript(x, y, down, up, clicks = 1) {
   const clickBody = down == null || up == null ? "" : `
 for ($i = 0; $i -lt ${clicks}; $i++) {
@@ -4505,6 +4724,26 @@ function run(command, timeout2) {
     };
   }
 }
+function runCaptureText(command, timeout2) {
+  try {
+    const output = execSync4(command, {
+      stdio: ["pipe", "pipe", "pipe"],
+      timeout: timeout2,
+      env: { ...process.env },
+      encoding: "utf8"
+    });
+    return { ok: true, output, message: "" };
+  } catch (err) {
+    const anyErr = err;
+    const stdout = bufferishToString(anyErr.stdout);
+    const stderr = bufferishToString(anyErr.stderr);
+    return {
+      ok: false,
+      output: stdout,
+      message: (stderr || stdout || anyErr.message || String(err)).trim().slice(0, 800)
+    };
+  }
+}
 function bufferishToString(value2) {
   if (Buffer.isBuffer(value2))
     return value2.toString("utf8");
@@ -4517,6 +4756,18 @@ function powershellCommand(script) {
 function quoteShell(value2) {
   return JSON.stringify(value2);
 }
+function quoteShellLiteral(value2) {
+  return `'${value2.replace(/'/g, `'"'"'`)}'`;
+}
+function envFlag(name10, fallback) {
+  const value2 = process.env[name10];
+  if (value2 === void 0)
+    return fallback;
+  return /^(1|true|yes|on)$/i.test(value2.trim());
+}
+function quoteAppleScript(value2) {
+  return JSON.stringify(value2);
+}
 function psString(value2) {
   return value2.replace(/'/g, "''");
 }
@@ -8507,7 +8758,7 @@ function resetMoondreamClient() {
 function getVisionPointDiagnostics() {
   return [...lastPointDiagnostics];
 }
-function envFlag(value2, fallback = false) {
+function envFlag2(value2, fallback = false) {
   if (value2 === void 0)
     return fallback;
   if (/^(1|true|yes|on)$/i.test(value2.trim()))
@@ -8627,8 +8878,8 @@ function resolveHuggingFaceVisionModelCandidates(preferredModel) {
   for (const model of splitModelList(process.env["OMNIUS_MOONDREAM_HF_MODELS"]))
     add2(model, true);
   add2(process.env["MOONDREAM_HF_MODEL"] || "", Boolean(process.env["MOONDREAM_HF_MODEL"]));
-  const fullPreviewAuto = envFlag(process.env["OMNIUS_MOONDREAM3_PREVIEW_AUTO"], true) || envFlag(process.env["OMNIUS_MOONDREAM3_PREVIEW"], false);
-  const compactFallbackAuto = envFlag(process.env["OMNIUS_MOONDREAM2_4BIT_AUTO"], true);
+  const fullPreviewAuto = envFlag2(process.env["OMNIUS_MOONDREAM3_PREVIEW_AUTO"], true) || envFlag2(process.env["OMNIUS_MOONDREAM3_PREVIEW"], false);
+  const compactFallbackAuto = envFlag2(process.env["OMNIUS_MOONDREAM2_4BIT_AUTO"], true);
   const hasExplicitHf = candidates.some((candidate) => candidate.explicit);
   const hasExplicitNonHf = Boolean(preferred) && !isHuggingFaceVisionModel(preferred);
   if (hasExplicitNonHf)
@@ -8937,7 +9188,7 @@ async function callOllamaVision(ollamaHost, model, prompt, imageBase64, timeoutM
   return typeof data.response === "string" && data.response.trim() ? data.response : null;
 }
 function shouldAutoPullOllamaVisionModel(model) {
-  if (!envFlag(process.env["OMNIUS_OLLAMA_VISION_AUTO_PULL"], true))
+  if (!envFlag2(process.env["OMNIUS_OLLAMA_VISION_AUTO_PULL"], true))
     return false;
   return Boolean(model.trim());
 }
@@ -9031,7 +9282,7 @@ async function prepareHuggingFaceVisionCandidate(candidate, diagnostics) {
   return { python, env: env2, gpuIndex: decision2.lease.gpuIndex, release: () => decision2.lease.release() };
 }
 function ensureHuggingFaceVisionPython(modelId) {
-  const managed = envFlag(process.env["OMNIUS_MOONDREAM_HF_MANAGED"], true);
+  const managed = envFlag2(process.env["OMNIUS_MOONDREAM_HF_MANAGED"], true);
   if (!managed)
     return legacyHuggingFaceVisionPython(modelId);
   ensureUnifiedCacheDirs();
@@ -282954,6 +283205,18 @@ var init_vision_action_loop = __esm({
             enum: ["single", "double"],
             description: "Click type for click operation. Default single."
           },
+          text: {
+            type: "string",
+            description: "Optional text to type after a live click, or into the currently focused desktop control when operation='none'."
+          },
+          key: {
+            type: "string",
+            description: "Optional key/chord to press after a live click/text entry, for example Enter, Escape, Tab, ctrl+f."
+          },
+          typing_delay_ms: {
+            type: "number",
+            description: "Per-character delay for desktop text entry. Default 10ms."
+          },
           index: {
             type: "number",
             description: "If multiple target points are found, use this 1-based index. Default 1."
@@ -282978,6 +283241,10 @@ var init_vision_action_loop = __esm({
             type: "string",
             description: "Optional screenshot output directory. Relative paths resolve from the workspace."
           },
+          window_title: {
+            type: "string",
+            description: "Optional X11/XWayland window title to capture instead of the whole desktop. Useful on Wayland when root screenshots are blocked."
+          },
           clear_artifacts: {
             type: "boolean",
             description: "Only for reset. If true, also deletes this session's screenshot directory."
@@ -283064,8 +283331,12 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
         const includeVision = asBoolean(args["include_vision"], true);
         const visionPromptOverride = asString(args["vision_prompt"]);
         const language = asString(args["language"]) || "eng";
+        const windowTitle = asString(args["window_title"]);
         const button = this.parseButton(args["button"]);
         const clickType = args["click_type"] === "double" ? "double" : "single";
+        const textToType = asString(args["text"]);
+        const keyToPress = asString(args["key"]);
+        const typingDelayMs = clampInteger2(args["typing_delay_ms"], 10, 0, 500);
         const index = clampInteger2(args["index"], 1, 1, 100);
         const delayMs = clampInteger2(args["delay_ms"], 0, 0, 6e4);
         const maxSteps = action === "run" ? clampInteger2(args["max_steps"], DEFAULT_MAX_STEPS, 1, HARD_MAX_STEPS) : 1;
@@ -283088,13 +283359,28 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
           }
           const stamp = timestampSlug2();
           const screenshotPath = join52(sessionDir2, `${stamp}-step-${step}-before.png`);
-          if (process.platform === "linux" && (process.env["WAYLAND_DISPLAY"] || /wayland/i.test(process.env["XDG_SESSION_TYPE"] || ""))) {
-            yield "Vision action loop: requesting desktop screenshot permission if the system prompts";
+          if (!windowTitle && process.platform === "linux" && (process.env["WAYLAND_DISPLAY"] || /wayland/i.test(process.env["XDG_SESSION_TYPE"] || ""))) {
+            yield "Vision action loop: capturing full desktop on Wayland. For unattended app loops, pass window_title to avoid compositor screenshot prompts.";
           }
           yield `Vision action loop: capturing screenshot ${step}/${maxSteps}`;
           let screenshotBackend = "";
+          let captureOffset = { x: 0, y: 0 };
+          let captureWindow;
           try {
-            screenshotBackend = captureDesktopScreenshot(screenshotPath);
+            if (windowTitle) {
+              const windowCapture = captureDesktopWindowScreenshot(screenshotPath, windowTitle);
+              screenshotBackend = windowCapture.backend;
+              captureOffset = { x: windowCapture.x, y: windowCapture.y };
+              captureWindow = {
+                windowId: windowCapture.windowId,
+                x: windowCapture.x,
+                y: windowCapture.y,
+                width: windowCapture.width,
+                height: windowCapture.height
+              };
+            } else {
+              screenshotBackend = captureDesktopScreenshot(screenshotPath);
+            }
             mutatedFiles.push(screenshotPath);
           } catch (err) {
             success = false;
@@ -283121,6 +283407,9 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
           outputLines.push(`Screenshot saved: ${screenshotPath}`);
           outputLines.push(`Screen: ${dims.width}x${dims.height}`);
           outputLines.push(`Screenshot backend: ${screenshotBackend}`);
+          if (captureWindow) {
+            outputLines.push(`Window: ${windowTitle} id=${captureWindow.windowId} geometry=${captureWindow.x},${captureWindow.y} ${captureWindow.width}x${captureWindow.height}`);
+          }
           let ocr = null;
           if (includeOcr) {
             yield `Vision action loop: running OCR for screenshot ${step}/${maxSteps}`;
@@ -283209,6 +283498,14 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
               selectedPoint = pointFromVisionResult({ points: ocrPoints, source: "tesseract-ocr" }, index, dims);
             }
             if (selectedPoint) {
+              if (captureOffset.x !== 0 || captureOffset.y !== 0) {
+                selectedPoint = {
+                  ...selectedPoint,
+                  pixelX: selectedPoint.pixelX + captureOffset.x,
+                  pixelY: selectedPoint.pixelY + captureOffset.y,
+                  source: `${selectedPoint.source}+window:${captureWindow?.windowId ?? "active"}`
+                };
+              }
               outputLines.push(`Target: ${effectiveTarget}`);
               outputLines.push(`Point source: ${selectedPoint.source}`);
               outputLines.push(`Mapped point: (${Math.round(selectedPoint.pixelX)}, ${Math.round(selectedPoint.pixelY)}) normalized (${selectedPoint.x.toFixed(4)}, ${selectedPoint.y.toFixed(4)})`);
@@ -283238,15 +283535,29 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
             } else {
               yield `Vision action loop: performing ${effectiveOperation} at (${pixelX}, ${pixelY})`;
               try {
+                if (captureWindow?.windowId) {
+                  const activationBackend = activateDesktopWindow(captureWindow.windowId);
+                  outputLines.push(`Activated window via ${activationBackend}: ${captureWindow.windowId}`);
+                }
                 const backend = effectiveOperation === "move" ? moveDesktopPointer(pixelX, pixelY) : clickDesktopAt(pixelX, pixelY, button, clickType);
                 actionTaken = effectiveOperation === "move" ? `Moved pointer to (${pixelX}, ${pixelY}) via ${backend}` : `Clicked at (${pixelX}, ${pixelY}) via ${backend} [${button} ${clickType}]`;
                 outputLines.push(actionTaken);
+                if (textToType) {
+                  const keyboardBackend = typeDesktopText(textToType, typingDelayMs);
+                  outputLines.push(`Typed text via ${keyboardBackend}: ${JSON.stringify(textToType)}`);
+                  actionTaken += `; typed text via ${keyboardBackend}`;
+                }
+                if (keyToPress) {
+                  const keyboardBackend = pressDesktopKey(keyToPress);
+                  outputLines.push(`Pressed key via ${keyboardBackend}: ${keyToPress}`);
+                  actionTaken += `; pressed key via ${keyboardBackend}: ${keyToPress}`;
+                }
                 afterScreenshotPath = join52(sessionDir2, `${timestampSlug2()}-step-${step}-after.png`);
-                if (process.platform === "linux" && (process.env["WAYLAND_DISPLAY"] || /wayland/i.test(process.env["XDG_SESSION_TYPE"] || ""))) {
-                  yield "Vision action loop: requesting desktop screenshot permission for post-action verification if the system prompts";
+                if (!windowTitle && process.platform === "linux" && (process.env["WAYLAND_DISPLAY"] || /wayland/i.test(process.env["XDG_SESSION_TYPE"] || ""))) {
+                  yield "Vision action loop: capturing post-action full desktop on Wayland. For unattended app loops, pass window_title to avoid compositor screenshot prompts.";
                 }
                 yield `Vision action loop: capturing post-action screenshot ${step}/${maxSteps}`;
-                const afterBackend = captureDesktopScreenshot(afterScreenshotPath);
+                const afterBackend = windowTitle ? captureDesktopWindowScreenshot(afterScreenshotPath, windowTitle).backend : captureDesktopScreenshot(afterScreenshotPath);
                 mutatedFiles.push(afterScreenshotPath);
                 outputLines.push(`Post-action screenshot: ${afterScreenshotPath}`);
                 outputLines.push(`Post-action screenshot backend: ${afterBackend}`);
@@ -283260,6 +283571,35 @@ State: ${statePath}${clearArtifacts ? "\nArtifacts cleared: true" : ""}`,
             }
           } else if (effectiveOperation !== "none" && (effectiveTarget || hasCoordinates)) {
             outputLines.push(`Action skipped: no usable point for operation '${effectiveOperation}'.`);
+          } else if (allowActions && !dryRun && (textToType || keyToPress)) {
+            try {
+              const keyboardActions = [];
+              if (captureWindow?.windowId) {
+                const activationBackend = activateDesktopWindow(captureWindow.windowId);
+                keyboardActions.push(`Activated window via ${activationBackend}: ${captureWindow.windowId}`);
+              }
+              if (textToType) {
+                const keyboardBackend = typeDesktopText(textToType, typingDelayMs);
+                keyboardActions.push(`Typed text via ${keyboardBackend}: ${JSON.stringify(textToType)}`);
+              }
+              if (keyToPress) {
+                const keyboardBackend = pressDesktopKey(keyToPress);
+                keyboardActions.push(`Pressed key via ${keyboardBackend}: ${keyToPress}`);
+              }
+              actionTaken = keyboardActions.join("; ");
+              outputLines.push(actionTaken);
+              afterScreenshotPath = join52(sessionDir2, `${timestampSlug2()}-step-${step}-after.png`);
+              const afterBackend = windowTitle ? captureDesktopWindowScreenshot(afterScreenshotPath, windowTitle).backend : captureDesktopScreenshot(afterScreenshotPath);
+              mutatedFiles.push(afterScreenshotPath);
+              outputLines.push(`Post-action screenshot: ${afterScreenshotPath}`);
+              outputLines.push(`Post-action screenshot backend: ${afterBackend}`);
+            } catch (err) {
+              success = false;
+              actionError = err instanceof Error ? err.message : String(err);
+              error = actionError;
+              outputLines.push(`Keyboard action failed: ${actionError}`);
+              outputLines.push(desktopAutomationRecoveryMessage());
+            }
           } else if (action === "run" && !effectiveTarget && !hasCoordinates) {
             success = false;
             error = "Vision loop stopped: no target or coordinates were provided and visual planning did not identify a clickable target. Stopping instead of repeating observe-only screenshots.";

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.205",
+  "version": "1.0.206",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.205",
+      "version": "1.0.206",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.205",
+  "version": "1.0.206",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",