@forsion/tangu-computer-use 0.6.1 → 0.6.2

package/scripts/probe-recorder-windows.mjs

@@ -0,0 +1,356 @@
+#!/usr/bin/env node
+// 电脑历史 Windows 采集器(windows-bridge.exe serve)的真机实测台架 —— 只能在 Windows 上跑(本机是 mac 时走 CI:
+// .github/workflows/windows-recorder-probe.yml,推 ci/windows-recorder* 分支触发)。
+//
+// 用真命名管道订阅 recordSubscribe,然后真的操作桌面:记事本打字 / 组合键 / 点菜单,stdio 助手 focusWindow(agent 标记),
+// Chrome 普通窗口(标题 + 网址 + 排除站点)、Chrome 无痕 / Edge InPrivate(只能留不带标题的切换)、网页输入框与密码框。
+// 输入走 SendKeys / mouse_event(注入的输入;采集器不按注入标记判 agent,所以照常记)。
+//
+// 用法:node scripts/probe-recorder-windows.mjs <windows-bridge.exe> [--out probe-results.json]
+// 退出码:必选断言有任何一条失败 → 1。所有事件与断言写进 --out(CI 作为 artifact 上传)。
+
+import { execFileSync, spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+
+const exe = path.resolve(process.argv[2] ?? "native/windows/bridge-rs/target/release/windows-bridge.exe");
+const outIndex = process.argv.indexOf("--out");
+const outPath = outIndex > 0 ? process.argv[outIndex + 1] : "probe-results.json";
+if (process.platform !== "win32") {
+	console.error("probe-recorder-windows only runs on Windows (use the CI workflow from a mac).");
+	process.exit(2);
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const results = [];
+const findings = {};
+function check(name, ok, detail = "", required = true) {
+	results.push({ name, ok: !!ok, required, detail });
+	console.log(`${ok ? "PASS" : required ? "FAIL" : "INFO"}  ${name}${detail ? ` — ${detail}` : ""}`);
+}
+
+function ps(script) {
+	try {
+		return execFileSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], { encoding: "utf8", timeout: 30_000 }).trim();
+	} catch (e) {
+		return `ERROR: ${String(e.stderr || e.message).trim()}`;
+	}
+}
+
+const USER32 = `Add-Type -Name U -Namespace W -MemberDefinition '
+[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
+[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int c);
+[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
+[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, IntPtr e);
+[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+[DllImport("user32.dll")] public static extern void keybd_event(byte v, byte s, uint f, IntPtr e);
+public struct RECT { public int L; public int T; public int R; public int B; }
+';`;
+
+function mainWindow(pid) {
+	return Number(ps(`(Get-Process -Id ${pid} -ErrorAction SilentlyContinue).MainWindowHandle`)) || 0;
+}
+function mainWindowOf(name) {
+	const out = ps(`Get-Process -Name ${name} -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | ForEach-Object { "$($_.Id)|$($_.MainWindowHandle)|$($_.MainWindowTitle)" }`);
+	return out.split(/\r?\n/).filter((l) => l.includes("|")).map((l) => {
+		const [pid, hwnd, ...title] = l.split("|");
+		return { pid: Number(pid), hwnd: Number(hwnd), title: title.join("|") };
+	});
+}
+function focus(hwnd) {
+	// 按住 Alt 解除前台锁(SetForegroundWindow 对后台进程的限制),再置前。Alt 按下抬起之间垫一个空键(F24):
+	// 单独轻敲 Alt 会让记事本进菜单模式、让 Chrome 聚焦到应用菜单按钮,焦点被抢,打的字进不了输入框。
+	ps(`${USER32} [W.U]::keybd_event(0x12,0,0,[IntPtr]::Zero); [W.U]::keybd_event(0x87,0,0,[IntPtr]::Zero); [W.U]::keybd_event(0x87,0,2,[IntPtr]::Zero); [W.U]::keybd_event(0x12,0,2,[IntPtr]::Zero); [W.U]::ShowWindow([IntPtr]${hwnd}, 9) | Out-Null; [W.U]::SetForegroundWindow([IntPtr]${hwnd}) | Out-Null`);
+}
+function sendKeys(keys) {
+	ps(`Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${keys.replace(/'/g, "''")}')`);
+}
+function clickAt(x, y) {
+	ps(`${USER32} [W.U]::SetCursorPos(${x}, ${y}) | Out-Null; Start-Sleep -Milliseconds 100; [W.U]::mouse_event(2,0,0,0,[IntPtr]::Zero); [W.U]::mouse_event(4,0,0,0,[IntPtr]::Zero)`);
+}
+function windowRect(hwnd) {
+	const out = ps(`${USER32} $r = New-Object W.U+RECT; [W.U]::GetWindowRect([IntPtr]${hwnd}, [ref]$r) | Out-Null; "$($r.L),$($r.T),$($r.R),$($r.B)"`);
+	const [l, t, r, b] = out.split(",").map(Number);
+	return { l, t, r, b };
+}
+
+// ── 管道订阅 ──
+
+const user = randomBytes(4).toString("hex");
+const pipe = `\\\\.\\pipe\\tangu-computer-use-recorder-probe-${user}`;
+const server = spawn(exe, ["serve", "--pipe", pipe], { stdio: ["ignore", "inherit", "inherit"], windowsHide: true, env: { ...process.env, TANGU_RECORDER_DEBUG: "1" } });
+server.on("exit", (code) => console.log(`[probe] serve exited with ${code}`));
+
+function subscribe(policy, label) {
+	return new Promise((resolve, reject) => {
+		const events = [];
+		const socket = net.createConnection(pipe);
+		let buffer = "";
+		let reply = null;
+		socket.setEncoding("utf8");
+		socket.on("connect", () => socket.write(`${JSON.stringify({ id: `probe_${label}`, cmd: "recordSubscribe", policy })}\n`));
+		socket.on("data", (chunk) => {
+			buffer += chunk;
+			const lines = buffer.split("\n");
+			buffer = lines.pop() ?? "";
+			for (const line of lines) {
+				if (!line.trim()) continue;
+				const msg = JSON.parse(line);
+				if (!reply) {
+					reply = msg;
+					clearTimeout(timer);
+					resolve({ reply, events, socket });
+					continue;
+				}
+				if (msg.ev) {
+					events.push(msg.ev);
+					console.log(`[ev:${label}] ${JSON.stringify(msg.ev)}`);
+				}
+			}
+		});
+		socket.on("error", reject);
+		// 只在回包之前生效(成功后清掉 —— 第四轮就是这个计时器到点把好好的订阅连接 destroy 了)。
+		const timer = setTimeout(() => { socket.destroy(); reject(new Error("subscribe timeout")); }, 5_000);
+	});
+}
+
+async function connectWithRetry(policy, label) {
+	for (let i = 0; i < 40; i++) {
+		try {
+			return await subscribe(policy, label);
+		} catch (e) {
+			if (i % 10 === 0) console.log(`[probe] connect attempt ${i} (${label}): ${e.code || e.message}`);
+			if (!String(e.code || e.message).match(/ENOENT|EBUSY|timeout/)) throw e;
+			await sleep(250);
+		}
+	}
+	throw new Error("could not connect to the recorder pipe");
+}
+
+const since = (events, t0) => events.filter((e) => e.t >= t0 - 50);
+/** recorder 线程此刻卡在哪一步(phase / phaseMs / backlog),每个场景后打一行,卡死时直接看出卡在哪个 UIA 调用。 */
+async function logDiag(label) {
+	const d = await diagnostics();
+	console.log(`[probe] diag after ${label}: ${JSON.stringify(d?.result)}`);
+	(findings.diag ??= []).push({ label, ...d?.result });
+}
+
+async function main() {
+	const main = await connectWithRetry({}, "open");
+	check("recordSubscribe reply", main.reply?.ok === true && main.reply?.result?.protocolVersion === 13 && main.reply?.result?.axTrusted === true, JSON.stringify(main.reply));
+	await sleep(1500);
+	findings.initialEvents = main.events.slice();
+	check("initial snapshot or system state", main.events.length > 0, JSON.stringify(main.events.slice(0, 3)));
+	if (main.events.some((e) => e.kind === "system" && e.state === "locked")) {
+		check("session is not locked on the runner", false, "the recorder thinks the input desktop is locked; nothing else can be observed");
+	}
+
+	// ── 记事本:切换、打字、组合键、点菜单 ──
+	let t0 = Date.now();
+	const notepad = spawn("notepad.exe", [], { detached: true, stdio: "ignore" });
+	await sleep(2500);
+	const npWindows = mainWindowOf("notepad");
+	const np = npWindows.find((w) => w.hwnd) ?? { hwnd: mainWindow(notepad.pid), pid: notepad.pid };
+	findings.notepad = np;
+	focus(np.hwnd);
+	await sleep(1500);
+	await logDiag("notepad-switch");
+	const npEvents = () => since(main.events, t0).filter((e) => e.app?.bundleId === "notepad.exe");
+	check("switching to Notepad emits an app event with the exe name", npEvents().some((e) => e.kind === "app"), JSON.stringify(npEvents().slice(0, 2)));
+	check("Notepad app event has a display name", npEvents().some((e) => e.kind === "app" && e.app?.name && e.app.name !== "notepad.exe"), JSON.stringify(npEvents().find((e) => e.kind === "app")?.app));
+
+	t0 = Date.now();
+	sendKeys("hello world");
+	await sleep(3500);
+	const typed = since(main.events, t0).filter((e) => e.kind === "text");
+	check("typing in Notepad produces a text event", typed.some((e) => e.text?.includes("hello world")), JSON.stringify(typed));
+	await logDiag("notepad-typing");
+	check("text event carries the field role", typed.some((e) => typeof e.el?.role === "string"), JSON.stringify(typed[0]?.el));
+
+	t0 = Date.now();
+	sendKeys("^a");
+	await sleep(800);
+	const keys = since(main.events, t0).filter((e) => e.kind === "key");
+	check("Ctrl+A is recorded as a shortcut", keys.some((e) => e.keys === "Ctrl+A"), JSON.stringify(keys));
+	sendKeys("{DEL}");
+	await sleep(2500);
+
+	t0 = Date.now();
+	const rect = windowRect(np.hwnd);
+	findings.notepadRect = rect;
+	// 经典记事本:标题栏下方第一项菜单「File / 文件」
+	clickAt(rect.l + 30, rect.t + 45);
+	await sleep(1200);
+	sendKeys("{ESC}");
+	await sleep(500);
+	await logDiag("notepad-click");
+	const clicks = since(main.events, t0).filter((e) => e.kind === "click");
+	check("clicking a menu item records role and label", clicks.some((e) => e.el?.role === "MenuItem" && e.el?.label), JSON.stringify(clicks), false);
+	check("a click in Notepad is recorded", clicks.length > 0, JSON.stringify(clicks));
+
+	// ── agent 标记:stdio 助手 focusWindow 期间切过去的前台 → origin:"agent" ──
+	// 另开一个记事本进程当「别的窗口」(Explorer 的主窗口句柄可能是桌面 Progman,置前后前台会落回记事本,前提不成立)。
+	const other = spawn("notepad.exe", [], { detached: true, stdio: "ignore" });
+	await sleep(2500);
+	const otherWin = mainWindowOf("notepad").find((w) => w.pid !== np.pid);
+	const fg = () => Number(ps(`${USER32} [W.U]::GetForegroundWindow().ToInt64()`)) || 0;
+	for (let i = 0; i < 3 && otherWin && fg() !== otherWin.hwnd; i++) {
+		focus(otherWin.hwnd);
+		await sleep(800);
+	}
+	findings.agentPrecondition = { foreground: fg(), notepad: np.hwnd, other: otherWin };
+	check("precondition: another window is in front before the agent acts", otherWin && fg() === otherWin.hwnd, JSON.stringify(findings.agentPrecondition));
+	await sleep(800);
+	t0 = Date.now();
+	const agentResult = await stdioFocus(np.pid);
+	findings.agentFocus = agentResult;
+	await sleep(1500);
+	await logDiag("agent-focus");
+	const agentEvents = since(main.events, t0).filter((e) => e.app?.bundleId === "notepad.exe");
+	check("foreground change by the agent carries origin=agent", agentEvents.some((e) => e.origin === "agent"), JSON.stringify(agentEvents));
+	other.unref();
+
+	// ── 浏览器 ──
+	const chrome = ["C:/Program Files/Google/Chrome/Application/chrome.exe", "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"].find(existsSync);
+	const edge = ["C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe", "C:/Program Files/Microsoft/Edge/Application/msedge.exe"].find(existsSync);
+	const flags = (dir) => [`--user-data-dir=${dir}`, "--no-first-run", "--no-default-browser-check", "--disable-search-engine-choice-screen", "--disable-features=PrivacySandboxSettings4"];
+	if (chrome) {
+		const siteOff = await connectWithRetry({ excludeDomains: ["example.com"] }, "siteOff");
+		t0 = Date.now();
+		const profile = mkdtempSync(path.join(os.tmpdir(), "probe-chrome-"));
+		spawn(chrome, [...flags(profile), "https://example.com/?q=secret#frag"], { detached: true, stdio: "ignore" }).unref();
+		await sleep(7000);
+		const win = mainWindowOf("chrome").find((w) => w.title);
+		findings.chromeNormal = win;
+		if (win) focus(win.hwnd);
+		// 第一次遍历可能走不完(Chrome 此刻才打开无障碍):采集器按无痕先不记,约 1 秒后自己重走,这里多等几轮。
+		await sleep(6000);
+		const ev = since(main.events, t0).filter((e) => e.app?.bundleId === "chrome.exe");
+		check("Chrome normal window has a title", ev.some((e) => e.title?.includes("Example")), JSON.stringify(ev.slice(-3)));
+		check("Chrome URL is read and sanitized", ev.some((e) => /^https:\/\/example\.com\/?$/.test(e.url ?? "")), JSON.stringify(ev.map((e) => e.url)));
+		const off = since(siteOff.events, t0).filter((e) => e.app?.bundleId === "chrome.exe");
+		check("an excluded site leaves only a content-free marker", off.length > 0 && off.every((e) => !e.title && !e.url && (e.app?.excluded || e.kind === "app")), JSON.stringify(off.slice(-3)));
+		siteOff.socket.destroy();
+
+		// 网页输入框与密码框
+		t0 = Date.now();
+		spawn(chrome, [...flags(profile), "data:text/html,<title>Form</title><input id=a autofocus placeholder=Name>"], { detached: true, stdio: "ignore" }).unref();
+		await sleep(4000);
+		const formWin = mainWindowOf("chrome").find((w) => w.title.startsWith("Form"));
+		if (formWin) focus(formWin.hwnd);
+		await sleep(1500);
+		sendKeys("plain words");
+		await sleep(3500);
+		const webText = since(main.events, t0).filter((e) => e.kind === "text");
+		check("typing in a web input is recorded", webText.some((e) => e.text?.includes("plain words")), JSON.stringify(webText), false);
+		t0 = Date.now();
+		spawn(chrome, [...flags(profile), "data:text/html,<title>Login</title><input type=password autofocus>"], { detached: true, stdio: "ignore" }).unref();
+		await sleep(4000);
+		const loginWin = mainWindowOf("chrome").find((w) => w.title.startsWith("Login"));
+		if (loginWin) focus(loginWin.hwnd);
+		await sleep(1500);
+		sendKeys("hunter2secret");
+		await sleep(3500);
+		const leaked = since(main.events, t0).filter((e) => JSON.stringify(e).includes("hunter2"));
+		check("password fields are never recorded", leaked.length === 0, JSON.stringify(leaked));
+
+		// 无痕:先关掉前面的普通窗口,免得前台落回普通窗口时误判
+		ps("Get-Process chrome -ErrorAction SilentlyContinue | Stop-Process -Force");
+		await sleep(1500);
+		t0 = Date.now();
+		spawn(chrome, [...flags(profile), "--incognito", "https://example.com/incognito-page"], { detached: true, stdio: "ignore" }).unref();
+		await sleep(6000);
+		const incognito = mainWindowOf("chrome");
+		findings.chromeIncognitoTitles = incognito.map((w) => w.title);
+		const incWin = incognito[0];
+		if (incWin) focus(incWin.hwnd);
+		await sleep(2500);
+		const incEvents = since(main.events, t0).filter((e) => e.app?.bundleId === "chrome.exe");
+		findings.chromeIncognitoEvents = incEvents;
+		check("the incognito window was observed at all", incEvents.length > 0, JSON.stringify(incEvents));
+		check("Chrome incognito never leaks title or URL", incEvents.every((e) => !String(e.title ?? "").length || !/incognito-page|Example/.test(e.title)) && incEvents.every((e) => !String(e.url ?? "").includes("incognito-page")), JSON.stringify(incEvents));
+	} else {
+		check("Chrome is installed on the runner", false, "skipped Chrome scenarios", false);
+	}
+	if (edge) {
+		t0 = Date.now();
+		const profile = mkdtempSync(path.join(os.tmpdir(), "probe-edge-"));
+		spawn(edge, [...flags(profile), "--inprivate", "https://example.com/inprivate-page"], { detached: true, stdio: "ignore" }).unref();
+		await sleep(8000);
+		const wins = mainWindowOf("msedge");
+		findings.edgeInPrivateTitles = wins.map((w) => w.title);
+		const w = wins.find((x) => /inprivate/i.test(x.title)) ?? wins.at(-1);
+		if (w) focus(w.hwnd);
+		await sleep(2500);
+		const edgeEvents = since(main.events, t0).filter((e) => e.app?.bundleId === "msedge.exe");
+		findings.edgeInPrivateEvents = edgeEvents;
+		check("the InPrivate window was observed at all", edgeEvents.length > 0, JSON.stringify(edgeEvents));
+		check("Edge InPrivate never leaks title or URL", edgeEvents.every((e) => !/inprivate-page|Example/.test(e.title ?? "") && !String(e.url ?? "").includes("inprivate-page")), JSON.stringify(edgeEvents));
+	} else {
+		check("Edge is installed on the runner", false, "skipped Edge scenarios", false);
+	}
+
+	// ── 退订后采集器拆干净、空闲后退出 ──
+	main.socket.destroy();
+	findings.allEvents = main.events;
+	await sleep(1000);
+	const diag = await diagnostics();
+	check("diagnostics after unsubscribe reports no subscribers", diag?.result?.subscribers === 0, JSON.stringify(diag));
+	notepad.unref();
+}
+
+function diagnostics() {
+	return new Promise((resolve) => {
+		const s = net.createConnection(pipe);
+		let buf = "";
+		s.setEncoding("utf8");
+		s.on("connect", () => s.write(`${JSON.stringify({ id: "diag", cmd: "diagnostics" })}\n`));
+		s.on("data", (c) => { buf += c; if (buf.includes("\n")) { s.destroy(); resolve(JSON.parse(buf.split("\n")[0])); } });
+		s.on("error", () => resolve(null));
+		setTimeout(() => resolve(null), 5000);
+	});
+}
+
+/** stdio 助手:listRoots(pid) → focusWindow(rootRef)。 */
+function stdioFocus(pid) {
+	return new Promise((resolve) => {
+		const child = spawn(exe, [], { stdio: ["pipe", "pipe", "inherit"] });
+		let buf = "";
+		const send = (id, cmd, args) => child.stdin.write(`${JSON.stringify({ protocolVersion: 4, id, cmd, args })}\n`);
+		child.stdout.setEncoding("utf8");
+		child.stdout.on("data", (chunk) => {
+			buf += chunk;
+			const lines = buf.split("\n");
+			buf = lines.pop() ?? "";
+			for (const line of lines) {
+				if (!line.trim()) continue;
+				const msg = JSON.parse(line);
+				if (msg.id === "roots") {
+					const root = (msg.result?.roots ?? msg.result?.windows ?? []).find((r) => r.rootRef);
+					if (!root) { child.kill(); return resolve({ error: "no root", msg }); }
+					send("focus", "focusWindow", { rootRef: root.rootRef });
+				} else if (msg.id === "focus") {
+					setTimeout(() => child.kill(), 1500);
+					resolve(msg);
+				}
+			}
+		});
+		send("roots", "listRoots", { pid });
+		setTimeout(() => { child.kill(); resolve({ error: "timeout" }); }, 15_000);
+	});
+}
+
+try {
+	await main();
+} catch (e) {
+	check("probe ran to completion", false, String(e?.stack || e));
+}
+writeFileSync(outPath, JSON.stringify({ results, findings }, null, 2));
+server.kill();
+const failed = results.filter((r) => r.required && !r.ok);
+console.log(`\n${results.length - failed.length}/${results.length} checks passed${failed.length ? `; FAILED: ${failed.map((f) => f.name).join(", ")}` : ""}`);
+process.exit(failed.length ? 1 : 0);

package/skills/computer-use/SKILL.md

@@ -11,6 +11,10 @@ The Computer Use tools let you observe and control any on-screen application thr
 tree, OCR, and screenshots. Reach for them only when the job genuinely needs a GUI: an API, a CLI, or
 reading a file directly is always cheaper and more reliable.
 
+Forsion's own windows are not a Computer Use target. To change a Forsion setting or open one of its
+pages, use `list_ui_commands` and then `run_ui_command` / `set_ui_setting`; a tool result marked
+`[forsion]` means you are looking at Forsion itself.
+
 For a page you can simply fetch, use `curl` through bash. For pure web work prefer `browser_task` /
 `browser_*` — they are lighter. Use Computer Use when the workflow lives in a native desktop app, or
 when a web page must be handled inside the same window forest as a desktop app.

package/src/foregroundNote.ts

@@ -41,3 +41,19 @@ export function foregroundNote(execution: unknown): string {
   const why = steps.map((s) => s?.escalationReason).find(Boolean);
   return `\n[foreground] This action took the foreground (focus/pointer was moved to the target)${why ? ` — ${why}` : ''}. To keep actions in the background, prefer setText to fill fields; strict background-only is the plugin's "strict background" setting (act_ui no longer takes a headless parameter).`;
 }
+
+/**
+ * 自家窗口提示:目标窗口属于拉起本引擎的那个桌面应用(= Forsion 自己;托管引擎是它的直接子进程)时,
+ * 明说一句「改用界面命令」。Forsion 反馈 6a239e58:agent 用 Computer Use 去点 Forsion 自己的设置窗口,
+ * 界面一重渲染元素引用就失效(一轮里 3 次 stale ref + 1 次 stale state),而同一件事 run_ui_command 一次就办完。
+ * 只提示不拦:没有对应界面命令的地方(插件自绘页)仍只能这样操作。
+ * TUI / 独立运行时父进程是 shell(没有窗口)、父进程已死时 ppid=1,都不会命中。
+ * 读的是 details.target.pid(observe / act / wait)与 details.windows[].pid(find_roots);形状变了就静默返回空串。
+ */
+export function selfWindowNote(details: unknown, hostPid: number = process.ppid): string {
+  const d = details as { target?: { pid?: unknown }; windows?: Array<{ pid?: unknown }> } | null | undefined;
+  if (!d || typeof d !== 'object' || !(hostPid > 1)) return '';
+  const own = d.target?.pid === hostPid || (Array.isArray(d.windows) && d.windows.some((w) => w?.pid === hostPid));
+  if (!own) return '';
+  return `\n[forsion] pid ${hostPid} is Forsion itself, the app you are running in. Operate Forsion through its own UI tools (list_ui_commands, then run_ui_command / set_ui_setting): they act directly and report the outcome. Computer Use on Forsion's own windows is unreliable because the UI re-renders under you and element refs go stale; keep it for controls that have no UI command.`;
+}

package/src/index.ts

@@ -41,6 +41,8 @@ const plugin: TanguPlugin = {
                 ? 'On macOS, if the target app may not be running yet, call ensure_app first — it starts the app in the background (no focus steal).'
                 : 'On Windows and Linux, ensure_app is unavailable. Start with find_roots to discover running apps. If necessary, launch an existing app using a verified executable or platform app launcher, then observe again. Do not guess app paths or treat ensure_app absence as failure of the observation tools.',
               'Prefer the background path: to fill a field use a single setText action (writes the value without taking focus) rather than click-then-type; the foreground is taken only when an action genuinely needs it, and the result says so on a [foreground] line — when it does, tell the user. Strict background-only is a plugin setting, not a tool parameter.',
+              // The desktop side panel (Forsion chat docked beside an app window) prefixes every message with this token.
+              'A user message that starts with <forsion-app app="…" bundleId="…" pid="…" windowId="…" title="…" /> was sent from a chat panel docked beside that app window: the user is working in it, so it is the default target. Questions about "this", "here" or "the selection" refer to it, and a quoted block right after the token is what they had selected there. Locate it with find_roots({pid}) (fall back to bundleId or app if the pid is gone) and continue with observe_ui. The token only names the window — you still need the tools to see its contents.',
               'For pure web tasks prefer browser_task / browser_* (lighter). Use the Computer Use browser tools (launch/navigate/evaluate_browser) only when a desktop workflow must touch a web page within the same @r root forest.',
               // Working discipline: re-observe before acting, derive element refs from the latest observation (they go stale), prefer the AX text and only screenshot when it is incomplete.
               'Re-observe (observe_ui) right before you act; take element references from the latest observation only — never reuse a reference from an earlier state. Prefer the accessibility text; fall back to a screenshot when the tree is incomplete.',

package/src/tools.ts

@@ -20,7 +20,7 @@ import {
 import { getComputerUseConfig } from './vendor/config.ts';
 import { isSupportedPlatform } from './helperState.ts';
 import { autoInstallFailed, ensureHelperCurrent, guideMissingPermissions } from './onboarding.ts';
-import { foregroundNote } from './foregroundNote.ts';
+import { foregroundNote, selfWindowNote } from './foregroundNote.ts';
 
 const execFileP = promisify(execFile);
 const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
@@ -189,7 +189,7 @@ export function makePiCtx(tcx: ToolContext, hasUI: boolean): ExtensionContext {
   };
 }
 
-/** AgentToolResult → 返回给模型的 string;image block(截图)经 collectImage 回灌;末尾附前台透明化提示。 */
+/** AgentToolResult → 返回给模型的 string;image block(截图)经 collectImage 回灌;末尾附前台透明化提示与「这是 Forsion 自己的窗口」提示。 */
 function render(result: AgentToolResult, tcx: ToolContext): string {
   const parts: string[] = [];
   for (const block of result.content) {
@@ -199,7 +199,8 @@ function render(result: AgentToolResult, tcx: ToolContext): string {
     }
   }
   const body = parts.join('\n') || '(no output)';
-  return body + foregroundNote((result as { details?: { execution?: unknown } }).details?.execution);
+  const details = (result as { details?: { execution?: unknown } }).details;
+  return body + foregroundNote(details?.execution) + selfWindowNote(details);
 }
 
 /** ensure_app:后台启动目标 app(open -g,不抢前台)+ 轮询 find_roots 直到有可观察窗口,回其 roots。 */

package/src/vendor/platform/macos/helper.ts

@@ -10,7 +10,7 @@ import type { PlatformDiagnostics } from "../types.ts";
 import { resolveMacosHelperAppPath } from "./helper-path.mjs";
 
 const COMMAND_TIMEOUT_MS = 15_000;
-const HELPER_PROTOCOL_VERSION = 13;
+const HELPER_PROTOCOL_VERSION = 14;
 const HELPER_SETUP_TIMEOUT_MS = 60_000;
 
 export const HELPER_BUNDLE_ID = "com.forsion.tangu-computer-use";

package/tangu-plugins/computer-use/dist/foregroundNote.js

@@ -14,6 +14,15 @@ function foregroundNote(execution) {
   return `
 [foreground] This action took the foreground (focus/pointer was moved to the target)${why ? ` \u2014 ${why}` : ""}. To keep actions in the background, prefer setText to fill fields; strict background-only is the plugin's "strict background" setting (act_ui no longer takes a headless parameter).`;
 }
+function selfWindowNote(details, hostPid = process.ppid) {
+  const d = details;
+  if (!d || typeof d !== "object" || !(hostPid > 1)) return "";
+  const own = d.target?.pid === hostPid || Array.isArray(d.windows) && d.windows.some((w) => w?.pid === hostPid);
+  if (!own) return "";
+  return `
+[forsion] pid ${hostPid} is Forsion itself, the app you are running in. Operate Forsion through its own UI tools (list_ui_commands, then run_ui_command / set_ui_setting): they act directly and report the outcome. Computer Use on Forsion's own windows is unreliable because the UI re-renders under you and element refs go stale; keep it for controls that have no UI command.`;
+}
 export {
-  foregroundNote
+  foregroundNote,
+  selfWindowNote
 };

package/tangu-plugins/computer-use/dist/index.js

@@ -171,7 +171,7 @@ var init_helper = __esm({
     init_coerce();
     init_helper_path();
     COMMAND_TIMEOUT_MS2 = 15e3;
-    HELPER_PROTOCOL_VERSION = 13;
+    HELPER_PROTOCOL_VERSION = 14;
     HELPER_SETUP_TIMEOUT_MS = 6e4;
     HELPER_BUNDLE_ID = "com.forsion.tangu-computer-use";
     HELPER_APP_PATH = resolveMacosHelperAppPath();
@@ -5283,6 +5283,14 @@ function foregroundNote(execution) {
   return `
 [foreground] This action took the foreground (focus/pointer was moved to the target)${why ? ` \u2014 ${why}` : ""}. To keep actions in the background, prefer setText to fill fields; strict background-only is the plugin's "strict background" setting (act_ui no longer takes a headless parameter).`;
 }
+function selfWindowNote(details, hostPid = process.ppid) {
+  const d = details;
+  if (!d || typeof d !== "object" || !(hostPid > 1)) return "";
+  const own = d.target?.pid === hostPid || Array.isArray(d.windows) && d.windows.some((w) => w?.pid === hostPid);
+  if (!own) return "";
+  return `
+[forsion] pid ${hostPid} is Forsion itself, the app you are running in. Operate Forsion through its own UI tools (list_ui_commands, then run_ui_command / set_ui_setting): they act directly and report the outcome. Computer Use on Forsion's own windows is unreliable because the UI re-renders under you and element refs go stale; keep it for controls that have no UI command.`;
+}
 
 // src/tools.ts
 var execFileP = promisify(execFile);
@@ -5467,7 +5475,8 @@ function render(result, tcx) {
     }
   }
   const body = parts.join("\n") || "(no output)";
-  return body + foregroundNote(result.details?.execution);
+  const details = result.details;
+  return body + foregroundNote(details?.execution) + selfWindowNote(details);
 }
 async function runEnsureApp(args, tcx, pctx) {
   if (process.platform !== "darwin") {
@@ -5648,6 +5657,8 @@ var plugin = {
         "These control ANY on-screen application through its accessibility tree + OCR + screenshots \u2014 use them when the task needs a desktop app rather than an API/CLI/file.",
         process.platform === "darwin" ? "On macOS, if the target app may not be running yet, call ensure_app first \u2014 it starts the app in the background (no focus steal)." : "On Windows and Linux, ensure_app is unavailable. Start with find_roots to discover running apps. If necessary, launch an existing app using a verified executable or platform app launcher, then observe again. Do not guess app paths or treat ensure_app absence as failure of the observation tools.",
         "Prefer the background path: to fill a field use a single setText action (writes the value without taking focus) rather than click-then-type; the foreground is taken only when an action genuinely needs it, and the result says so on a [foreground] line \u2014 when it does, tell the user. Strict background-only is a plugin setting, not a tool parameter.",
+        // The desktop side panel (Forsion chat docked beside an app window) prefixes every message with this token.
+        'A user message that starts with <forsion-app app="\u2026" bundleId="\u2026" pid="\u2026" windowId="\u2026" title="\u2026" /> was sent from a chat panel docked beside that app window: the user is working in it, so it is the default target. Questions about "this", "here" or "the selection" refer to it, and a quoted block right after the token is what they had selected there. Locate it with find_roots({pid}) (fall back to bundleId or app if the pid is gone) and continue with observe_ui. The token only names the window \u2014 you still need the tools to see its contents.',
         "For pure web tasks prefer browser_task / browser_* (lighter). Use the Computer Use browser tools (launch/navigate/evaluate_browser) only when a desktop workflow must touch a web page within the same @r root forest.",
         // Working discipline: re-observe before acting, derive element refs from the latest observation (they go stale), prefer the AX text and only screenshot when it is incomplete.
         "Re-observe (observe_ui) right before you act; take element references from the latest observation only \u2014 never reuse a reference from an earlier state. Prefer the accessibility text; fall back to a screenshot when the tree is incomplete.",

package/tangu-plugins/computer-use/tangu-plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "computer-use",
   "name": "Computer Use",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "apiVersion": 1,
   "entry": "dist/index.js",
   "commands": ["computer-use"],