mocode-ai 1.7.2 → 1.7.4

package/dist/tools/builtins/computer.js

@@ -8,12 +8,18 @@
  * 安全:risk=dangerous(权限弹窗默认高亮「拒绝」),plan 模式常驻屏蔽,
  * /cu off 时既不进 schema 也被运行时兜底拦截。
  */
-import { mkdir, readFile } from 'node:fs/promises';
+import { mkdir, readFile, rm } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { jailResolve } from '../../sandbox/index.js';
 import { captureDesktop } from '../../runtime/screen-capture.js';
 import { decodePng, encodePng, downscale, fitToLongEdge, crop, readPngSize, diffRatio, normToPhysical, normRectToPhysical, physicalToNorm, isNormCoord, } from '../../runtime/screen-pipeline.js';
 import { createInputInjector } from '../../runtime/input-injector.js';
+import { getUiaService, UiaElementGoneError, UiaWindowGoneError } from '../../runtime/uia-service.js';
+import { formatWindowList, pickWindow, pickWindowForReplay, } from '../../runtime/window-list.js';
+import { clearElementRefNames, formatSnapshot, matchSelector, rectCenter, rectToNorm, selectDisplayNodes, setElementRefNames, } from '../../runtime/uia-selector.js';
+import { describeCondition, parseWaitCondition, pickTypeMethod, waitUntil, WAIT_UNTIL_DEFAULT_TIMEOUT_MS, WAIT_UNTIL_MAX_TIMEOUT_MS, WAIT_UNTIL_MIN_TIMEOUT_MS, } from '../../runtime/wait-until.js';
+import { appendTrace, buildTraceEntry, isRecordedAction, isTraceSuspended, nextTraceSeq, outputIndicatesChange, traceEnabled, } from '../../flows/trace.js';
+import { lookupElementRefName } from '../../runtime/uia-selector.js';
 /**
  * 喂模型的截图长边上限。1280 是 Anthropic 的兼容下限,1568 对齐主流视觉模型原生分辨率 ——
  * 4K 屏上小字/密集 UI 的定位精度提升明显,代价是每张图多约 300 token。可用
@@ -60,8 +66,18 @@ const ACTIONS = [
     'key',
     'scroll',
     'wait',
+    'wait_until',
     'cursor_position',
+    'inspect',
+    'click_element',
+    'set_value',
+    'list_windows',
+    'focus_window',
 ];
+/** inspect 默认/最大展示节点数(design-notes/computer-use-rpa.md §2.4)。 */
+const INSPECT_DEFAULT_NODES = 120;
+const INSPECT_MAX_NODES = 300;
+const ELEMENT_ACTIONS = new Set(['click_element', 'set_value']);
 const ACTIONS_NEEDING_COORDINATE = new Set([
     'mouse_move',
     'left_click',
@@ -106,6 +122,24 @@ export function validateComputerArgs(args) {
             return `action "${action}" requires a non-empty text string`;
         }
     }
+    if (action === 'type' && args.method !== undefined) {
+        if (args.method !== 'auto' && args.method !== 'keys' && args.method !== 'paste') {
+            return 'type method must be auto|keys|paste';
+        }
+    }
+    if (action === 'wait_until') {
+        const parsed = parseWaitCondition(args.condition, diffThreshold());
+        if (!parsed.ok)
+            return parsed.error;
+        const to = args.timeout_ms;
+        if (to !== undefined &&
+            (typeof to !== 'number' ||
+                !Number.isInteger(to) ||
+                to < WAIT_UNTIL_MIN_TIMEOUT_MS ||
+                to > WAIT_UNTIL_MAX_TIMEOUT_MS)) {
+            return `timeout_ms must be an integer in ${WAIT_UNTIL_MIN_TIMEOUT_MS}-${WAIT_UNTIL_MAX_TIMEOUT_MS}`;
+        }
+    }
     if (action === 'scroll') {
         const dir = args.scroll_direction;
         if (dir !== 'up' && dir !== 'down' && dir !== 'left' && dir !== 'right') {
@@ -122,6 +156,53 @@ export function validateComputerArgs(args) {
             return `action "wait" requires duration_ms: integer in 1-${WAIT_MAX_MS}`;
         }
     }
+    if (action === 'inspect' && args.max_nodes !== undefined) {
+        const n = args.max_nodes;
+        if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > INSPECT_MAX_NODES) {
+            return `max_nodes must be an integer in 1-${INSPECT_MAX_NODES}`;
+        }
+    }
+    if (ELEMENT_ACTIONS.has(action)) {
+        if (typeof args.ref !== 'string' || !/^e\d+$/.test(args.ref)) {
+            return `action "${action}" requires ref: an element ref like "e12" from the latest inspect`;
+        }
+    }
+    if (action === 'click_element') {
+        const b = args.button;
+        if (b !== undefined && b !== 'left' && b !== 'right' && b !== 'middle') {
+            return 'click_element button must be left|right|middle';
+        }
+        const c = args.click_count;
+        if (c !== undefined && c !== 1 && c !== 2 && c !== 3)
+            return 'click_element click_count must be 1, 2 or 3';
+        if (args.via !== undefined && args.via !== 'mouse' && args.via !== 'pattern') {
+            return 'click_element via must be mouse|pattern';
+        }
+    }
+    if (action === 'set_value') {
+        if (typeof args.text !== 'string')
+            return 'action "set_value" requires a text string';
+        if (args.via !== undefined && args.via !== 'pattern' && args.via !== 'keys') {
+            return 'set_value via must be pattern|keys';
+        }
+    }
+    // 窗口作用域(design-notes/computer-use-rpa.md §4):window("wN",来自最近一次 list_windows)与 title_regex 二选一。
+    const hasWin = args.window !== undefined;
+    const hasRe = args.title_regex !== undefined;
+    if (hasWin && (typeof args.window !== 'string' || !/^w\d+$/.test(args.window))) {
+        return 'window must look like "w3" (a ref from the latest list_windows); use title_regex to match by title';
+    }
+    if (hasRe && (typeof args.title_regex !== 'string' || args.title_regex.length === 0)) {
+        return 'title_regex must be a non-empty string';
+    }
+    if (hasWin && hasRe)
+        return 'pass either window or title_regex, not both';
+    if (action === 'focus_window' && !hasWin && !hasRe) {
+        return 'action "focus_window" requires window ("wN" from list_windows) or title_regex';
+    }
+    if (args.move_to_primary !== undefined && typeof args.move_to_primary !== 'boolean') {
+        return 'move_to_primary must be a boolean';
+    }
     return null;
 }
 /**
@@ -135,9 +216,40 @@ export function validateComputerArgs(args) {
 let geometryCache = null;
 /** 会话级上一帧(已 downscale),用于差分。/cu off 时清空。 */
 let lastFrame = null;
+let elementSnapshot = null;
+let snapshotGeneration = 0;
+/**
+ * 最近一次 list_windows 的窗口快照(w1..wN → hwnd)。ref 只在下一次 list_windows 之前有效;
+ * title_regex 选择走实时枚举、不覆盖它,以免悄悄让模型手里的 wN 失效。
+ */
+let windowSnapshot = [];
+/** 窗口选择失败(ref 过期 / 正则无匹配或歧义)。execute 据此返回专门前缀,让模型重新 list_windows。 */
+class WindowSelectionError extends Error {
+}
+/** 按 window:'wN'(快照)或 title_regex(实时枚举)选窗口。 */
+async function resolveWindow(args, screen, signal) {
+    let picked;
+    if (typeof args.window === 'string') {
+        picked = pickWindow(windowSnapshot, { ref: args.window });
+    }
+    else {
+        const { items } = formatWindowList(await getUiaService().windows(signal), screen);
+        picked = pickWindow(items, { titleRegex: args.title_regex });
+    }
+    if (!picked.ok)
+        throw new WindowSelectionError(picked.error);
+    return picked.item;
+}
+function windowLabel(w) {
+    const title = w.title.replace(/\s+/g, ' ').trim();
+    return `${w.ref} ${w.processName ? `${w.processName}.exe` : 'unknown'} ${JSON.stringify(title.length > 60 ? `${title.slice(0, 60)}…` : title)}`;
+}
 export function resetComputerState() {
     geometryCache = null;
     lastFrame = null;
+    elementSnapshot = null;
+    windowSnapshot = [];
+    clearElementRefNames();
 }
 /** 抓一次主屏并返回几何信息。Windows 走 captureDesktop 的 geometry(零解码);其余平台只读 IHDR。 */
 async function capturePrimary(signal) {
@@ -234,7 +346,42 @@ async function toAttachment(path, m) {
 function commitFrame(img) {
     lastFrame = img;
 }
-async function executeAction(injector, action, args, screen) {
+/** 元素动作无法定位目标(ref 过期 / 元素消失)。execute 据此返回专门提示,让模型重新 inspect。 */
+class ElementNotFoundError extends Error {
+}
+/** 按 ref 查当前快照;未 inspect 或 ref 不在当前 generation → ElementNotFoundError。 */
+function lookupElement(ref) {
+    if (!elementSnapshot) {
+        throw new ElementNotFoundError(`no element snapshot yet; call inspect before using ref ${ref}`);
+    }
+    const item = elementSnapshot.items.get(ref);
+    if (!item) {
+        throw new ElementNotFoundError(`ref ${ref} is not in the current snapshot (generation ${elementSnapshot.generation}); call inspect again`);
+    }
+    return { hwnd: elementSnapshot.tree.window.hwnd, item };
+}
+/** via:'pattern' 时按优先级挑一个可用的点击语义 pattern。 */
+export function pickClickPattern(patterns) {
+    for (const p of ['invoke', 'toggle', 'select', 'expand']) {
+        if (patterns.includes(p))
+            return p;
+    }
+    return null;
+}
+/** 调 UIA act;元素已变化/消失统一转成 ElementNotFoundError。 */
+async function actOnElement(el, kind, signal, text) {
+    const { node } = el.item;
+    try {
+        return await getUiaService().act({ hwnd: el.hwnd, path: node.path, kind, text, expectRole: node.role, expectName: node.name }, signal);
+    }
+    catch (error) {
+        if (error instanceof UiaElementGoneError) {
+            throw new ElementNotFoundError(`element ${el.item.ref} ${node.role} ${JSON.stringify(node.name)} is gone or changed (${error.message}); call inspect again`);
+        }
+        throw error;
+    }
+}
+async function executeAction(injector, action, args, screen, signal) {
     const { physW, physH, originX, originY } = screen;
     // norm1000 → 虚拟桌面物理像素:先按物理分辨率缩放,再叠加主屏原点(多屏)。
     const toPhys = (pair) => {
@@ -279,8 +426,18 @@ async function executeAction(injector, action, args, screen) {
             return `dragged from (${x0}, ${y0}) to (${x1}, ${y1})`;
         }
         case 'type': {
-            await injector.typeText(args.text);
-            return `typed ${args.text.length} characters`;
+            const text = args.text;
+            const rawMethod = args.method;
+            const method = pickTypeMethod(text, rawMethod === 'auto' ? undefined : rawMethod);
+            if (method === 'paste') {
+                const { clipboardOverwritten } = await injector.paste(text);
+                return (`pasted ${text.length} characters via clipboard` +
+                    (clipboardOverwritten
+                        ? ' (WARNING: the previous clipboard held non-text data (image/files) and could not be restored)'
+                        : ''));
+            }
+            await injector.typeText(text);
+            return `typed ${text.length} characters`;
         }
         case 'key': {
             await injector.pressKey(args.text);
@@ -302,6 +459,56 @@ async function executeAction(injector, action, args, screen) {
             const [nx, ny] = physicalToNorm(pos.x - originX, pos.y - originY, physW, physH);
             return `cursor at physical (${pos.x}, ${pos.y}) = normalized (${nx}, ${ny})`;
         }
+        case 'click_element': {
+            const el = lookupElement(args.ref);
+            const { node } = el.item;
+            const label = `element ${el.item.ref} ${node.role} ${JSON.stringify(node.name)}`;
+            if (args.via === 'pattern') {
+                const kind = pickClickPattern(node.patterns);
+                if (!kind) {
+                    throw new Error(`${label} exposes no invoke/toggle/select/expand pattern; retry click_element with via "mouse"`);
+                }
+                await actOnElement(el, kind, signal);
+                return `clicked ${label} via ${kind} pattern`;
+            }
+            // mouse:按 hwnd+path 重定位拿**实时**矩形,不信任 inspect 时的旧矩形(窗口可能已移动)。
+            const { rect } = await actOnElement(el, 'rect', signal);
+            if (rect.w <= 0 || rect.h <= 0) {
+                throw new Error(`${label} has an empty bounding rectangle (hidden or collapsed); call inspect again`);
+            }
+            const [x, y] = rectCenter(rect);
+            await injector.moveTo(x, y);
+            const button = args.button ?? 'left';
+            const count = args.click_count ?? 1;
+            await injector.click(button, count);
+            return `clicked ${label} at (${x}, ${y})`;
+        }
+        case 'set_value': {
+            const el = lookupElement(args.ref);
+            const { node } = el.item;
+            const text = args.text;
+            const label = `element ${el.item.ref} ${node.role} ${JSON.stringify(node.name)}`;
+            // 密码框:模型不代填密码(design-notes/computer-use-rpa.md §2.6),用户需要时手动输入。
+            if (node.isPassword)
+                throw new Error(`${label} is a password field; refusing to fill it — ask the user to type it`);
+            const hasValuePattern = node.patterns.includes('value');
+            const via = args.via ?? (hasValuePattern ? 'pattern' : 'keys');
+            if (via === 'pattern') {
+                if (!hasValuePattern) {
+                    throw new Error(`${label} does not support ValuePattern; retry set_value with via "keys"`);
+                }
+                await actOnElement(el, 'setvalue', signal, text);
+            }
+            else {
+                await actOnElement(el, 'focus', signal);
+                await injector.pressKey('ctrl+a');
+                if (text.length > 0)
+                    await injector.typeText(text);
+                else
+                    await injector.pressKey('Delete');
+            }
+            return `set value of ${label} (${text.length} chars) via ${via}`;
+        }
         default:
             throw new Error(`no executor for action "${action}"`);
     }
@@ -321,11 +528,28 @@ function pct(values, p) {
     return Math.round(s[i]);
 }
 export function getComputerMetrics() {
+    return summarizeMetrics(metrics);
+}
+/** 纯函数:把一批度量汇总成分位数(导出供单测用合成数据覆盖,不依赖模块级 ring buffer)。 */
+export function summarizeMetrics(metrics) {
     const totals = metrics.map((m) => m.totalMs);
     const caps = metrics.map((m) => m.captureMs);
     const skipped = metrics.filter((m) => m.skipped).length;
     const bytes = metrics.reduce((a, m) => a + m.bytes, 0);
+    const grouped = new Map();
+    for (const m of metrics) {
+        const list = grouped.get(m.action);
+        if (list)
+            list.push(m.totalMs);
+        else
+            grouped.set(m.action, [m.totalMs]);
+    }
+    const byAction = {};
+    for (const [action, list] of grouped) {
+        byAction[action] = { count: list.length, p50: pct(list, 50), p95: pct(list, 95) };
+    }
     return {
+        byAction,
         count: metrics.length,
         totalP50: pct(totals, 50),
         totalP95: pct(totals, 95),
@@ -338,15 +562,17 @@ export function getComputerMetrics() {
 export function resetComputerMetrics() {
     metrics.length = 0;
 }
-export const computerTool = {
+const computerToolBase = {
     name: 'computer',
     description: 'Control the desktop GUI: move/click the mouse, type text, press keys, scroll, and zoom into screen regions. ' +
         'Every action re-captures the screen. If the screen is unchanged since the last screenshot you saw, no image is ' +
         'attached and the result says so — treat the previous screenshot as still current instead of assuming the action failed. ' +
         'Inspect each attached screenshot before the next action and self-correct. ' +
         'Coordinates use a normalized 0-1000 grid (x right, y down) over the primary screen; you never need the physical resolution. ' +
-        'Use zoom on a tight region before clicking small or dense UI elements. Destructive or sensitive targets ' +
-        '(form submit, payment, credentials, delete/send) require explicit user intent — ask first.',
+        'Use zoom on a tight region before clicking small or dense UI elements. ' +
+        'Prefer inspect + click_element/set_value for standard desktop apps (Windows UI Automation): element refs are exact and ' +
+        'cheaper than screenshots. Fall back to coordinates when the tree is sparse or the target is not listed. ' +
+        'Destructive or sensitive targets (form submit, payment, credentials, delete/send) require explicit user intent — ask first.',
     risk: 'dangerous',
     parameters: {
         type: 'object',
@@ -368,7 +594,7 @@ export const computerTool = {
             },
             text: {
                 type: 'string',
-                description: 'Text to type (type) or key combo like "ctrl+s" / "Return" / "ctrl+shift+t" (key).',
+                description: 'Text to type (type), key combo like "ctrl+s" / "Return" / "ctrl+shift+t" (key), or the new value (set_value; may be empty to clear).',
             },
             scroll_direction: { type: 'string', enum: ['up', 'down', 'left', 'right'] },
             scroll_amount: { type: 'integer', description: `Scroll clicks 1-${SCROLL_MAX} (default 3).` },
@@ -385,6 +611,62 @@ export const computerTool = {
                 enum: ['primary', 'all'],
                 description: 'screenshot only: display scope (default primary). All other actions act on the primary screen.',
             },
+            max_nodes: {
+                type: 'integer',
+                description: `inspect only: max elements listed (1-${INSPECT_MAX_NODES}, default ${INSPECT_DEFAULT_NODES}).`,
+            },
+            include_screenshot: {
+                type: 'boolean',
+                description: 'inspect only: also attach a screenshot (default false; the element list is usually enough).',
+            },
+            window: {
+                type: 'string',
+                description: 'Window ref like "w3" from the latest list_windows. focus_window: the window to bring to front. ' +
+                    'inspect / screenshot: scope to that window (works even when it is covered by other windows).',
+            },
+            title_regex: {
+                type: 'string',
+                description: 'Alternative to window: case-insensitive regex matched against window titles (must match exactly one window). ' +
+                    'focus_window / inspect / screenshot.',
+            },
+            move_to_primary: {
+                type: 'boolean',
+                description: 'focus_window only: if the window is on a secondary display or off-screen, move it to the primary screen first ' +
+                    '(default false: a window on another display is only reported, not moved).',
+            },
+            ref: {
+                type: 'string',
+                description: 'click_element/set_value: element ref like "e12" from the latest inspect (refs expire on the next inspect).',
+            },
+            button: {
+                type: 'string',
+                enum: ['left', 'right', 'middle'],
+                description: 'click_element only: mouse button (default left).',
+            },
+            click_count: { type: 'integer', description: 'click_element only: 1-3 (default 1).' },
+            via: {
+                type: 'string',
+                enum: ['mouse', 'pattern', 'keys'],
+                description: 'click_element: "mouse" (default, real click at the live element center) or "pattern" (UIA Invoke/Toggle/Select/Expand, no mouse). ' +
+                    'set_value: "pattern" (ValuePattern, default when supported) or "keys" (focus, select all, type).',
+            },
+            condition: {
+                type: 'object',
+                description: 'wait_until only. { kind: "stable" | "changed" | "element_present" | "element_absent", ' +
+                    'frames?: 2-5 (stable), threshold?: 0-1 frame-diff ratio, interval_ms?: 200-2000, ' +
+                    'ref?: "e12" or selector_text?: \'Button:"OK"\' (element_* only, exactly one) }. ' +
+                    '"stable" = screen stops changing (page/app finished loading); "changed" = screen differs from the last screenshot you saw.',
+            },
+            timeout_ms: {
+                type: 'integer',
+                description: `wait_until only: give up after this many ms (${WAIT_UNTIL_MIN_TIMEOUT_MS}-${WAIT_UNTIL_MAX_TIMEOUT_MS}, default ${WAIT_UNTIL_DEFAULT_TIMEOUT_MS}).`,
+            },
+            method: {
+                type: 'string',
+                enum: ['auto', 'keys', 'paste'],
+                description: 'type only: "keys" (per-character key events), "paste" (clipboard paste, fast and IME-safe; restores text clipboard), ' +
+                    'or "auto" (default: paste for text >200 chars or containing newlines, else keys).',
+            },
         },
         required: ['action'],
         additionalProperties: false,
@@ -429,7 +711,9 @@ export const computerTool = {
             // (抓屏是整条链路最贵的一步,省掉这一步等于把每步的本地开销砍半)。
             let screen;
             let freshPath;
-            if (action === 'zoom' || action === 'screenshot') {
+            // 窗口作用域的 screenshot 走 PrintWindow,不需要整屏位图,只要几何(命中缓存即免抓屏)。
+            const windowScoped = args.window !== undefined || args.title_regex !== undefined;
+            if (action === 'zoom' || (action === 'screenshot' && !windowScoped)) {
                 const t = Date.now();
                 const shot = await capturePrimary(ctx?.signal);
                 m.captureMs += Date.now() - t;
@@ -483,6 +767,91 @@ export const computerTool = {
                     ],
                 };
             }
+            // list_windows:枚举顶层窗口并刷新 wN 快照;不注入输入、不附截图。
+            if (action === 'list_windows') {
+                const t = Date.now();
+                const raw = await getUiaService().windows(ctx?.signal);
+                m.injectMs += Date.now() - t;
+                const { text, items } = formatWindowList(raw, screen);
+                windowSnapshot = items;
+                return { status: 'success', code: 'OK', retryable: false, output: text };
+            }
+            // focus_window:还原 + 置前。副屏窗口默认只报告不搬动(computer 的动作只作用于主屏),move_to_primary 才挪到主屏。
+            if (action === 'focus_window') {
+                const w = await resolveWindow(args, screen, ctx?.signal);
+                const moveToPrimary = args.move_to_primary === true;
+                if (!w.minimized && !w.onPrimary && !moveToPrimary) {
+                    return {
+                        status: 'success',
+                        code: 'OK',
+                        retryable: false,
+                        output: `${windowLabel(w)} is on a secondary display or off-screen, so it was NOT focused or moved ` +
+                            '(computer actions only reach the primary screen). Retry focus_window with move_to_primary: true to bring it onto the primary screen.',
+                    };
+                }
+                const t = Date.now();
+                const res = await getUiaService().focusWindow({
+                    hwnd: w.hwnd,
+                    moveTo: moveToPrimary && !w.onPrimary ? { x: screen.originX + 40, y: screen.originY + 40 } : undefined,
+                }, ctx?.signal);
+                m.injectMs += Date.now() - t;
+                const norm = rectToNorm(res.rect, screen);
+                const cx = res.rect.x + res.rect.w / 2;
+                const cy = res.rect.y + res.rect.h / 2;
+                const nowOnPrimary = cx >= screen.originX &&
+                    cx < screen.originX + screen.physW &&
+                    cy >= screen.originY &&
+                    cy < screen.originY + screen.physH;
+                const notes = [];
+                if (!res.foreground) {
+                    notes.push('The OS refused the foreground switch (focus-stealing protection); the window may still be behind others — retry once or click its title bar.');
+                }
+                if (!nowOnPrimary) {
+                    notes.push('The window is not on the primary screen; retry with move_to_primary: true.');
+                }
+                const t2 = Date.now();
+                const shot = await capturePrimary(ctx?.signal);
+                m.captureMs += Date.now() - t2;
+                syncGeometry(shot);
+                const att = await toAttachment(shot.path, m);
+                commitFrame(att.img);
+                m.bytes = att.bytes;
+                return {
+                    status: 'success',
+                    code: 'OK',
+                    retryable: false,
+                    output: `Focused ${windowLabel(w)}${nowOnPrimary ? ` rect=[${norm.join(', ')}] (normalized 0-1000 over the primary screen)` : ''}. ` +
+                        `Screen re-captured (shown at ${att.dispW}×${att.dispH}).${notes.length ? ` ${notes.join(' ')}` : ''}`,
+                    modelAttachments: [{ type: 'image', name: 'computer-result.png', mime: 'image/png', dataUrl: att.dataUrl }],
+                };
+            }
+            // screenshot + window/title_regex:PrintWindow 截单个窗口(被遮挡也能截),不是整屏基准,不参与差分。
+            if (action === 'screenshot' && windowScoped) {
+                const w = await resolveWindow(args, screen, ctx?.signal);
+                if (w.minimized) {
+                    throw new WindowSelectionError(`${windowLabel(w)} is minimized; call focus_window first`);
+                }
+                const ts = new Date().toISOString().replace(/[:.]/g, '-');
+                const winPath = jailResolve(`.mocode/screenshots/computer-${ts}-window.png`);
+                await mkdir(dirname(winPath), { recursive: true });
+                const t = Date.now();
+                const cap = await getUiaService().captureWindow({ hwnd: w.hwnd, path: winPath }, ctx?.signal);
+                m.captureMs += Date.now() - t;
+                const att = await toAttachment(winPath, m);
+                lastFrame = null;
+                m.bytes = att.bytes;
+                const norm = rectToNorm(cap.rect, screen);
+                return {
+                    status: 'success',
+                    code: 'OK',
+                    retryable: false,
+                    output: `Window screenshot of ${windowLabel(w)} (${cap.method}, ${cap.width}×${cap.height} physical, shown at ${att.dispW}×${att.dispH}). ` +
+                        `On the primary screen this window occupies normalized rect [${norm.join(', ')}] (x, y, w, h): ` +
+                        'the attached image shows ONLY the window, so convert image-local fractions to screen coordinates ' +
+                        '(x = rect.x + fx·rect.w, y = rect.y + fy·rect.h) before clicking. If other windows cover it, focus_window first — clicks hit whatever is on top.',
+                    modelAttachments: [{ type: 'image', name: 'computer-window.png', mime: 'image/png', dataUrl: att.dataUrl }],
+                };
+            }
             // screenshot:只截屏回灌(target 可指定 all)。
             if (action === 'screenshot') {
                 let shotPath = freshPath;
@@ -511,11 +880,144 @@ export const computerTool = {
                     ],
                 };
             }
+            // inspect:读前台窗口 UIA 元素树,文本回灌;不注入输入,默认不附截图(元素清单本身就是感知结果)。
+            if (action === 'inspect') {
+                // window/title_regex:按 hwnd 读指定窗口(可被遮挡);省略 = 前台窗口。
+                let inspectHwnd;
+                if (windowScoped) {
+                    const w = await resolveWindow(args, screen, ctx?.signal);
+                    if (w.minimized) {
+                        throw new WindowSelectionError(`${windowLabel(w)} is minimized; call focus_window first`);
+                    }
+                    inspectHwnd = w.hwnd;
+                }
+                const t = Date.now();
+                const tree = await getUiaService().tree(inspectHwnd !== undefined ? { hwnd: inspectHwnd } : {}, ctx?.signal);
+                m.injectMs += Date.now() - t;
+                const maxNodes = args.max_nodes ?? INSPECT_DEFAULT_NODES;
+                const { items, truncated } = selectDisplayNodes(tree, maxNodes);
+                snapshotGeneration += 1;
+                elementSnapshot = {
+                    generation: snapshotGeneration,
+                    tree,
+                    items: new Map(items.map((it) => [it.ref, it])),
+                };
+                setElementRefNames(items);
+                const text = formatSnapshot(tree, items, {
+                    generation: snapshotGeneration,
+                    truncated,
+                    maxNodes,
+                    geometry: screen,
+                });
+                if (args.include_screenshot === true) {
+                    const t2 = Date.now();
+                    const shot = await capturePrimary(ctx?.signal);
+                    m.captureMs += Date.now() - t2;
+                    syncGeometry(shot);
+                    const att = await toAttachment(shot.path, m);
+                    commitFrame(att.img);
+                    m.bytes = att.bytes;
+                    return {
+                        status: 'success',
+                        code: 'OK',
+                        retryable: false,
+                        output: text,
+                        modelAttachments: [
+                            { type: 'image', name: 'computer-inspect.png', mime: 'image/png', dataUrl: att.dataUrl },
+                        ],
+                    };
+                }
+                return { status: 'success', code: 'OK', retryable: false, output: text };
+            }
+            // wait_until:轮询直到画面稳定/变化或元素出现/消失;不注入输入(design-notes/computer-use-rpa.md §3.1)。
+            // 轮询帧只在内存里比对、用完即删文件;仅结束时抓一张正式截图回灌并设为新的差分基准。
+            if (action === 'wait_until') {
+                const parsed = parseWaitCondition(args.condition, diffThreshold());
+                if (!parsed.ok)
+                    throw new Error(parsed.error); // validate 已拦过,这里只为收窄类型
+                const cond = parsed.condition;
+                const timeoutMs = args.timeout_ms ?? WAIT_UNTIL_DEFAULT_TIMEOUT_MS;
+                const grab = async () => {
+                    const t0 = Date.now();
+                    const shot = await capturePrimary(ctx?.signal);
+                    m.captureMs += Date.now() - t0;
+                    try {
+                        return downscale(decodePng(await readFile(shot.path)), maxEdge()).img;
+                    }
+                    finally {
+                        await rm(shot.path, { force: true }).catch(() => undefined);
+                    }
+                };
+                // 元素存在性:在前台窗口当前 UIA 树里按 role+name 找(ref 取自最近一次 inspect 快照)。
+                const elementExists = async (target) => {
+                    let role;
+                    let name;
+                    if ('ref' in target) {
+                        const known = lookupElement(target.ref).item.node;
+                        role = known.role;
+                        name = known.name;
+                    }
+                    else {
+                        role = target.step.role;
+                        name = target.step.name;
+                    }
+                    const tree = await getUiaService().tree({}, ctx?.signal);
+                    return tree.nodes.some((n) => n.id !== 0 && (role === undefined || n.role === role) && (name === undefined || n.name === name));
+                };
+                const res = await waitUntil(cond, timeoutMs, {
+                    captureFrame: grab,
+                    elementExists,
+                    baseline: lastFrame,
+                    signal: ctx?.signal,
+                });
+                if (res.outcome === 'aborted') {
+                    return { status: 'aborted', code: 'ABORTED', retryable: false, output: 'wait_until aborted' };
+                }
+                const timedOut = res.outcome === 'timeout';
+                const desc = describeCondition(cond);
+                const stats = `${res.elapsedMs}ms, ${res.polls} polls` +
+                    (res.lastDiff !== undefined ? `, last frame diff ${(res.lastDiff * 100).toFixed(2)}%` : '');
+                // 画面类条件、以及任何超时,都回灌一张最终截图;元素类条件满足时纯文本即可。
+                const wantShot = timedOut || cond.kind === 'stable' || cond.kind === 'changed';
+                let attachments;
+                if (wantShot) {
+                    const t2 = Date.now();
+                    const shot = await capturePrimary(ctx?.signal);
+                    m.captureMs += Date.now() - t2;
+                    syncGeometry(shot);
+                    const att = await toAttachment(shot.path, m);
+                    commitFrame(att.img);
+                    m.bytes = att.bytes;
+                    m.diff = att.diff;
+                    attachments = [{ type: 'image', name: 'computer-wait.png', mime: 'image/png', dataUrl: att.dataUrl }];
+                }
+                const shotNote = wantShot
+                    ? ` Screen re-captured (primary screen ${screen.physW}×${screen.physH} physical): inspect the attached screenshot.`
+                    : '';
+                if (timedOut) {
+                    // 超时不自动重试(TIMEOUT 契约):已烧掉整个等待窗口,由模型决定改条件/加长超时/换思路。
+                    return {
+                        status: 'error',
+                        code: 'TIMEOUT',
+                        retryable: false,
+                        output: `waited until ${desc}: timed out (${stats}). The condition did not hold within ${timeoutMs}ms.` +
+                            `${shotNote} Check the screenshot, then adjust the condition, raise timeout_ms, or take a different approach.`,
+                        modelAttachments: attachments,
+                    };
+                }
+                return {
+                    status: 'success',
+                    code: 'OK',
+                    retryable: false,
+                    output: `waited until ${desc}: met (${stats}).${shotNote}`,
+                    modelAttachments: attachments,
+                };
+            }
             // 输入动作:注入 → 重截屏回灌(wait 也回灌,等待后界面往往已变化)。
             if (action === 'left_mouse_down')
                 pendingMouseDown = true;
             let t = Date.now();
-            const summary = await executeAction(injector, action, args, screen);
+            const summary = await executeAction(injector, action, args, screen, ctx?.signal);
             m.injectMs += Date.now() - t;
             if (action === 'left_mouse_up')
                 pendingMouseDown = false;
@@ -558,6 +1060,25 @@ export const computerTool = {
             if (ctx?.signal?.aborted || /aborted/i.test(message)) {
                 return { status: 'aborted', code: 'ABORTED', retryable: false, output: message };
             }
+            if (error instanceof ElementNotFoundError) {
+                // ToolOutcomeCode 无 ELEMENT_NOT_FOUND;用 POSTCONDITION_FAILED 表达「前置定位不成立」,
+                // 输出前缀固定,供台账识别。不自动重试:界面已变,必须让模型重新 inspect。
+                return {
+                    status: 'error',
+                    code: 'POSTCONDITION_FAILED',
+                    retryable: false,
+                    output: `computer action failed: ELEMENT_NOT_FOUND: ${message}`,
+                };
+            }
+            if (error instanceof WindowSelectionError || error instanceof UiaWindowGoneError) {
+                // 窗口选不到 / 已关闭:同样是「前置定位不成立」,不自动重试,让模型重新 list_windows。
+                return {
+                    status: 'error',
+                    code: 'POSTCONDITION_FAILED',
+                    retryable: false,
+                    output: `computer action failed: WINDOW_NOT_FOUND: ${message}`,
+                };
+            }
             return {
                 status: 'error',
                 code: 'EXECUTION_ERROR',
@@ -579,3 +1100,119 @@ export const computerTool = {
         }
     },
 };
+/**
+ * 构造 trace 条目的上下文:必须在执行前读取模块级快照(执行后 inspect/list_windows 可能已替换)。
+ * 任何异常都按"不录制"处理,绝不影响动作本身。
+ */
+function buildTraceForCall(args) {
+    const ctx = {
+        seq: nextTraceSeq(),
+        geometry: geometryCache ?? undefined,
+        lookupRef: lookupElementRefName,
+    };
+    if ((args.action === 'click_element' || args.action === 'set_value') && typeof args.ref === 'string') {
+        const item = elementSnapshot?.items.get(args.ref);
+        if (elementSnapshot && item)
+            ctx.element = { tree: elementSnapshot.tree, nodeId: item.node.id };
+    }
+    if (args.action === 'focus_window' && typeof args.window === 'string') {
+        ctx.windowProcess = windowSnapshot.find((w) => w.ref === args.window)?.processName || undefined;
+    }
+    return buildTraceEntry(args, ctx);
+}
+/**
+ * 对外导出的 computer 工具 = 基础工具 + 结构化录制(design-notes/computer-use-rpa.md §5.1)。
+ * 仅在动作成功后追加一行 gui-trace.jsonl;回放期间(withTraceSuspended)与 MOCODE_FLOW_TRACE=false 时不录。
+ */
+export const computerTool = {
+    ...computerToolBase,
+    async execute(args, ctx) {
+        let entry = null;
+        if (traceEnabled() && !isTraceSuspended() && typeof args.action === 'string' && isRecordedAction(args.action)) {
+            try {
+                entry = buildTraceForCall(args);
+            }
+            catch {
+                entry = null;
+            }
+        }
+        const result = await computerToolBase.execute(args, ctx);
+        if (entry && typeof result !== 'string' && result.status === 'success') {
+            entry.changed = outputIndicatesChange(entry.action, result.output);
+            appendTrace(entry);
+        }
+        return result;
+    },
+};
+// ── flow 回放接入点(src/flows/runner.ts 的 RunnerDeps 由 run_flow 工具接到这里) ─────────────
+/**
+ * 把录制的 selector 现场解析成当前快照里的 ref:按 processName(+titleRegex) 选窗 → 读该窗口 UIA 树 →
+ * matchSelector(歧义即失败)→ 目标必须在展示集中(与 inspect 同口径),并把整棵展示集登记为新快照,
+ * 这样随后的 click_element/set_value 带 ref 走原有"执行前按 hwnd+path 重定位并校验 role/name"路径。
+ */
+export async function resolveSelectorToRef(selector, signal) {
+    try {
+        const raw = await getUiaService().windows(signal);
+        const picked = pickWindowForReplay(raw, {
+            processName: selector.window.processName,
+            titleRegex: selector.window.titleRegex,
+        });
+        if (!picked.ok)
+            return { ok: false, error: picked.error };
+        if (picked.window.minimized) {
+            return {
+                ok: false,
+                error: `window ${JSON.stringify(picked.window.title.slice(0, 40))} is minimized; a focus_window step must run first`,
+            };
+        }
+        const tree = await getUiaService().tree({ hwnd: picked.window.hwnd }, signal);
+        const node = matchSelector(tree, selector);
+        if (!node) {
+            return { ok: false, error: 'no unique element matches the recorded selector in the current window' };
+        }
+        const { items } = selectDisplayNodes(tree, tree.nodes.length);
+        const hit = items.find((it) => it.node.id === node.id);
+        if (!hit) {
+            return {
+                ok: false,
+                error: `the matched ${node.role} ${JSON.stringify(node.name.slice(0, 40))} is not an interactive, visible element right now`,
+            };
+        }
+        snapshotGeneration += 1;
+        elementSnapshot = {
+            generation: snapshotGeneration,
+            tree,
+            items: new Map(items.map((it) => [it.ref, it])),
+        };
+        setElementRefNames(items);
+        return { ok: true, ref: hit.ref };
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        return { ok: false, error: error instanceof Error ? error.message : String(error) };
+    }
+}
+/**
+ * focus_window 回放:按 processName(+titleRegex) 现场选窗,刷新 wN 快照并返回 computer 能接受的 {window:'wN'}。
+ */
+export async function resolveWindowForReplay(win, signal) {
+    try {
+        const raw = await getUiaService().windows(signal);
+        const picked = pickWindowForReplay(raw, win);
+        if (!picked.ok)
+            return { ok: false, error: picked.error };
+        const { screen } = await resolveScreen(signal);
+        const { items } = formatWindowList(raw, screen);
+        const item = items.find((i) => i.hwnd === picked.window.hwnd);
+        if (!item)
+            return { ok: false, error: 'the matching window is not in the visible window list' };
+        windowSnapshot = items;
+        return { ok: true, args: { window: item.ref } };
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        return { ok: false, error: error instanceof Error ? error.message : String(error) };
+    }
+}