mocode-ai 1.7.2 → 1.7.5

package/dist/runtime/input-injector.js

@@ -137,42 +137,38 @@ const PS_INJECTOR_SCRIPT = [
     '  [StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; }',
     '  [StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public UIntPtr dwExtraInfo; }',
     '  [StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public UIntPtr dwExtraInfo; }',
+    '  public static void Key(ushort vk, ushort scan, uint flags) {',
+    '    INPUT[] a = new INPUT[1];',
+    '    a[0].type = 1;',
+    '    a[0].U.ki.wVk = vk; a[0].U.ki.wScan = scan; a[0].U.ki.dwFlags = flags;',
+    '    SendInput(1, a, Marshal.SizeOf(typeof(INPUT)));',
+    '  }',
     '}',
     '"@',
     // DPI 感知:SetCursorPos/GetCursorPos 始终按物理像素解释,而进程默认 DPI UNAWARE。
     // 声明 PER_MONITOR_AWARE_V2(失败回落 SYSTEM_AWARE)后 GetCursorPos 与调用方的物理坐标口径一致,
     // 避免 cursor_position 读回的坐标与主屏物理分辨率不同源。
     'try { if (-not [Inject]::SetProcessDpiAwarenessContext([IntPtr](-4))) { [void][Inject]::SetProcessDPIAware() } } catch { }',
-    'function Send-VkCombo([int[]]$vks) {',
-    '  $size = [Runtime.InteropServices.Marshal]::SizeOf([type]"Inject+INPUT")',
-    '  foreach ($vk in $vks) {',
-    '    $inp = New-Object "Inject+INPUT"',
-    '    $inp.type = 1',
-    '    $inp.U.ki.wVk = $vk',
-    '    $inp.U.ki.dwFlags = 0',
-    '    [Inject]::SendInput(1, [Inject+INPUT[]]@($inp), $size) | Out-Null',
+    // 剪贴板 API(paste op 用)要求 STA 线程,进程以 -STA 启动;剪贴板偶发被其它进程占用,重试几次。
+    'Add-Type -AssemblyName System.Windows.Forms',
+    'function Set-ClipText([string]$s) {',
+    '  for ($i = 0; $i -lt 5; $i++) {',
+    '    try { [System.Windows.Forms.Clipboard]::SetText($s); return } catch { Start-Sleep -Milliseconds 50 }',
     '  }',
+    '  throw "clipboard is busy"',
+    '}',
+    // 键盘事件的 INPUT 必须在 C# 里构造:PowerShell 对嵌套值类型(`$inp.U.ki.wVk = ...`)的赋值作用在副本上,
+    // 结果是发出全零的空事件(实测 wVk 读回 0)。Inject.Key 在 C# 内就地填充后调用 SendInput。
+    'function Send-VkCombo([int[]]$vks) {',
+    '  foreach ($vk in $vks) { [Inject]::Key([uint16]$vk, [uint16]0, [uint32]0) }',
     '  [array]::Reverse($vks)',
-    '  foreach ($vk in $vks) {',
-    '    $inp = New-Object "Inject+INPUT"',
-    '    $inp.type = 1',
-    '    $inp.U.ki.wVk = $vk',
-    '    $inp.U.ki.dwFlags = 2',
-    '    [Inject]::SendInput(1, [Inject+INPUT[]]@($inp), $size) | Out-Null',
-    '  }',
+    '  foreach ($vk in $vks) { [Inject]::Key([uint16]$vk, [uint16]0, [uint32]2) }',
     '}',
     'function Send-UnicodeText([string]$text) {',
-    '  $size = [Runtime.InteropServices.Marshal]::SizeOf([type]"Inject+INPUT")',
     '  for ($i = 0; $i -lt $text.Length; $i++) {',
-    '    $code = [int][char]$text[$i]',
-    '    foreach ($flag in @(4, 6)) {',
-    '      $inp = New-Object "Inject+INPUT"',
-    '      $inp.type = 1',
-    '      $inp.U.ki.wVk = 0',
-    '      $inp.U.ki.wScan = $code',
-    '      $inp.U.ki.dwFlags = $flag',
-    '      [Inject]::SendInput(1, [Inject+INPUT[]]@($inp), $size) | Out-Null',
-    '    }',
+    '    $code = [uint16][char]$text[$i]',
+    '    [Inject]::Key([uint16]0, $code, [uint32]4)',
+    '    [Inject]::Key([uint16]0, $code, [uint32]6)',
     '  }',
     '}',
     'while ($null -ne ($line = [Console]::In.ReadLine())) {',
@@ -201,6 +197,19 @@ const PS_INJECTOR_SCRIPT = [
     '        [Inject]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero); $resp.ok = $true',
     '      }',
     '      "type" { Send-UnicodeText([string]$req.text); $resp.ok = $true }',
+    '      "paste" {',
+    // 剪贴板粘贴:保存文本剪贴板 → 写入 → ctrl+v → 等目标应用读完 → 还原。
+    // 原剪贴板是图片/文件等非文本时无法还原,如实上报 overwritten,不假装还原。
+    '        $hadText = [System.Windows.Forms.Clipboard]::ContainsText()',
+    '        $old = if ($hadText) { [System.Windows.Forms.Clipboard]::GetText() } else { $null }',
+    '        $nonText = (-not $hadText) -and ([System.Windows.Forms.Clipboard]::ContainsImage() -or [System.Windows.Forms.Clipboard]::ContainsFileDropList() -or [System.Windows.Forms.Clipboard]::ContainsAudio())',
+    '        Set-ClipText ([string]$req.text)',
+    '        Send-VkCombo([int[]]@(17, 86))',
+    '        Start-Sleep -Milliseconds 150',
+    '        if ($hadText) { Set-ClipText $old } elseif (-not $nonText) { [System.Windows.Forms.Clipboard]::Clear() }',
+    '        $resp.overwritten = [bool]$nonText',
+    '        $resp.ok = $true',
+    '      }',
     '      "key" { Send-VkCombo([int[]]$req.vks); $resp.ok = $true }',
     '      "scroll" {',
     '        $amt = [int]$req.amount * 120',
@@ -244,7 +253,7 @@ class PowerShellInjector {
     ensureProcess() {
         if (this.child && this.child.exitCode === null && !this.child.killed)
             return this.child;
-        this.child = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', PS_INJECTOR_SCRIPT], { env: filterEnv(process.env), windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] });
+        this.child = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-STA', '-ExecutionPolicy', 'Bypass', '-Command', PS_INJECTOR_SCRIPT], { env: filterEnv(process.env), windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] });
         this.stdoutBuf = '';
         this.lastError = '';
         this.child.stdout.on('data', (chunk) => this.onStdout(chunk.toString('utf8')));
@@ -355,6 +364,10 @@ class PowerShellInjector {
     async typeText(text) {
         await this.call('type', { text }, this.signal);
     }
+    async paste(text) {
+        const resp = await this.call('paste', { text }, this.signal);
+        return { clipboardOverwritten: resp.overwritten === true };
+    }
     async pressKey(combo) {
         const vks = mapKeyComboToVk(combo);
         await this.call('key', { vks }, this.signal);

package/dist/runtime/shutdown.js

@@ -14,12 +14,14 @@ export async function shutdownRuntime() {
     try {
         // 动态 import:未用过 computer 工具时不该把这两个常驻进程拉进来。
         // 常驻 PowerShell 进程不随主进程自动退出,必须显式回收(否则每次会话留一组孤儿)。
-        const [{ disposeInputInjector }, { disposeCaptureService }] = await Promise.all([
+        const [{ disposeInputInjector }, { disposeCaptureService }, { disposeUiaService }] = await Promise.all([
             import('./input-injector.js'),
             import('./screen-capture-service.js'),
+            import('./uia-service.js'),
         ]);
         await disposeInputInjector();
         await disposeCaptureService();
+        await disposeUiaService();
     }
     catch {
         // 从未启动过:忽略。

package/dist/runtime/uia-selector.js

@@ -0,0 +1,324 @@
+/**
+ * UIA 元素树的纯函数层(design-notes/computer-use-rpa.md §2.2–2.4)。
+ *
+ * 只处理 JSON 化的元素树:解析/规整 PowerShell 侧输出、选择器生成与匹配、inspect 文本渲染、
+ * ref → 名称登记表(供权限层展示与敏感判定)。不碰进程、屏幕或 COM,跨平台可单测。
+ *
+ * 树形约定:nodes[0] 是窗口根元素,id 即数组下标,parent=-1 表示根;path 是从窗口根出发
+ * 按 ControlViewWalker 子序号走下来的路径(如 "0.3.1"),供 PowerShell 侧重新定位元素。
+ */
+import { physicalToNorm } from './screen-pipeline.js';
+// ── 解析 PowerShell 输出 ────────────────────────────────────────────────
+const str = (v) => (typeof v === 'string' ? v : v == null ? '' : String(v));
+const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
+function toRect(v) {
+    const o = (v && typeof v === 'object' ? v : {});
+    return {
+        x: Math.round(num(o.x)),
+        y: Math.round(num(o.y)),
+        w: Math.max(0, Math.round(num(o.w))),
+        h: Math.max(0, Math.round(num(o.h))),
+    };
+}
+/** PowerShell 5.1 的 ConvertTo-Json 会把单元素数组压成标量,这里统一还原成数组。 */
+function toArray(v) {
+    if (Array.isArray(v))
+        return v;
+    if (v === undefined || v === null || v === '')
+        return [];
+    return [v];
+}
+/**
+ * 把 PowerShell `tree` op 的响应规整为 UiaRawTree。缺字段按保守默认补齐;
+ * 节点 id 以数组下标为准,parent 越界视为根的子节点。没有任何节点时抛错。
+ */
+export function parseRawTree(resp) {
+    const w = (resp.window && typeof resp.window === 'object' ? resp.window : {});
+    const rawNodes = toArray(resp.nodes);
+    if (rawNodes.length === 0)
+        throw new Error('UIA returned an empty element tree');
+    const nodes = rawNodes.map((item, i) => {
+        const o = (item && typeof item === 'object' ? item : {});
+        const parent = i === 0 ? -1 : Math.round(num(o.parent));
+        const isPassword = o.isPassword === true;
+        const node = {
+            id: i,
+            parent: i === 0 ? -1 : parent >= 0 && parent < i ? parent : 0,
+            path: str(o.path),
+            depth: Math.max(0, Math.round(num(o.depth))),
+            role: str(o.role) || 'Custom',
+            name: str(o.name),
+            automationId: str(o.automationId),
+            className: str(o.className),
+            rect: toRect(o.rect),
+            enabled: o.enabled !== false,
+            offscreen: o.offscreen === true,
+            isPassword,
+            patterns: toArray(o.patterns).map(str).filter(Boolean),
+        };
+        // 双保险:即便 PowerShell 侧漏判,密码节点的 value 也绝不进入 TS 侧结构。
+        if (!isPassword && typeof o.value === 'string' && o.value.length > 0)
+            node.value = o.value;
+        return node;
+    });
+    return {
+        window: {
+            hwnd: Math.round(num(w.hwnd)),
+            title: str(w.title),
+            processName: str(w.processName),
+            pid: Math.round(num(w.pid)),
+            rect: toRect(w.rect),
+        },
+        nodes,
+        truncated: resp.truncated === true,
+        elapsedMs: Math.round(num(resp.elapsedMs)),
+    };
+}
+// ── 可交互判定 ──────────────────────────────────────────────────────────
+const INTERACTIVE_ROLES = new Set([
+    'Button',
+    'Edit',
+    'ComboBox',
+    'CheckBox',
+    'RadioButton',
+    'MenuItem',
+    'TabItem',
+    'ListItem',
+    'TreeItem',
+    'Hyperlink',
+    'Document',
+    'SplitButton',
+    'DataItem',
+    'Slider',
+    'Spinner',
+]);
+const ACTIONABLE_PATTERNS = new Set(['invoke', 'value', 'toggle', 'select', 'expand']);
+export function isInteractive(n) {
+    return INTERACTIVE_ROLES.has(n.role) || n.patterns.some((p) => ACTIONABLE_PATTERNS.has(p));
+}
+/** 自增数字 id / GUID 形态的 AutomationId 在重启或重排后会变,不能作为选择器锚点。 */
+export function isStableAutomationId(id) {
+    const s = id.trim();
+    if (!s)
+        return false;
+    if (/^\d+$/.test(s))
+        return false;
+    if (/^[{(]?[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}[})]?$/i.test(s))
+        return false;
+    return true;
+}
+// ── 选择器匹配 ──────────────────────────────────────────────────────────
+function childrenMap(tree) {
+    const kids = new Map();
+    for (const n of tree.nodes) {
+        if (n.parent < 0)
+            continue;
+        const list = kids.get(n.parent);
+        if (list)
+            list.push(n.id);
+        else
+            kids.set(n.parent, [n.id]);
+    }
+    return kids;
+}
+/** ctx 的全部后代,文档序(先序 DFS)。 */
+function descendants(tree, kids, ctx) {
+    const out = [];
+    const stack = [...(kids.get(ctx) ?? [])].reverse();
+    while (stack.length) {
+        const id = stack.pop();
+        out.push(tree.nodes[id]);
+        const ch = kids.get(id);
+        if (ch)
+            for (let i = ch.length - 1; i >= 0; i--)
+                stack.push(ch[i]);
+    }
+    return out;
+}
+function stepMatches(n, step) {
+    if (step.role !== undefined && n.role !== step.role)
+        return false;
+    if (step.automationId !== undefined && n.automationId !== step.automationId)
+        return false;
+    if (step.className !== undefined && n.className !== step.className)
+        return false;
+    if (step.name !== undefined && n.name !== step.name)
+        return false;
+    if (step.nameRegex !== undefined) {
+        try {
+            if (!new RegExp(step.nameRegex).test(n.name))
+                return false;
+        }
+        catch {
+            return false;
+        }
+    }
+    return true;
+}
+function windowMatches(w, sel) {
+    if (sel.processName && w.processName.toLowerCase() !== sel.processName.toLowerCase())
+        return false;
+    if (sel.titleRegex) {
+        try {
+            if (!new RegExp(sel.titleRegex).test(w.title))
+                return false;
+        }
+        catch {
+            return false;
+        }
+    }
+    return true;
+}
+/**
+ * 在树中按选择器定位元素。每一步在上一步命中节点的**全部后代**里找(不要求直接子节点);
+ * 有 index 取第 index 个候选,无 index 时必须唯一——多个候选视为歧义返回 null,宁可失败也不点错。
+ */
+export function matchSelector(tree, sel) {
+    if (!tree.nodes.length || !sel.path.length)
+        return null;
+    if (!windowMatches(tree.window, sel.window))
+        return null;
+    const kids = childrenMap(tree);
+    let ctx = 0;
+    for (const step of sel.path) {
+        const cands = descendants(tree, kids, ctx).filter((n) => stepMatches(n, step));
+        const pick = step.index !== undefined ? cands[step.index] : cands.length === 1 ? cands[0] : undefined;
+        if (!pick)
+            return null;
+        ctx = pick.id;
+    }
+    return tree.nodes[ctx];
+}
+// ── 选择器生成 ──────────────────────────────────────────────────────────
+function baseStep(n) {
+    if (isStableAutomationId(n.automationId))
+        return { role: n.role, automationId: n.automationId };
+    return { role: n.role, name: n.name };
+}
+/** 对给定祖先链逐步生成 step,在不唯一处补 index。链中节点必须按深度递增且互为祖孙。 */
+function stepsForChain(tree, kids, chain) {
+    const steps = [];
+    let ctx = 0;
+    for (const n of chain) {
+        const step = baseStep(n);
+        const cands = descendants(tree, kids, ctx).filter((c) => stepMatches(c, step));
+        const idx = cands.findIndex((c) => c.id === n.id);
+        if (idx < 0)
+            return null;
+        if (cands.length > 1)
+            step.index = idx;
+        steps.push(step);
+        ctx = n.id;
+    }
+    return steps;
+}
+const indexCount = (steps) => steps.filter((s) => s.index !== undefined).length;
+/**
+ * 为 nodeId 生成选择器:先写出从窗口根到目标的完整祖先链,再贪心省略对定位无区分度的祖先
+ * (省略后仍能唯一命中目标、且 index 数不增加才接受)——路径越短,对布局变化越宽容。
+ * 不在热路径上调用:inspect 只登记 ref,真正点击时才按需生成。
+ */
+export function buildSelector(tree, nodeId) {
+    const target = tree.nodes[nodeId];
+    if (!target || nodeId === 0)
+        return null;
+    let chain = [];
+    for (let cur = target; cur && cur.id !== 0; cur = tree.nodes[cur.parent]) {
+        chain.unshift(cur);
+        if (cur.parent < 0)
+            break;
+    }
+    const kids = childrenMap(tree);
+    const window = { processName: tree.window.processName };
+    let steps = stepsForChain(tree, kids, chain);
+    if (!steps)
+        return null;
+    let i = 0;
+    while (i < chain.length - 1) {
+        const trialChain = chain.filter((_, k) => k !== i);
+        const trial = stepsForChain(tree, kids, trialChain);
+        if (trial &&
+            indexCount(trial) <= indexCount(steps) &&
+            matchSelector(tree, { window, path: trial })?.id === nodeId) {
+            chain = trialChain;
+            steps = trial;
+        }
+        else {
+            i++;
+        }
+    }
+    return { window, path: steps };
+}
+/** 选出给模型看的节点:可交互、可见、矩形非空;按文档序编号 e1..eN。 */
+export function selectDisplayNodes(tree, maxNodes) {
+    const visible = tree.nodes.slice(1).filter((n) => isInteractive(n) && !n.offscreen && n.rect.w > 0 && n.rect.h > 0);
+    const items = visible.slice(0, maxNodes).map((node, i) => ({ ref: `e${i + 1}`, node }));
+    return { items, truncated: visible.length > maxNodes };
+}
+/** 物理矩形 → 主屏 norm1000 [x, y, w, h]。 */
+export function rectToNorm(rect, g) {
+    const [x, y] = physicalToNorm(rect.x - g.originX, rect.y - g.originY, g.physW, g.physH);
+    const [x1, y1] = physicalToNorm(rect.x + rect.w - g.originX, rect.y + rect.h - g.originY, g.physW, g.physH);
+    return [x, y, Math.max(0, x1 - x), Math.max(0, y1 - y)];
+}
+/** 矩形中心(物理像素,虚拟桌面坐标)——click_element 的落点。 */
+export function rectCenter(rect) {
+    return [Math.round(rect.x + rect.w / 2), Math.round(rect.y + rect.h / 2)];
+}
+const BROWSER_PROCESSES = new Set(['chrome', 'msedge', 'firefox', 'brave', 'opera', 'vivaldi']);
+function clip(s, max) {
+    const flat = s.replace(/\s+/g, ' ').trim();
+    return flat.length > max ? `${flat.slice(0, max)}…` : flat;
+}
+/** 节点数低于此值视为稀疏树(游戏、自绘 UI、未开无障碍的 Electron)。 */
+export const SPARSE_TREE_THRESHOLD = 3;
+/** inspect 输出前缀;session/gui-actions.ts 靠它识别台账条目,改动须同步。 */
+export const INSPECT_OUTPUT_PREFIX = 'UI elements of window ';
+export function formatSnapshot(tree, items, opts) {
+    const w = tree.window;
+    const proc = w.processName ? `${w.processName}.exe` : 'unknown process';
+    const lines = [
+        `${INSPECT_OUTPUT_PREFIX}${JSON.stringify(clip(w.title, 80))} (${proc}), generation ${opts.generation}, ` +
+            `${items.length} nodes (interactive only).`,
+        'Coordinates are normalized 0-1000 [x, y, w, h] over the primary screen. Use click_element/set_value with ref; ' +
+            'refs expire on the next inspect.',
+    ];
+    const minDepth = items.reduce((m, it) => Math.min(m, it.node.depth), Number.POSITIVE_INFINITY);
+    for (const { ref, node } of items) {
+        const indent = ' '.repeat(Math.min(6, Math.max(0, node.depth - minDepth)));
+        const value = node.value !== undefined ? ` value=${JSON.stringify(clip(node.value, 60))}` : '';
+        const rect = rectToNorm(node.rect, opts.geometry);
+        const flags = [...node.patterns.filter((p) => ACTIONABLE_PATTERNS.has(p))];
+        if (!node.enabled)
+            flags.push('disabled');
+        if (node.isPassword)
+            flags.push('password');
+        lines.push(`${indent}${ref}  ${node.role}  ${JSON.stringify(clip(node.name, 80))}${value}  [${rect.join(', ')}]` +
+            (flags.length ? `  ${flags.join(' ')}` : ''));
+    }
+    if (opts.truncated)
+        lines.push(`(truncated at ${opts.maxNodes} nodes — narrow scope or use screenshot)`);
+    if (tree.truncated)
+        lines.push('(element tree walk hit its size/time limit; some elements may be missing)');
+    if (items.length < SPARSE_TREE_THRESHOLD) {
+        lines.push('UIA tree is sparse for this window; fall back to screenshot + coordinates.');
+    }
+    if (BROWSER_PROCESSES.has(w.processName.toLowerCase())) {
+        lines.push('This is a web browser: for page content prefer the browser tool over UI elements.');
+    }
+    return lines.join('\n');
+}
+// ── ref 名称登记表(权限层只读)──────────────────────────────────────────
+const refNames = new Map();
+/** inspect 后整体替换:旧 generation 的 ref 立即失效。 */
+export function setElementRefNames(items) {
+    refNames.clear();
+    for (const { ref, node } of items)
+        refNames.set(ref, { role: node.role, name: node.name });
+}
+export function lookupElementRefName(ref) {
+    return refNames.get(ref);
+}
+export function clearElementRefNames() {
+    refNames.clear();
+}