mocode-ai 1.7.2 → 1.7.5

package/dist/runtime/wait-until.js

@@ -0,0 +1,199 @@
+/**
+ * computer.wait_until 的纯逻辑层(design-notes/computer-use-rpa.md §3.1)。
+ *
+ * 条件解析、轮询状态机、type 输入方式选择都在这里;抓屏与元素存在性检查通过 WaitDeps 注入,
+ * 因此可以用合成 PngImage 序列 + 假时钟离线单测,不碰屏幕/进程。
+ */
+import { diffRatio } from './screen-pipeline.js';
+export const WAIT_UNTIL_DEFAULT_TIMEOUT_MS = 10000;
+export const WAIT_UNTIL_MIN_TIMEOUT_MS = 500;
+export const WAIT_UNTIL_MAX_TIMEOUT_MS = 30000;
+export const WAIT_INTERVAL_MIN_MS = 200;
+export const WAIT_INTERVAL_MAX_MS = 2000;
+export const WAIT_INTERVAL_DEFAULT_MS = 400;
+export const STABLE_FRAMES_MIN = 2;
+export const STABLE_FRAMES_MAX = 5;
+const KINDS = ['stable', 'changed', 'element_present', 'element_absent'];
+/**
+ * 解析 selector_text 简写:`Button:"确定"` 或仅 `"确定"`(不限 role)。
+ * 引号内按 JSON 字符串转义解析(可写 \" ),失败则取原文。
+ */
+export function parseSelectorText(text) {
+    const m = /^\s*(?:([A-Za-z][A-Za-z0-9]*)\s*:\s*)?("(?:[^"\\]|\\.)*")\s*$/s.exec(text);
+    if (!m)
+        return null;
+    let name;
+    try {
+        name = JSON.parse(m[2]);
+    }
+    catch {
+        name = m[2].slice(1, -1);
+    }
+    if (!name)
+        return null;
+    const step = { name };
+    if (m[1])
+        step.role = m[1];
+    return step;
+}
+function inRange(v, min, max) {
+    return typeof v === 'number' && Number.isInteger(v) && v >= min && v <= max;
+}
+/** 校验并规整 wait_until 的 condition 参数;defaultThreshold 为 computer.ts 的画面差分阈值。 */
+export function parseWaitCondition(raw, defaultThreshold) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        return { ok: false, error: `action "wait_until" requires condition: { kind: ${KINDS.join('|')}, ... }` };
+    }
+    const c = raw;
+    const kind = c.kind;
+    if (typeof kind !== 'string' || !KINDS.includes(kind)) {
+        return { ok: false, error: `condition.kind must be one of: ${KINDS.join(', ')}` };
+    }
+    let intervalMs = WAIT_INTERVAL_DEFAULT_MS;
+    if (c.interval_ms !== undefined) {
+        if (!inRange(c.interval_ms, WAIT_INTERVAL_MIN_MS, WAIT_INTERVAL_MAX_MS)) {
+            return {
+                ok: false,
+                error: `condition.interval_ms must be an integer in ${WAIT_INTERVAL_MIN_MS}-${WAIT_INTERVAL_MAX_MS}`,
+            };
+        }
+        intervalMs = c.interval_ms;
+    }
+    let threshold = defaultThreshold;
+    if (c.threshold !== undefined) {
+        if (typeof c.threshold !== 'number' || !Number.isFinite(c.threshold) || c.threshold < 0 || c.threshold > 1) {
+            return { ok: false, error: 'condition.threshold must be a number in 0-1' };
+        }
+        threshold = c.threshold;
+    }
+    if (kind === 'stable') {
+        let frames = STABLE_FRAMES_MIN;
+        if (c.frames !== undefined) {
+            if (!inRange(c.frames, STABLE_FRAMES_MIN, STABLE_FRAMES_MAX)) {
+                return { ok: false, error: `condition.frames must be an integer in ${STABLE_FRAMES_MIN}-${STABLE_FRAMES_MAX}` };
+            }
+            frames = c.frames;
+        }
+        return { ok: true, condition: { kind, frames, intervalMs, threshold } };
+    }
+    if (kind === 'changed')
+        return { ok: true, condition: { kind, intervalMs, threshold } };
+    // element_present / element_absent:ref 与 selector_text 二选一。
+    const hasRef = c.ref !== undefined;
+    const hasText = c.selector_text !== undefined;
+    if (hasRef === hasText) {
+        return { ok: false, error: `condition kind "${kind}" requires exactly one of ref or selector_text` };
+    }
+    if (hasRef) {
+        if (typeof c.ref !== 'string' || !/^e\d+$/.test(c.ref)) {
+            return { ok: false, error: 'condition.ref must look like "e12" (from the latest inspect)' };
+        }
+        return {
+            ok: true,
+            condition: { kind: kind, intervalMs, target: { ref: c.ref } },
+        };
+    }
+    const text = typeof c.selector_text === 'string' ? c.selector_text : '';
+    const step = parseSelectorText(text);
+    if (!step) {
+        return { ok: false, error: 'condition.selector_text must look like Button:"OK" or "OK" (quoted element name)' };
+    }
+    return {
+        ok: true,
+        condition: { kind: kind, intervalMs, target: { step, text: text.trim() } },
+    };
+}
+/** 给台账/输出用的简短条件描述。 */
+export function describeCondition(c) {
+    if (c.kind === 'stable' || c.kind === 'changed')
+        return c.kind;
+    const t = 'ref' in c.target ? c.target.ref : c.target.text;
+    return `${c.kind} ${t}`;
+}
+function defaultSleep(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal?.aborted)
+            return resolve();
+        const done = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', done);
+            resolve();
+        };
+        const timer = setTimeout(done, ms);
+        signal?.addEventListener('abort', done, { once: true });
+    });
+}
+/** 尺寸不同意味着显示配置变了——视为"变化"最安全(diff=1),而不是抛错中断等待。 */
+function safeDiff(a, b) {
+    try {
+        return diffRatio(a, b);
+    }
+    catch {
+        return 1;
+    }
+}
+/**
+ * 轮询直到条件满足 / 超时 / abort。
+ * stable:连续 `frames` 次相邻两帧 diff ≤ threshold;changed:相对基准 diff > threshold(互补判定)。
+ */
+export async function waitUntil(cond, timeoutMs, deps) {
+    const now = deps.now ?? Date.now;
+    const sleep = deps.sleep ?? defaultSleep;
+    const start = now();
+    let polls = 0;
+    let lastDiff;
+    let prev = null;
+    let base = cond.kind === 'changed' ? (deps.baseline ?? null) : null;
+    let stableRun = 0;
+    const finish = (outcome) => ({
+        outcome,
+        elapsedMs: now() - start,
+        polls,
+        lastDiff,
+    });
+    for (;;) {
+        if (deps.signal?.aborted)
+            return finish('aborted');
+        polls++;
+        let met = false;
+        if (cond.kind === 'stable') {
+            const frame = await deps.captureFrame();
+            if (prev) {
+                lastDiff = safeDiff(prev, frame);
+                stableRun = lastDiff <= cond.threshold ? stableRun + 1 : 0;
+                met = stableRun >= cond.frames;
+            }
+            prev = frame;
+        }
+        else if (cond.kind === 'changed') {
+            const frame = await deps.captureFrame();
+            if (base) {
+                lastDiff = safeDiff(base, frame);
+                met = lastDiff > cond.threshold;
+            }
+            else {
+                base = frame;
+            }
+        }
+        else {
+            const exists = await deps.elementExists(cond.target);
+            met = cond.kind === 'element_present' ? exists : !exists;
+        }
+        if (deps.signal?.aborted)
+            return finish('aborted');
+        if (met)
+            return finish('met');
+        const elapsed = now() - start;
+        if (elapsed >= timeoutMs)
+            return finish('timeout');
+        await sleep(Math.min(cond.intervalMs, timeoutMs - elapsed), deps.signal);
+    }
+}
+/** 超过此长度的文本逐字 SendInput 太慢且易被 IME/自动补全打断,auto 时改走剪贴板。 */
+export const PASTE_AUTO_MIN_LENGTH = 200;
+/** method 缺省(auto)时:长文本或含换行用 paste,否则 keys;显式 method 原样返回。 */
+export function pickTypeMethod(text, method) {
+    if (method === 'keys' || method === 'paste')
+        return method;
+    return text.length > PASTE_AUTO_MIN_LENGTH || /[\r\n]/.test(text) ? 'paste' : 'keys';
+}

package/dist/runtime/window-list.js

@@ -0,0 +1,164 @@
+/**
+ * computer.list_windows / focus_window 的纯逻辑层(design-notes/computer-use-rpa.md §4)。
+ *
+ * 窗口枚举的原始数据来自 uia-service 的 `windows` op;过滤、编号(w1..wN)、norm1000 换算、
+ * 按 ref / title_regex 选窗口都在这里,可用 fixture 离线单测,不碰 Win32。
+ */
+import { rectToNorm } from './uia-selector.js';
+/** list_windows 输出前缀;session/gui-actions.ts 靠它识别台账条目,改动须同步。 */
+export const LIST_WINDOWS_OUTPUT_PREFIX = 'Windows (';
+/** 最多展示的窗口数:桌面上几十个窗口再多对模型只是噪声。 */
+export const LIST_WINDOWS_MAX = 40;
+function toInt(v) {
+    return typeof v === 'number' && Number.isFinite(v) ? Math.round(v) : 0;
+}
+/** 解析 PowerShell 响应里的 windows 数组;字段缺失一律取保守默认值。 */
+export function parseRawWindows(resp) {
+    const arr = Array.isArray(resp.windows) ? resp.windows : resp.windows ? [resp.windows] : [];
+    const out = [];
+    for (const entry of arr) {
+        if (!entry || typeof entry !== 'object')
+            continue;
+        const w = entry;
+        const hwnd = toInt(w.hwnd);
+        if (!hwnd)
+            continue;
+        const r = (w.rect && typeof w.rect === 'object' ? w.rect : {});
+        out.push({
+            hwnd,
+            title: typeof w.title === 'string' ? w.title : '',
+            processName: typeof w.processName === 'string' ? w.processName : '',
+            pid: toInt(w.pid),
+            rect: { x: toInt(r.x), y: toInt(r.y), w: Math.max(0, toInt(r.w)), h: Math.max(0, toInt(r.h)) },
+            minimized: w.minimized === true,
+            foreground: w.foreground === true,
+            toolWindow: w.toolWindow === true,
+            cloaked: w.cloaked === true,
+        });
+    }
+    return out;
+}
+function onPrimaryScreen(rect, g) {
+    const cx = rect.x + rect.w / 2;
+    const cy = rect.y + rect.h / 2;
+    return cx >= g.originX && cx < g.originX + g.physW && cy >= g.originY && cy < g.originY + g.physH;
+}
+function clip(s, max) {
+    const flat = s.replace(/\s+/g, ' ').trim();
+    return flat.length > max ? `${flat.slice(0, max)}…` : flat;
+}
+/**
+ * 过滤(空标题 / 工具窗口 / cloaked)→ 编号 → norm1000 换算 → 文本。
+ * 保持原始 z-order(前 → 后),最多 LIST_WINDOWS_MAX 个。
+ */
+export function formatWindowList(raw, geometry) {
+    const visible = raw.filter((w) => w.title.trim().length > 0 && !w.toolWindow && !w.cloaked);
+    const truncated = visible.length > LIST_WINDOWS_MAX;
+    const items = visible.slice(0, LIST_WINDOWS_MAX).map((w, i) => {
+        const onPrimary = !w.minimized && onPrimaryScreen(w.rect, geometry);
+        return {
+            ref: `w${i + 1}`,
+            hwnd: w.hwnd,
+            title: w.title,
+            processName: w.processName,
+            pid: w.pid,
+            rect: w.rect,
+            minimized: w.minimized,
+            foreground: w.foreground,
+            onPrimary,
+            norm: onPrimary ? rectToNorm(w.rect, geometry) : null,
+        };
+    });
+    const lines = [
+        `${LIST_WINDOWS_OUTPUT_PREFIX}${items.length}${truncated ? `, first ${LIST_WINDOWS_MAX} shown` : ''}, front to back). ` +
+            'Rects are normalized 0-1000 [x, y, w, h] over the primary screen. ' +
+            'Use focus_window with window: "wN" to bring one to the front; refs expire on the next list_windows.',
+    ];
+    if (items.length === 0)
+        lines.push('(no top-level windows with a title)');
+    for (const it of items) {
+        const flags = [];
+        if (it.foreground)
+            flags.push('foreground');
+        if (it.minimized)
+            flags.push('minimized');
+        else if (!it.onPrimary)
+            flags.push('on secondary display');
+        const proc = it.processName ? `${it.processName}.exe` : 'unknown';
+        const rect = it.norm ? ` rect=[${it.norm.join(', ')}]` : '';
+        const tag = flags.length ? ` (${flags.join(', ')})` : '';
+        lines.push(`${it.ref} ${proc} ${JSON.stringify(clip(it.title, 80))}${rect}${tag}`);
+    }
+    return { text: lines.join('\n'), items, truncated };
+}
+/**
+ * 按 ref(w3)或 title_regex 选窗口。regex 命中多个视为歧义,返回错误并列出候选——
+ * 宁可让模型再指定一次,也不替它在两个同名窗口里挑一个。
+ */
+export function pickWindow(items, sel) {
+    if (sel.ref !== undefined) {
+        const item = items.find((i) => i.ref === sel.ref);
+        if (!item) {
+            return { ok: false, error: `window ${sel.ref} is not in the current window list; call list_windows again` };
+        }
+        return { ok: true, item };
+    }
+    if (sel.titleRegex === undefined) {
+        return { ok: false, error: 'a window selector (window: "wN" or title_regex) is required' };
+    }
+    let re;
+    try {
+        re = new RegExp(sel.titleRegex, 'i');
+    }
+    catch {
+        return { ok: false, error: `title_regex is not a valid regular expression: ${sel.titleRegex}` };
+    }
+    const hits = items.filter((i) => re.test(i.title));
+    if (hits.length === 0) {
+        return { ok: false, error: `no window title matches /${sel.titleRegex}/i; call list_windows to see what is open` };
+    }
+    if (hits.length > 1) {
+        const names = hits.map((h) => `${h.ref} ${JSON.stringify(clip(h.title, 50))}`).join(', ');
+        return {
+            ok: false,
+            error: `title_regex /${sel.titleRegex}/i is ambiguous (${hits.length} windows: ${names}); use window: "wN" or a narrower regex`,
+        };
+    }
+    return { ok: true, item: hits[0] };
+}
+/**
+ * flow 回放用:按进程名(+可选标题正则)在原始窗口里选唯一目标。
+ * 与 pickWindow 的区别:不依赖 wN 快照;同进程多窗口时,只在恰有一个前台窗口时取前台,否则报歧义。
+ */
+export function pickWindowForReplay(raw, sel) {
+    let re;
+    if (sel.titleRegex !== undefined) {
+        try {
+            re = new RegExp(sel.titleRegex, 'i');
+        }
+        catch {
+            return { ok: false, error: `window titleRegex is not a valid regular expression: ${sel.titleRegex}` };
+        }
+    }
+    const proc = sel.processName?.toLowerCase();
+    const label = [sel.processName ? `${sel.processName}.exe` : undefined, re ? `title /${sel.titleRegex}/i` : undefined]
+        .filter(Boolean)
+        .join(' ');
+    const hits = raw.filter((w) => w.title.trim().length > 0 &&
+        !w.toolWindow &&
+        !w.cloaked &&
+        (proc === undefined || w.processName.toLowerCase() === proc) &&
+        (!re || re.test(w.title)));
+    if (hits.length === 0)
+        return { ok: false, error: `no open window matches ${label || '(any)'}` };
+    if (hits.length === 1)
+        return { ok: true, window: hits[0] };
+    const foreground = hits.filter((w) => w.foreground);
+    if (foreground.length === 1)
+        return { ok: true, window: foreground[0] };
+    const names = hits.map((h) => JSON.stringify(clip(h.title, 40))).join(', ');
+    return {
+        ok: false,
+        error: `${hits.length} windows match ${label || '(any)'} and none is uniquely in front: ${names}`,
+    };
+}

package/dist/session/compact.js

@@ -693,7 +693,7 @@ export async function compactHistory(history, opts) {
     }
     let oldGroups = groups.slice(0, groups.length - kept.length);
     // 非空 user 兜底:保留区从尾部累积,长回合里最近的若干组常常全是 assistant+tool,
-    // 一个 8000 字符的工具结果就能吃满小窗口的 keepBudget → user 全落进旧区被摘要掉 →
+    // 一个 MAX_HISTORY_RESULT 量级的工具结果就能吃满小窗口的 keepBudget → user 全落进旧区被摘要掉 →
     // 重建后 history 无 user → 下一步 chat() 被 hasNonEmptyUser 守卫直接拒(整轮中断)。
     // 把紧邻保留区之前的那条 user 移进保留区:它是当前任务的原始请求,且 user 消息本身极小。
     // 放这里(而非重建后补 stub)是因为:原文保留,不造新消息形状,也不必进摘要器。

package/dist/session/gui-actions.js

@@ -16,6 +16,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { getSandboxRoot } from '../sandbox/root.js';
+import { LIST_WINDOWS_OUTPUT_PREFIX } from '../runtime/window-list.js';
 import { getCurrentSessionId } from './state.js';
 /** 台账文件名:与 notes.md 同目录(会话目录)。 */
 export const GUI_ACTIONS_FILENAME = 'gui-actions.log';
@@ -50,11 +51,18 @@ function normalizeIntent(summary) {
     // 顺序敏感:先处理带自身句式的动作,最后才是泛化的 "<action> at (x, y)"。
     text = text.replace(/^moved cursor to (\(.*\))$/, 'mouse_move $1');
     text = text.replace(/^typed ([\d.]+) characters$/, 'type $1 chars');
+    // type 的 paste 路径(computer.ts):`pasted 300 characters via clipboard[ (WARNING: ...)]` → `paste 300 chars`。
+    text = text.replace(/^pasted ([\d.]+) characters via clipboard(?: \(WARNING:.*\))?$/, 'paste $1 chars');
     text = text.replace(/^pressed key combo (".*")$/, 'key $1');
     text = text.replace(/^dragged from (\([^)]*\)) to (\([^)]*\))$/, 'drag $1 -> $2');
     text = text.replace(/^scrolled (\w+) by ([\d.]+) at (\(.*\))$/, 'scroll $1 $2 $3');
     text = text.replace(/^waited ([\d.]+)ms$/, 'wait $1ms');
     text = text.replace(/^cursor at physical \([^)]*\) = normalized (\(.*\))$/, 'cursor_position $1');
+    // UIA 元素动作(computer.ts executeAction 的 click_element/set_value 句式):
+    // `clicked element e9 Button "关闭" at (978, 11)` / `... via invoke pattern` → `click_element e9 "关闭"`;
+    // `set value of element e7 Document "文本" (18 chars) via pattern` → `set_value e7 18 chars`。
+    text = text.replace(/^clicked element (e\d+) \S+ (".*") (?:at \([^)]*\)|via \w+ pattern)$/, 'click_element $1 $2');
+    text = text.replace(/^set value of element (e\d+) \S+ ".*" \((\d+) chars\) via \w+$/, 'set_value $1 $2 chars');
     text = text.replace(/^(\w+) at (\(.*\))$/, '$1 $2');
     return text;
 }
@@ -78,6 +86,45 @@ export function parseComputerOutput(output) {
     if (text.startsWith('Screenshot captured (')) {
         return { intent: 'screenshot', observation: 're-captured' };
     }
+    // inspect 输出前缀与 runtime/uia-selector.ts 的 INSPECT_OUTPUT_PREFIX 一致;只回灌文本,不改差分基准。
+    if (text.startsWith('UI elements of window ')) {
+        const n = /, (\d+) nodes \(/.exec(text);
+        return { intent: 'inspect', observation: n ? `${n[1]} nodes` : 'listed' };
+    }
+    // 窗口作用域(computer.ts, design-notes/computer-use-rpa.md §4)。
+    // list_windows:`Windows (N, front to back). ...`,只回灌文本。
+    if (text.startsWith(LIST_WINDOWS_OUTPUT_PREFIX)) {
+        const n = /^Windows \((\d+)/.exec(text);
+        return { intent: 'list_windows', observation: n ? `${n[1]} windows` : 'listed' };
+    }
+    // focus_window 副屏窗口默认不搬动:`w3 app.exe "title" is on a secondary display or off-screen, so it was NOT focused ...`。
+    const notFocused = /^(w\d+) .* is on a secondary display or off-screen, so it was NOT focused/.exec(text);
+    if (notFocused) {
+        return { intent: `focus_window ${notFocused[1]}`, observation: 'not focused (secondary display)' };
+    }
+    // focus_window 成功路径:`Focused w3 app.exe "title" rect=[...]. Screen re-captured ...`;系统拒绝前台切换时如实标注。
+    const focused = /^Focused (w\d+) /.exec(text);
+    if (focused) {
+        const refused = text.includes('The OS refused the foreground switch');
+        return {
+            intent: `focus_window ${focused[1]}`,
+            observation: refused ? 're-captured, OS refused foreground switch' : 're-captured',
+        };
+    }
+    // 窗口截图:`Window screenshot of w3 app.exe "title" (printwindow, 923×587 physical, ...)`;不是整屏基准,不参与差分。
+    const winShot = /^Window screenshot of (w\d+) .*? \((printwindow|screen),/.exec(text);
+    if (winShot) {
+        return { intent: `screenshot window ${winShot[1]}`, observation: `captured (${winShot[2]}, window only)` };
+    }
+    // wait_until(computer.ts):`waited until <cond>: met (1200ms, 4 polls...)` / `... timed out (10000ms, ...)`。
+    // 台账只记条件与耗时;met 只代表轮询条件成立,仍遵循"像素优先"纪律,不升级成任务成功。
+    const waited = /^waited until (.+?): (met|timed out) \((\d+)ms/.exec(text);
+    if (waited) {
+        return {
+            intent: `wait_until ${truncateDetail(waited[1])}`,
+            observation: `${waited[2]}, ${waited[3]}ms`,
+        };
+    }
     const diffMatch = /\.\s*No visible change on screen \(frame diff ([\d.]+)%/.exec(text);
     if (diffMatch) {
         // 「no visible change」即"未达成"的现成措辞:保留原措辞,不升级成任何成功断言(§10.5)。