mocode-ai 1.4.7 → 1.5.0

package/dist/tools/builtins/computer.js

@@ -12,10 +12,36 @@ import { mkdir, readFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { jailResolve } from '../../sandbox/index.js';
 import { captureDesktop } from '../../runtime/screen-capture.js';
-import { decodePng, encodePng, downscale, fitToLongEdge, crop, normToPhysical, normRectToPhysical, physicalToNorm, isNormCoord, } from '../../runtime/screen-pipeline.js';
+import { decodePng, encodePng, downscale, fitToLongEdge, crop, readPngSize, diffRatio, normToPhysical, normRectToPhysical, physicalToNorm, isNormCoord, } from '../../runtime/screen-pipeline.js';
 import { createInputInjector } from '../../runtime/input-injector.js';
-/** 喂模型的截图长边上限(Anthropic 推荐区间)。 */
-const MAX_EDGE = 1280;
+/**
+ * 喂模型的截图长边上限。1280 是 Anthropic 的兼容下限,1568 对齐主流视觉模型原生分辨率 ——
+ * 4K 屏上小字/密集 UI 的定位精度提升明显,代价是每张图多约 300 token。可用
+ * MOCODE_CU_MAX_EDGE 覆盖。
+ */
+const DEFAULT_MAX_EDGE = 1568;
+function maxEdge() {
+    const raw = Number(process.env.MOCODE_CU_MAX_EDGE);
+    if (!Number.isFinite(raw) || raw < 256 || raw > 4096)
+        return DEFAULT_MAX_EDGE;
+    return Math.round(raw);
+}
+/**
+ * 帧间差异低于此比例就认为「屏幕没变」,跳过回灌。
+ * 0.005 = 0.5%,足以吃掉抗锯齿/光标抖动,又不会漏掉真实重绘(局部重绘通常 >1%)。
+ */
+const DEFAULT_DIFF_THRESHOLD = 0.005;
+function diffThreshold() {
+    // 空串必须显式回落:Number('') === 0,而 0 能通过下面的区间校验 —— 用户在 .env 里清空这个
+    // 变量(注释掉的常见写法)会静默把去重关掉。0 是合法的显式值(关闭去重),空不是。
+    const rawEnv = process.env.MOCODE_CU_DIFF_THRESHOLD;
+    if (rawEnv === undefined || rawEnv.trim() === '')
+        return DEFAULT_DIFF_THRESHOLD;
+    const raw = Number(rawEnv);
+    if (!Number.isFinite(raw) || raw < 0 || raw > 1)
+        return DEFAULT_DIFF_THRESHOLD;
+    return raw;
+}
 const WAIT_MAX_MS = 10000;
 const SCROLL_MAX = 10;
 const ACTIONS = [
@@ -98,7 +124,22 @@ export function validateComputerArgs(args) {
     }
     return null;
 }
-/** 抓一次主屏并返回物理分辨率。 */
+/**
+ * 会话级几何缓存。显示器分辨率/DPI 在一次会话里几乎不变,而每次调用都重抓一次屏只为拿
+ * 宽高是纯粹的浪费 —— 抓屏是整条链路最贵的一步(冷启动 PowerShell + 4K 位图 CopyFromScreen)。
+ * 缓存后每个输入动作只需**一次**抓屏(动作后的重截屏),比原来少一半。
+ *
+ * 失效:每次动作后的重截屏都会回传 geometry,与之不符就立即更新缓存并提示模型重试
+ * (插拔外接屏 / 改缩放比时,这一步的坐标是用旧几何算的,必须让模型知道)。
+ */
+let geometryCache = null;
+/** 会话级上一帧(已 downscale),用于差分。/cu off 时清空。 */
+let lastFrame = null;
+export function resetComputerState() {
+    geometryCache = null;
+    lastFrame = null;
+}
+/** 抓一次主屏并返回几何信息。Windows 走 captureDesktop 的 geometry(零解码);其余平台只读 IHDR。 */
 async function capturePrimary(signal) {
     const ts = new Date().toISOString().replace(/[:.]/g, '-');
     const outputPath = jailResolve(`.mocode/screenshots/computer-${ts}-${Math.random().toString(36).slice(2, 8)}.png`);
@@ -107,24 +148,99 @@ async function capturePrimary(signal) {
     if (cap.status !== 'passed') {
         throw new Error(`screenshot capture failed: ${cap.detail}`);
     }
-    const png = decodePng(await readFile(outputPath));
-    return { path: outputPath, physW: png.width, physH: png.height };
+    const g = cap.geometry;
+    if (g && g.shotW > 0 && g.shotH > 0 && g.physW > 0 && g.physH > 0) {
+        return {
+            path: outputPath,
+            shotW: g.shotW,
+            shotH: g.shotH,
+            physW: g.physW,
+            physH: g.physH,
+            originX: g.originX,
+            originY: g.originY,
+        };
+    }
+    // 非 Windows 或 geometry 缺失:只读 IHDR,不要为了两个数字解码 14MB 像素。
+    const size = readPngSize(await readFile(outputPath));
+    if (!size)
+        throw new Error('screenshot captured but its dimensions could not be read');
+    return {
+        path: outputPath,
+        shotW: size.width,
+        shotH: size.height,
+        physW: size.width,
+        physH: size.height,
+        originX: 0,
+        originY: 0,
+    };
+}
+/** 取当前屏幕几何:命中缓存则不抓屏(path 为 undefined)。 */
+async function resolveScreen(signal) {
+    if (geometryCache)
+        return { screen: geometryCache };
+    const shot = await capturePrimary(signal);
+    geometryCache = {
+        shotW: shot.shotW,
+        shotH: shot.shotH,
+        physW: shot.physW,
+        physH: shot.physH,
+        originX: shot.originX,
+        originY: shot.originY,
+    };
+    return { screen: geometryCache, path: shot.path };
 }
-/** 读 PNG 文件 → 缩放 → 编码 → dataUrl。 */
-async function toAttachment(path) {
+/** 用最新一次抓屏的几何校正缓存;返回 true 表示本次动作的坐标可能是用过期几何算的。 */
+function syncGeometry(shot) {
+    const prev = geometryCache;
+    const changed = !prev ||
+        prev.physW !== shot.physW ||
+        prev.physH !== shot.physH ||
+        prev.shotW !== shot.shotW ||
+        prev.shotH !== shot.shotH ||
+        prev.originX !== shot.originX ||
+        prev.originY !== shot.originY;
+    geometryCache = { ...shot };
+    return changed && prev !== null;
+}
+/** 读 PNG 文件 → 缩放 → 编码 → dataUrl,顺带算出与上一帧的差异。 */
+async function toAttachment(path, m) {
+    let t = Date.now();
     const png = decodePng(await readFile(path));
-    const { img } = downscale(png, MAX_EDGE);
+    if (m)
+        m.decodeMs += Date.now() - t;
+    t = Date.now();
+    const { img } = downscale(png, maxEdge());
+    if (m)
+        m.scaleMs += Date.now() - t;
+    const prev = lastFrame;
+    let diff;
+    if (prev && prev.width === img.width && prev.height === img.height) {
+        diff = diffRatio(prev, img);
+    }
+    t = Date.now();
     const buf = encodePng(img);
+    if (m)
+        m.encodeMs += Date.now() - t;
     return {
         dataUrl: `data:image/png;base64,${buf.toString('base64')}`,
         bytes: buf.length,
         dispW: img.width,
         dispH: img.height,
+        diff,
+        img,
     };
 }
+/** 回灌了一帧:把它设为新的差分基准。 */
+function commitFrame(img) {
+    lastFrame = img;
+}
 async function executeAction(injector, action, args, screen) {
-    const { physW, physH } = screen;
-    const toPhys = (pair) => normToPhysical(pair[0], pair[1], physW, physH);
+    const { physW, physH, originX, originY } = screen;
+    // norm1000 → 虚拟桌面物理像素:先按物理分辨率缩放,再叠加主屏原点(多屏)。
+    const toPhys = (pair) => {
+        const [x, y] = normToPhysical(pair[0], pair[1], physW, physH);
+        return [originX + x, originY + y];
+    };
     switch (action) {
         case 'mouse_move': {
             const [x, y] = toPhys(args.coordinate);
@@ -183,17 +299,51 @@ async function executeAction(injector, action, args, screen) {
         }
         case 'cursor_position': {
             const pos = await injector.cursorPosition();
-            const [nx, ny] = physicalToNorm(pos.x, pos.y, physW, physH);
+            const [nx, ny] = physicalToNorm(pos.x - originX, pos.y - originY, physW, physH);
             return `cursor at physical (${pos.x}, ${pos.y}) = normalized (${nx}, ${ny})`;
         }
         default:
             throw new Error(`no executor for action "${action}"`);
     }
 }
+const METRIC_CAP = 50;
+const metrics = [];
+function recordMetric(m) {
+    metrics.push(m);
+    if (metrics.length > METRIC_CAP)
+        metrics.shift();
+}
+function pct(values, p) {
+    if (!values.length)
+        return 0;
+    const s = [...values].sort((a, b) => a - b);
+    const i = Math.min(s.length - 1, Math.max(0, Math.round((p / 100) * (s.length - 1))));
+    return Math.round(s[i]);
+}
+export function getComputerMetrics() {
+    const totals = metrics.map((m) => m.totalMs);
+    const caps = metrics.map((m) => m.captureMs);
+    const skipped = metrics.filter((m) => m.skipped).length;
+    const bytes = metrics.reduce((a, m) => a + m.bytes, 0);
+    return {
+        count: metrics.length,
+        totalP50: pct(totals, 50),
+        totalP95: pct(totals, 95),
+        captureP50: pct(caps, 50),
+        skippedRatio: metrics.length ? skipped / metrics.length : 0,
+        avgBytes: metrics.length ? Math.round(bytes / metrics.length) : 0,
+        last: metrics[metrics.length - 1],
+    };
+}
+export function resetComputerMetrics() {
+    metrics.length = 0;
+}
 export const computerTool = {
     name: 'computer',
     description: 'Control the desktop GUI: move/click the mouse, type text, press keys, scroll, and zoom into screen regions. ' +
-        'Every action returns a fresh screenshot as a visual attachment — inspect it before the next action and self-correct. ' +
+        'Every action re-captures the screen. If the screen is unchanged since the last screenshot you saw, no image is ' +
+        'attached and the result says so — treat the previous screenshot as still current instead of assuming the action failed. ' +
+        'Inspect each attached screenshot before the next action and self-correct. ' +
         'Coordinates use a normalized 0-1000 grid (x right, y down) over the primary screen; you never need the physical resolution. ' +
         'Use zoom on a tight region before clicking small or dense UI elements. Destructive or sensitive targets ' +
         '(form submit, payment, credentials, delete/send) require explicit user intent — ask first.',
@@ -242,9 +392,26 @@ export const computerTool = {
     async execute(args, ctx) {
         const invalid = validateComputerArgs(args);
         if (invalid) {
-            return { status: 'error', code: 'INVALID_ARGUMENTS', retryable: false, output: `Invalid computer action: ${invalid}` };
+            return {
+                status: 'error',
+                code: 'INVALID_ARGUMENTS',
+                retryable: false,
+                output: `Invalid computer action: ${invalid}`,
+            };
         }
         const action = args.action;
+        const started = Date.now();
+        const m = {
+            action,
+            captureMs: 0,
+            decodeMs: 0,
+            scaleMs: 0,
+            encodeMs: 0,
+            injectMs: 0,
+            totalMs: 0,
+            bytes: 0,
+            skipped: false,
+        };
         let injector;
         try {
             injector = createInputInjector();
@@ -254,60 +421,135 @@ export const computerTool = {
             return { status: 'error', code: 'EXECUTION_ERROR', retryable: false, output: message };
         }
         injector.bindSignal(ctx?.signal);
+        // left_mouse_down 之后若中途失败/中断,鼠标会一直处于按下状态 —— 用户的桌面就废了。
+        // 这里统一在 finally 里兜底释放。
+        let pendingMouseDown = false;
         try {
-            // 动作前截屏:拿物理分辨率(norm1000 → 物理像素换算的基础)。
-            const before = await capturePrimary(ctx?.signal);
-            const screen = { physW: before.physW, physH: before.physH };
-            // zoom:不注入输入,裁 region 放大回灌。
-            if (action === 'zoom') {
-                const rect = normRectToPhysical(args.region, screen.physW, screen.physH);
-                const png = decodePng(await readFile(before.path));
-                const zoomed = fitToLongEdge(crop(png, rect.x, rect.y, rect.w, rect.h), MAX_EDGE);
+            // 只有 zoom / screenshot 真正需要一张新位图;输入动作只需要**几何**,命中会话缓存就不抓屏
+            // (抓屏是整条链路最贵的一步,省掉这一步等于把每步的本地开销砍半)。
+            let screen;
+            let freshPath;
+            if (action === 'zoom' || action === 'screenshot') {
+                const t = Date.now();
+                const shot = await capturePrimary(ctx?.signal);
+                m.captureMs += Date.now() - t;
+                screen = {
+                    shotW: shot.shotW,
+                    shotH: shot.shotH,
+                    physW: shot.physW,
+                    physH: shot.physH,
+                    originX: shot.originX,
+                    originY: shot.originY,
+                };
+                syncGeometry(screen);
+                freshPath = shot.path;
+            }
+            else {
+                const t = Date.now();
+                const r = await resolveScreen(ctx?.signal);
+                m.captureMs += Date.now() - t;
+                screen = r.screen;
+                freshPath = r.path;
+            }
+            // zoom:不注入输入,裁 region 放大回灌。裁剪发生在截图位图上,故用 shotW/shotH。
+            if (action === 'zoom' && freshPath) {
+                const rect = normRectToPhysical(args.region, screen.shotW, screen.shotH);
+                let t = Date.now();
+                const png = decodePng(await readFile(freshPath));
+                m.decodeMs += Date.now() - t;
+                t = Date.now();
+                const zoomed = fitToLongEdge(crop(png, rect.x, rect.y, rect.w, rect.h), maxEdge());
+                m.scaleMs += Date.now() - t;
+                t = Date.now();
                 const buf = encodePng(zoomed);
+                m.encodeMs += Date.now() - t;
+                m.bytes = buf.length;
+                // zoom 出的图不是全屏基准,不能喂给差分。
+                lastFrame = null;
                 return {
                     status: 'success',
                     code: 'OK',
                     retryable: false,
-                    output: `Zoomed into region ${JSON.stringify(args.region)} → physical (${rect.x}, ${rect.y}, ${rect.w}×${rect.h}), ` +
+                    output: `Zoomed into region ${JSON.stringify(args.region)} → screenshot pixels (${rect.x}, ${rect.y}, ${rect.w}×${rect.h}), ` +
                         `magnified to ${zoomed.width}×${zoomed.height}. The attached image shows ONLY this region — ` +
                         `coordinates you output still map to the FULL screen, so do not reuse this region's local coordinates for clicks.`,
                     modelAttachments: [
-                        { type: 'image', name: 'computer-zoom.png', mime: 'image/png', dataUrl: `data:image/png;base64,${buf.toString('base64')}` },
+                        {
+                            type: 'image',
+                            name: 'computer-zoom.png',
+                            mime: 'image/png',
+                            dataUrl: `data:image/png;base64,${buf.toString('base64')}`,
+                        },
                     ],
                 };
             }
             // screenshot:只截屏回灌(target 可指定 all)。
             if (action === 'screenshot') {
-                let shotPath = before.path;
+                let shotPath = freshPath;
                 if (args.target === 'all') {
                     const ts = new Date().toISOString().replace(/[:.]/g, '-');
                     const allPath = jailResolve(`.mocode/screenshots/computer-${ts}-all.png`);
                     await mkdir(dirname(allPath), { recursive: true });
+                    const t = Date.now();
                     const cap = await captureDesktop(allPath, 'all', ctx?.signal);
+                    m.captureMs += Date.now() - t;
                     if (cap.status !== 'passed')
                         throw new Error(`screenshot capture failed: ${cap.detail}`);
                     shotPath = allPath;
                 }
-                const att = await toAttachment(shotPath);
+                const att = await toAttachment(shotPath, m);
+                commitFrame(att.img);
+                m.bytes = att.bytes;
                 return {
                     status: 'success',
                     code: 'OK',
                     retryable: false,
-                    output: `Screenshot captured (physical ${screen.physW}×${screen.physH}, shown at ${att.dispW}×${att.dispH}). ` +
+                    output: `Screenshot captured (primary screen ${screen.physW}×${screen.physH} physical, shown at ${att.dispW}×${att.dispH}). ` +
                         'Coordinates remain normalized 0-1000 over the primary screen. Visual content is attached to the next model request.',
-                    modelAttachments: [{ type: 'image', name: 'computer-screenshot.png', mime: 'image/png', dataUrl: att.dataUrl }],
+                    modelAttachments: [
+                        { type: 'image', name: 'computer-screenshot.png', mime: 'image/png', dataUrl: att.dataUrl },
+                    ],
                 };
             }
             // 输入动作:注入 → 重截屏回灌(wait 也回灌,等待后界面往往已变化)。
+            if (action === 'left_mouse_down')
+                pendingMouseDown = true;
+            let t = Date.now();
             const summary = await executeAction(injector, action, args, screen);
+            m.injectMs += Date.now() - t;
+            if (action === 'left_mouse_up')
+                pendingMouseDown = false;
+            t = Date.now();
             const after = await capturePrimary(ctx?.signal);
-            const att = await toAttachment(after.path);
+            m.captureMs += Date.now() - t;
+            const geometryStale = syncGeometry(after);
+            const att = await toAttachment(after.path, m);
+            m.bytes = att.bytes;
+            m.diff = att.diff;
+            const staleNote = geometryStale
+                ? ' WARNING: display geometry changed mid-action (monitor or scaling change); ' +
+                    'coordinates for this action used the previous geometry — verify and repeat if the click landed wrong.'
+                : '';
+            // 差分命中:屏幕与模型上次看到的相比没变,再发一张同样的图只是白烧 token 和 prefill。
+            if (att.diff !== undefined && att.diff < diffThreshold()) {
+                m.skipped = true;
+                return {
+                    status: 'success',
+                    code: 'OK',
+                    retryable: false,
+                    output: `${summary}. No visible change on screen (frame diff ${(att.diff * 100).toFixed(2)}% < ` +
+                        `${(diffThreshold() * 100).toFixed(2)}% threshold) — no new screenshot is attached; ` +
+                        'the previous screenshot is still current. If you expected a change, the action likely did not ' +
+                        `take effect: re-check the target coordinates or try a different approach.${staleNote}`,
+                };
+            }
+            commitFrame(att.img);
             return {
                 status: 'success',
                 code: 'OK',
                 retryable: false,
-                output: `${summary}. Screen re-captured (physical ${screen.physW}×${screen.physH}, shown at ${att.dispW}×${att.dispH}): ` +
-                    'inspect the attached screenshot to verify the result before the next action.',
+                output: `${summary}. Screen re-captured (primary screen ${screen.physW}×${screen.physH} physical, shown at ${att.dispW}×${att.dispH}): ` +
+                    `inspect the attached screenshot to verify the result before the next action.${staleNote}`,
                 modelAttachments: [{ type: 'image', name: 'computer-result.png', mime: 'image/png', dataUrl: att.dataUrl }],
             };
         }
@@ -316,7 +558,24 @@ export const computerTool = {
             if (ctx?.signal?.aborted || /aborted/i.test(message)) {
                 return { status: 'aborted', code: 'ABORTED', retryable: false, output: message };
             }
-            return { status: 'error', code: 'EXECUTION_ERROR', retryable: false, output: `computer action failed: ${message}` };
+            return {
+                status: 'error',
+                code: 'EXECUTION_ERROR',
+                retryable: false,
+                output: `computer action failed: ${message}`,
+            };
+        }
+        finally {
+            if (pendingMouseDown) {
+                try {
+                    await injector.mouseUp('left');
+                }
+                catch {
+                    // 释放失败也要让上层拿到原本的错误,不掩盖。
+                }
+            }
+            m.totalMs = Date.now() - started;
+            recordMetric(m);
         }
     },
 };

package/dist/tools/builtins/screenshot.js

@@ -1,9 +1,12 @@
-import { mkdir } from 'node:fs/promises';
-import { dirname, extname } from 'node:path';
+import { mkdir, readFile } from 'node:fs/promises';
+import { basename, dirname, extname } from 'node:path';
 import { loadImageAttachment, MAX_INLINE_BYTES_DEFAULT } from '../../attachments/image.js';
 import { jailResolve } from '../../sandbox/index.js';
 // 平台捕获逻辑已抽到 runtime/screen-capture.ts(computer 工具闭环共用),本文件只做薄封装。
 import { captureDesktop } from '../../runtime/screen-capture.js';
+import { decodePng, downscale, encodePng } from '../../runtime/screen-pipeline.js';
+/** 原图超过内联上限时降级缩放的长边(对齐主流视觉模型的原生分辨率)。 */
+const FALLBACK_MAX_EDGE = 1568;
 export const screenshotTool = {
     name: 'screenshot',
     description: 'Capture the desktop and inspect it as visual model input. Use this to diagnose visible UI state, dialogs, rendering problems, or applications that cannot be understood from source files alone. The user must approve each capture because screenshots may contain sensitive information.',
@@ -68,15 +71,41 @@ export const screenshotTool = {
         const loaded = await loadImageAttachment(outputPath, {
             maxBytes: MAX_INLINE_BYTES_DEFAULT,
         });
+        const detail = args.detail === 'low' || args.detail === 'auto' ? args.detail : 'high';
+        // 高分屏(尤其 DPI aware 后抓到物理分辨率)的 PNG 可能超过内联上限:
+        // 原图仍留在磁盘上,回灌改用缩放版本,不让工具因此失败。
         if (!loaded.ok) {
-            return {
-                status: 'error',
-                code: 'EXECUTION_ERROR',
-                retryable: false,
-                output: `Screenshot saved to ${requestedPath}, but it could not be attached: ${loaded.reason}`,
-            };
+            try {
+                const png = decodePng(await readFile(outputPath));
+                const { img } = downscale(png, FALLBACK_MAX_EDGE);
+                const buf = encodePng(img);
+                return {
+                    status: 'success',
+                    code: 'OK',
+                    retryable: false,
+                    output: `Captured ${target} display to ${requestedPath} (${png.width}×${png.height}); ` +
+                        `it exceeded the inline limit, so the attached copy was resized to ${img.width}×${img.height}.`,
+                    modelAttachments: [
+                        {
+                            type: 'image',
+                            name: basename(outputPath),
+                            mime: 'image/png',
+                            dataUrl: `data:image/png;base64,${buf.toString('base64')}`,
+                            detail,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                return {
+                    status: 'error',
+                    code: 'EXECUTION_ERROR',
+                    retryable: false,
+                    output: `Screenshot saved to ${requestedPath}, but it could not be attached: ${loaded.reason}${message ? ` (resize fallback failed: ${message})` : ''}`,
+                };
+            }
         }
-        const detail = args.detail === 'low' || args.detail === 'auto' ? args.detail : 'high';
         const { att } = loaded;
         return {
             status: 'success',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mocode-ai",
-  "version": "1.4.7",
+  "version": "1.5.0",
   "description": "终端编码 agent:LLM + tool-call 循环 + 流式输出(含思考)+ 25 个工具,接任意 OpenAI 兼容后端。",
   "type": "module",
   "bin": {