@dipoleml/dcode-cli 1.2.1 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dcode.cjs +2214 -1891
- package/dist/packages/browser-use-plugin/.dcode-plugin/plugin.json +8 -8
- package/dist/packages/dcode-cua-plugin/.dcode-plugin/plugin.json +10 -0
- package/dist/packages/dcode-cua-plugin/docs/computer-use.md +470 -0
- package/dist/packages/dcode-cua-plugin/package.json +12 -0
- package/dist/packages/dcode-cua-plugin/scripts/bump-dcode-cua-producer.mjs +305 -0
- package/dist/packages/dcode-cua-plugin/scripts/check-cua-baseline.mjs +155 -0
- package/dist/packages/dcode-cua-plugin/scripts/check-sdk.mjs +4 -0
- package/dist/packages/dcode-cua-plugin/scripts/check-version-coherence.mjs +478 -0
- package/dist/packages/dcode-cua-plugin/scripts/computer-use-client.mjs +1208 -0
- package/dist/packages/dcode-cua-plugin/scripts/resolve-pnpm-invocation.mjs +26 -0
- package/dist/packages/dcode-cua-plugin/scripts/sync-cache.mjs +116 -0
- package/dist/packages/dcode-cua-plugin/scripts/sync-skill-from-dcode-cua.mjs +115 -0
- package/dist/packages/dcode-cua-plugin/skills/computer-use/SKILL.md +306 -0
- package/dist/packages/node-repl-host/dist/mcp/server.js +3 -3
- package/dist/packages/node-repl-host/package.json +1 -1
- package/dist/provider/dcode-builtin.json +6748 -6748
- package/node_modules/@dcode/contracts/dist/events/event-reducer-helpers.d.ts +1 -1
- package/node_modules/@dcode/contracts/dist/events/event-reducer-helpers.js +5 -1
- package/node_modules/@dcode/contracts/dist/events/event-reducer.js +26 -5
- package/node_modules/@dcode/contracts/dist/interfaces/session.port.d.ts +5 -1
- package/node_modules/@dcode/contracts/dist/tools/bash.js +4 -1
- package/node_modules/@dcode/i18n/dist/locales/en-US.js +11 -23
- package/node_modules/@dcode/i18n/dist/locales/fr-CA.js +9 -21
- package/node_modules/@dcode/i18n/dist/types.d.ts +0 -12
- package/node_modules/@dcode/model-option-map/package.json +1 -1
- package/node_modules/@dcode/shared/package.json +1 -1
- package/node_modules/@dcode/tui/dist/app-model-ref.js +9 -2
- package/node_modules/@dcode/tui/dist/index.js +57 -65
- package/node_modules/@dcode/tui/dist/theme/defaults.js +21 -15
- package/package.json +3 -3
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 解析"怎么调用 pnpm"这一个决策,独立成模块只为可测:bump 脚本本体是 top-level await,
|
|
3
|
+
* import 它就会真的执行一次 bump,没法在单测里覆盖这段逻辑。
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @param {{ npmExecpath?: string | undefined, platform: string }} input
|
|
8
|
+
* @returns {{ command: string, args: string[], options: { shell?: true } }}
|
|
9
|
+
*/
|
|
10
|
+
export function resolvePnpmInvocation(input) {
|
|
11
|
+
const { npmExecpath, platform } = input;
|
|
12
|
+
const args = ["install", "--prefer-offline"];
|
|
13
|
+
// 首选让当前 Node 直接跑 pnpm 的 JS 入口:不经 shell、不碰 `.cmd`,三平台同一条码路。
|
|
14
|
+
// pnpm 运行 package script 时会注入 npm_execpath。
|
|
15
|
+
const entry = typeof npmExecpath === "string" ? npmExecpath.trim() : "";
|
|
16
|
+
if (entry) {
|
|
17
|
+
return { command: process.execPath, args: [entry, ...args], options: {} };
|
|
18
|
+
}
|
|
19
|
+
// 回落。Windows 上 pnpm 是 `pnpm.cmd`:execFile 不走 shell 时不补 PATHEXT(ENOENT),
|
|
20
|
+
// 而 Node 自 18.20/20.12 起(CVE-2024-27980)又禁止不带 shell spawn `.cmd`(EINVAL)。
|
|
21
|
+
// 两条都堵死,所以这里只能借 shell;参数是硬编码字面量,没有拼接也没有外部输入。
|
|
22
|
+
if (platform === "win32") {
|
|
23
|
+
return { command: "pnpm", args, options: { shell: true } };
|
|
24
|
+
}
|
|
25
|
+
return { command: "pnpm", args, options: {} };
|
|
26
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// Sync the built dev plugin into the CLI's plugin cache so the running dcode CLI
|
|
2
|
+
// (which loads plugins from ~/.dcode/cli/plugins/cache/..., NOT from this repo)
|
|
3
|
+
// picks up skill / MCP / manifest changes without a manual copy.
|
|
4
|
+
//
|
|
5
|
+
// This closes the last hop of the sync chain:
|
|
6
|
+
// dcode-cua repo --sync:skill/sync:mcp--> dev plugin (here) --sync:cache--> CLI cache
|
|
7
|
+
//
|
|
8
|
+
// The cache dir name (e.g. 0.1.31) is just an install-time cache key; copying newer
|
|
9
|
+
// content (plugin.json says 0.2.0) on top of it is tolerated by the CLI and is exactly
|
|
10
|
+
// what makes a dev skill/MCP edit go live. Run after `pnpm sync:skill` (+ `pnpm build`
|
|
11
|
+
// when the MCP wrapper or skill changed).
|
|
12
|
+
//
|
|
13
|
+
// Usage:
|
|
14
|
+
// node scripts/sync-cache.mjs # auto-resolve cache path from marketplace.json
|
|
15
|
+
// node scripts/sync-cache.mjs <cacheDir> # explicit cache dir
|
|
16
|
+
import { cp, mkdir, readFile, readdir, stat } from "node:fs/promises";
|
|
17
|
+
import { existsSync } from "node:fs";
|
|
18
|
+
import { dirname, join, resolve } from "node:path";
|
|
19
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
20
|
+
import { homedir } from "node:os";
|
|
21
|
+
import { stageSharpIntoBundledAgents } from "../../../../../packages/desktop/scripts/sharp-package-assets.mjs";
|
|
22
|
+
import { stageKoffiIntoBundledAgents } from "../../../../../packages/desktop/scripts/koffi-package-assets.mjs";
|
|
23
|
+
|
|
24
|
+
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
25
|
+
|
|
26
|
+
const devPlugin = packageRoot;
|
|
27
|
+
// Build artifacts that constitute an installed plugin in the cache.
|
|
28
|
+
const ENTRIES = [
|
|
29
|
+
"skills",
|
|
30
|
+
"docs",
|
|
31
|
+
"scripts/computer-use-client.mjs",
|
|
32
|
+
".dcode-plugin",
|
|
33
|
+
"package.json",
|
|
34
|
+
// CUA screenshot/zoom 的 sharp native closure 属于 CUA plugin 自身;同步开发缓存时
|
|
35
|
+
// 不能只复制 skill/docs,否则下一次 cache seed 会再次丢失原生依赖。
|
|
36
|
+
"node_modules",
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Resolve the CLI cache dir for dcode-cua. Prefer the cachePath declared in the
|
|
41
|
+
* dcode-plugins-official marketplace.json; fall back to the single version dir under
|
|
42
|
+
* the cache root. Throws if neither yields an existing dir (plugin not installed).
|
|
43
|
+
*/
|
|
44
|
+
async function resolveCacheDir() {
|
|
45
|
+
if (process.argv[2]) return resolve(process.argv[2]);
|
|
46
|
+
const marketplace = join(
|
|
47
|
+
homedir(),
|
|
48
|
+
".dcode/cli/plugins/marketplaces/dcode-plugins-official/marketplace.json",
|
|
49
|
+
);
|
|
50
|
+
try {
|
|
51
|
+
const text = await readFile(marketplace, "utf8");
|
|
52
|
+
const parsed = JSON.parse(text);
|
|
53
|
+
const entry = (parsed.plugins ?? parsed.items ?? []).find(
|
|
54
|
+
(p) => p?.name === "computer-use" && p?.cachePath,
|
|
55
|
+
);
|
|
56
|
+
if (entry?.cachePath && existsSync(entry.cachePath)) return entry.cachePath;
|
|
57
|
+
} catch {
|
|
58
|
+
// marketplace.json missing/unreadable — fall through to glob.
|
|
59
|
+
}
|
|
60
|
+
const cacheRoot = join(homedir(), ".dcode/cli/plugins/cache/dcode-plugins-official/computer-use");
|
|
61
|
+
if (existsSync(cacheRoot)) {
|
|
62
|
+
const dirs = await readdir(cacheRoot);
|
|
63
|
+
if (dirs.length === 1) return join(cacheRoot, dirs[0]);
|
|
64
|
+
if (dirs.length > 1) {
|
|
65
|
+
// Pick the newest by mtime — the most recently installed version.
|
|
66
|
+
let newest = null;
|
|
67
|
+
for (const d of dirs) {
|
|
68
|
+
const s = await stat(join(cacheRoot, d));
|
|
69
|
+
if (!newest || s.mtimeMs > newest.mtimeMs) newest = { d, mtimeMs: s.mtimeMs };
|
|
70
|
+
}
|
|
71
|
+
if (newest) return join(cacheRoot, newest.d);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
throw new Error(
|
|
75
|
+
`Could not resolve the dcode-cua CLI cache dir. The plugin does not appear to be ` +
|
|
76
|
+
`installed — run it once from the dcode UI/TUI to install, then re-run sync:cache. ` +
|
|
77
|
+
`(looked in marketplace.json and ${cacheRoot})`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const cacheDir = await resolveCacheDir();
|
|
82
|
+
await mkdir(cacheDir, { recursive: true });
|
|
83
|
+
|
|
84
|
+
const copied = [];
|
|
85
|
+
for (const entry of ENTRIES) {
|
|
86
|
+
const src = join(devPlugin, entry);
|
|
87
|
+
if (!existsSync(src)) continue; // e.g. dist/ before first build
|
|
88
|
+
const dest = join(cacheDir, entry);
|
|
89
|
+
// cp with recursive + force replacement; file/dir both handled.
|
|
90
|
+
await cp(pathToFileURL(src), pathToFileURL(dest), { recursive: true, force: true });
|
|
91
|
+
copied.push(entry);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Bug 根因:开发态 CLI 实际从 ~/.dcode/cli/plugins/cache 启动 MCP,
|
|
95
|
+
// 只同步 dist 会让 externalized sharp 仍从一个隔离目录解析,导致 screenshot/zoom
|
|
96
|
+
// 在源码和 Helper 都正确时依旧报 MODULE_NOT_FOUND。与产品打包共用同一个 staging
|
|
97
|
+
// 函数,把当前平台的 JS/native 运行时放在插件自身 node_modules,避免两条链路漂移。
|
|
98
|
+
stageSharpIntoBundledAgents({
|
|
99
|
+
desktopPackageRoot: resolve(packageRoot, "../../../../packages/desktop"),
|
|
100
|
+
glmDir: cacheDir,
|
|
101
|
+
targetPlatform: { os: process.platform, arch: process.arch },
|
|
102
|
+
});
|
|
103
|
+
copied.push("node_modules/sharp runtime");
|
|
104
|
+
stageKoffiIntoBundledAgents({
|
|
105
|
+
koffiPackageRoot: resolve(packageRoot, "../adapters"),
|
|
106
|
+
glmDir: cacheDir,
|
|
107
|
+
targetPlatform: { os: process.platform, arch: process.arch },
|
|
108
|
+
});
|
|
109
|
+
copied.push("node_modules/koffi runtime");
|
|
110
|
+
|
|
111
|
+
console.log(`sync:cache → ${cacheDir}`);
|
|
112
|
+
console.log(` copied: ${copied.join(", ") || "(nothing)"}`);
|
|
113
|
+
console.log(
|
|
114
|
+
` note: the cache dir name is an install-time key and may lag the plugin.json ` +
|
|
115
|
+
`version inside — that is expected and does not affect loading.`,
|
|
116
|
+
);
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { execFile as execFileCallback } from "node:child_process";
|
|
3
|
+
import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, resolve } from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
|
|
8
|
+
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
9
|
+
const workspacePath = resolve(packageRoot, "../../../../pnpm-workspace.yaml");
|
|
10
|
+
const execFile = promisify(execFileCallback);
|
|
11
|
+
const defaultRepositoryRoot = resolve(packageRoot, "../../../../../dcode-cua");
|
|
12
|
+
const repositoryRoot = resolve(
|
|
13
|
+
process.argv[2] ?? process.env.DCODE_CUA_REPO ?? defaultRepositoryRoot,
|
|
14
|
+
);
|
|
15
|
+
const sourceSkill = resolve(repositoryRoot, "plugin/skills/computer-use/SKILL.md");
|
|
16
|
+
const sourceManifest = resolve(repositoryRoot, "plugin/.dcode-plugin/plugin.json");
|
|
17
|
+
const targetSkill = resolve(packageRoot, "skills/computer-use/SKILL.md");
|
|
18
|
+
const provenancePath = resolve(packageRoot, "upstream.json");
|
|
19
|
+
// 修复:bundled wrapper 的 plugin.json / package.json 之前各自维护 0.1.0,
|
|
20
|
+
// 与实际 dcode-cua MCP server 版本割裂(UI 展示、缓存路径、marketplace 条目都对不上)。
|
|
21
|
+
// 这里把上游 manifest 的 version 作为单一真相源,回写到 bundled wrapper,避免再次漂移。
|
|
22
|
+
const bundledManifestPath = resolve(packageRoot, ".dcode-plugin", "plugin.json");
|
|
23
|
+
const bundledPackageJsonPath = resolve(packageRoot, "package.json");
|
|
24
|
+
|
|
25
|
+
const [skillBytes, manifestText, workspaceText, revision, sourceStatus] = await Promise.all([
|
|
26
|
+
readFile(sourceSkill),
|
|
27
|
+
readFile(sourceManifest, "utf8"),
|
|
28
|
+
readFile(workspacePath, "utf8"),
|
|
29
|
+
execFile("git", ["-C", repositoryRoot, "rev-parse", "HEAD"], {
|
|
30
|
+
encoding: "utf8",
|
|
31
|
+
}),
|
|
32
|
+
execFile(
|
|
33
|
+
"git",
|
|
34
|
+
[
|
|
35
|
+
"-C",
|
|
36
|
+
repositoryRoot,
|
|
37
|
+
"status",
|
|
38
|
+
"--porcelain",
|
|
39
|
+
"--untracked-files=all",
|
|
40
|
+
"--",
|
|
41
|
+
"plugin/skills/computer-use/SKILL.md",
|
|
42
|
+
"plugin/.dcode-plugin/plugin.json",
|
|
43
|
+
],
|
|
44
|
+
{ encoding: "utf8" },
|
|
45
|
+
),
|
|
46
|
+
]);
|
|
47
|
+
const upstreamCommit = revision.stdout.trim();
|
|
48
|
+
// 不记录本地 worktree 的临时分支名;不可变 commit 同时是可重放的 ref。
|
|
49
|
+
const upstreamRef = upstreamCommit;
|
|
50
|
+
const pins = [
|
|
51
|
+
...workspaceText.matchAll(
|
|
52
|
+
/^\s+"@dcode\/dcode-cua":\s+"git\+https:\/\/dev\.aminer\.cn\/codegeex\/dcode-cua\.git#([0-9a-f]{40})"\s*$/gmu,
|
|
53
|
+
),
|
|
54
|
+
].map((match) => match[1]);
|
|
55
|
+
if (pins.length !== 1) {
|
|
56
|
+
throw new Error("The canonical dcode-cua catalog entry must pin one full producer commit");
|
|
57
|
+
}
|
|
58
|
+
if (!/^[0-9a-f]{40}$/u.test(upstreamCommit) || upstreamCommit !== pins[0]) {
|
|
59
|
+
throw new Error(
|
|
60
|
+
`Refusing to sync Skill from ${upstreamCommit || "unknown"}; catalog pins ${pins[0]}`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
if (!/^(?!.*\.\.)(?!\/)(?!.*\/$)[A-Za-z0-9._/-]+$/u.test(upstreamRef)) {
|
|
64
|
+
throw new Error(`Refusing to record invalid producer ref ${upstreamRef || "HEAD"}`);
|
|
65
|
+
}
|
|
66
|
+
if (sourceStatus.stdout.trim() !== "") {
|
|
67
|
+
throw new Error("Refusing to sync Skill or plugin manifest from uncommitted producer content");
|
|
68
|
+
}
|
|
69
|
+
const manifest = JSON.parse(manifestText);
|
|
70
|
+
if (manifest.name !== "computer-use" || typeof manifest.version !== "string") {
|
|
71
|
+
throw new Error(`Invalid computer-use plugin manifest at ${sourceManifest}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Patch a JSON file's top-level `version` field in place, preserving key order and
|
|
76
|
+
* 2-space indentation + trailing newline. Only rewrites when the value actually changes,
|
|
77
|
+
* so a no-op sync leaves the working tree clean.
|
|
78
|
+
*/
|
|
79
|
+
async function syncVersionField(filePath, version) {
|
|
80
|
+
const text = await readFile(filePath, "utf8");
|
|
81
|
+
const parsed = JSON.parse(text);
|
|
82
|
+
if (parsed.version === version) return false;
|
|
83
|
+
parsed.version = version;
|
|
84
|
+
await writeFile(filePath, `${JSON.stringify(parsed, null, 2)}\n`);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// 重命名后目标目录可能尚不存在(skills/<name>/),copyFile 不会自动建目录,先确保它存在。
|
|
89
|
+
await mkdir(dirname(targetSkill), { recursive: true });
|
|
90
|
+
await copyFile(sourceSkill, targetSkill);
|
|
91
|
+
await writeFile(
|
|
92
|
+
provenancePath,
|
|
93
|
+
`${JSON.stringify(
|
|
94
|
+
{
|
|
95
|
+
repository: "https://dev.aminer.cn/codegeex/dcode-cua.git",
|
|
96
|
+
ref: upstreamRef,
|
|
97
|
+
commit: upstreamCommit,
|
|
98
|
+
skillPath: "plugin/skills/computer-use/SKILL.md",
|
|
99
|
+
skillSha256: createHash("sha256").update(skillBytes).digest("hex"),
|
|
100
|
+
},
|
|
101
|
+
null,
|
|
102
|
+
2,
|
|
103
|
+
)}\n`,
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
const manifestUpdated = await syncVersionField(bundledManifestPath, manifest.version);
|
|
107
|
+
const packageJsonUpdated = await syncVersionField(bundledPackageJsonPath, manifest.version);
|
|
108
|
+
|
|
109
|
+
console.log(`Synced computer-use skill from dcode-cua ${manifest.version}`);
|
|
110
|
+
if (manifestUpdated || packageJsonUpdated) {
|
|
111
|
+
console.log(
|
|
112
|
+
`Aligned bundled wrapper version to ${manifest.version} ` +
|
|
113
|
+
`(plugin.json${manifestUpdated ? " updated" : ""}, package.json${packageJsonUpdated ? " updated" : ""})`,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: computer-use
|
|
3
|
+
description: "Use when a task needs a native desktop app's own UI or the OS. For anything inside a web page, use Browser Use. Main agent only."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Computer Use
|
|
7
|
+
|
|
8
|
+
Read or operate the UI of native apps on the user's computer.
|
|
9
|
+
|
|
10
|
+
- Prefer a dedicated connector, API, CLI or skill when one can complete the task.
|
|
11
|
+
- For browser and web tasks, use Browser Use instead.
|
|
12
|
+
- Do not use AppleScript, `osascript`, JXA, System Events, shell commands, or any
|
|
13
|
+
other UI-automation technology unless the user explicitly asks for that
|
|
14
|
+
technology.
|
|
15
|
+
- Main agent only. Never delegate Computer Use to a subagent.
|
|
16
|
+
|
|
17
|
+
## Bootstrap every call
|
|
18
|
+
|
|
19
|
+
Each `mcp__node_repl__js` call runs in a fresh Worker. Globals, imports, module
|
|
20
|
+
cache and any binding are gone by the next call, so `const app` does not survive
|
|
21
|
+
the end of the cell. The UI state does survive, so re-binding in the next cell is
|
|
22
|
+
cheap.
|
|
23
|
+
|
|
24
|
+
Put the bootstrap and the actions in the **same** cell, bootstrap first:
|
|
25
|
+
|
|
26
|
+
```js
|
|
27
|
+
const root =
|
|
28
|
+
process.env.DCODE_CUA_PLUGIN_ROOT ??
|
|
29
|
+
process.env.DCODE_PLUGIN_ROOT ??
|
|
30
|
+
process.env.CLAUDE_PLUGIN_ROOT;
|
|
31
|
+
const { join } = await import("node:path");
|
|
32
|
+
const { pathToFileURL } = await import("node:url");
|
|
33
|
+
const { setupComputerUseRuntime } = await import(
|
|
34
|
+
pathToFileURL(join(root, "scripts", "computer-use-client.mjs")).href,
|
|
35
|
+
);
|
|
36
|
+
await setupComputerUseRuntime({ globals: globalThis });
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Accessibility first
|
|
40
|
+
|
|
41
|
+
Accessibility is the primary action path. It is semantic, precise, works on a
|
|
42
|
+
background app and does not steal the user's focus.
|
|
43
|
+
|
|
44
|
+
1. Observe, then search the returned tree for the target by its role, name,
|
|
45
|
+
title, value or other visible identity.
|
|
46
|
+
2. When a matching element exists, act on it by **index**. For a control that
|
|
47
|
+
advertises a semantic action, `performSecondaryAction` is also correct.
|
|
48
|
+
3. For a settable element prefer `setValue` over typing or pasting. Reach for
|
|
49
|
+
`paste` only when the target is not settable or the content is rich text.
|
|
50
|
+
4. Keyboard input is the fallback: use `pressKey` only when no element expresses
|
|
51
|
+
the operation, or the user asked for keyboard interaction. Coordinates are the
|
|
52
|
+
last fallback, for canvas, games and Electron content accessibility cannot see.
|
|
53
|
+
|
|
54
|
+
Do not replace an available element action with a keyboard shortcut just because
|
|
55
|
+
the shortcut is shorter. Do not run both the accessibility and visual paths for
|
|
56
|
+
the same action.
|
|
57
|
+
|
|
58
|
+
**Observation works on a background app, screenshots included** — on any
|
|
59
|
+
display, including windows at negative coordinates.
|
|
60
|
+
`has_image: false` means the call did not ask for pixels (`getAXState`), never
|
|
61
|
+
that capture failed.
|
|
62
|
+
|
|
63
|
+
Success means the API accepted an action, not that the app acted. Typing into a
|
|
64
|
+
web-content editor can be accepted and change nothing, so re-observe to confirm
|
|
65
|
+
the text landed.
|
|
66
|
+
|
|
67
|
+
## API
|
|
68
|
+
|
|
69
|
+
```typescript
|
|
70
|
+
type Vec2 = [x: number, y: number];
|
|
71
|
+
type ObservationOptions = { emit?: boolean };
|
|
72
|
+
type StateOptions = ObservationOptions & { disableDiffing?: boolean };
|
|
73
|
+
type StateAndScreenshot = { state: string; screenshot?: Uint8Array };
|
|
74
|
+
type Direction = "up" | "down" | "left" | "right" | "u" | "d" | "l" | "r";
|
|
75
|
+
type MouseButton = "left" | "right" | "middle" | "l" | "r" | "m";
|
|
76
|
+
type SelectionType = "text" | "cursor_before" | "cursor_after";
|
|
77
|
+
type Strategy = "auto" | "a11y" | "event";
|
|
78
|
+
|
|
79
|
+
type ClickOptions = {
|
|
80
|
+
mouseButton?: MouseButton;
|
|
81
|
+
clickCount?: number;
|
|
82
|
+
modifiers?: string;
|
|
83
|
+
strategy?: Strategy;
|
|
84
|
+
};
|
|
85
|
+
type SelectTextOptions = { prefix?: string; suffix?: string; selectionType?: SelectionType };
|
|
86
|
+
type PasteOptions = { format?: "text" | "md" | "html" };
|
|
87
|
+
type PressKeyOptions = { holdSeconds?: number; strategy?: Strategy };
|
|
88
|
+
|
|
89
|
+
interface Target {
|
|
90
|
+
getAXState(options?: StateOptions): Promise<string>;
|
|
91
|
+
getScreenshot(options?: ObservationOptions): Promise<Uint8Array>;
|
|
92
|
+
getAXStateAndScreenshot(options?: StateOptions): Promise<StateAndScreenshot>;
|
|
93
|
+
elements(): Promise<{ index: number; kind: string; title: string | null; value: string | null; actions: string[] }[]>;
|
|
94
|
+
|
|
95
|
+
paste(text: string, options?: PasteOptions): Promise<void>;
|
|
96
|
+
click(target: number | Vec2, options?: ClickOptions): Promise<void>;
|
|
97
|
+
drag(from: number | Vec2, to: number | Vec2, options?: { modifiers?: string }): Promise<void>;
|
|
98
|
+
pressKey(key: string, options?: PressKeyOptions): Promise<void>;
|
|
99
|
+
scroll(target: number | Vec2, direction: Direction, pages?: number): Promise<void>;
|
|
100
|
+
selectText(elementIndex: number, text: string, options?: SelectTextOptions): Promise<void>;
|
|
101
|
+
setValue(elementIndex: number, value: string): Promise<void>;
|
|
102
|
+
typeText(text: string): Promise<void>;
|
|
103
|
+
performSecondaryAction(elementIndex: number, action: string): Promise<void>;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
interface App extends Target {}
|
|
107
|
+
|
|
108
|
+
type AppRef = { name?: string; bundle_id?: string; pid?: number; window_id?: number };
|
|
109
|
+
type AppInfo = { pid: number; name: string | null; bundle_id: string | null; active: boolean };
|
|
110
|
+
type State = { apps: AppInfo[] };
|
|
111
|
+
|
|
112
|
+
declare const agent: {
|
|
113
|
+
computerUse: {
|
|
114
|
+
getState(options?: ObservationOptions): Promise<State>;
|
|
115
|
+
getApp(target: string | AppRef): Promise<App>;
|
|
116
|
+
listApps(options?: ObservationOptions): Promise<AppInfo[]>;
|
|
117
|
+
computer: Record<string, (args: object) => Promise<unknown>>;
|
|
118
|
+
|
|
119
|
+
requestAccess(capabilities?: string[]): Promise<Record<string, unknown>>;
|
|
120
|
+
stop(reason?: string): Promise<void>;
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The full reference is `agent.documentation.get("computer-use")`, fetched on demand;
|
|
126
|
+
read it only for an argument shape or response field this page does not give.
|
|
127
|
+
|
|
128
|
+
## The loop
|
|
129
|
+
|
|
130
|
+
Observe once, act, then observe again before deciding the next step.
|
|
131
|
+
|
|
132
|
+
`agent.computerUse.getApp(...)` binds an app and shows nothing; call `getAXState`
|
|
133
|
+
when you need to see its state. Binding also launches it if not running; there is
|
|
134
|
+
no separate launch tool. Its argument is a display name or a bundle identifier —
|
|
135
|
+
the same strings `listApps()` returns — or an `AppRef`.
|
|
136
|
+
|
|
137
|
+
When the user names an application, copy it character-for-character into the app
|
|
138
|
+
identifier. Do not translate, localize, normalize, shorten, or remove a suffix:
|
|
139
|
+
`{"name":"网易云音乐app"}` is not `{"name":"网易云音乐"}`; `{"name":"日历"}` is not
|
|
140
|
+
`{"name":"Calendar"}`. A rewritten name resolves to a different app or to nothing,
|
|
141
|
+
and the failure reads "app not found". If the exact string does not resolve, call
|
|
142
|
+
`agent.computerUse.listApps()` once and pick the matching identifier.
|
|
143
|
+
|
|
144
|
+
Batch related actions and a single closing observation into one cell:
|
|
145
|
+
|
|
146
|
+
```js
|
|
147
|
+
const app = await agent.computerUse.getApp("Notes");
|
|
148
|
+
await app.click(box);
|
|
149
|
+
await app.typeText("hello");
|
|
150
|
+
await app.pressKey("Return");
|
|
151
|
+
await app.getAXState();
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
An element index addresses that app's latest observation, so several actions may
|
|
155
|
+
reuse one index without re-observing between them — click an index, then type into
|
|
156
|
+
it, in the same cell. Observing renumbers the tree, so take indices from the newest
|
|
157
|
+
one. A vanished element fails closed with `ELEMENT_UNAVAILABLE`.
|
|
158
|
+
|
|
159
|
+
The tree comes back as a diff only against a tree this cell already showed you,
|
|
160
|
+
listing the elements that were removed, added or changed; unchanged rows are
|
|
161
|
+
omitted and their indices stay valid. The first tree after binding, and the first
|
|
162
|
+
after `getScreenshot` or `elements()`, are always complete. Pass
|
|
163
|
+
`{ disableDiffing: true }` for a full tree at any point. If a standalone
|
|
164
|
+
observation reports no change, do not immediately repeat it without an
|
|
165
|
+
intervening action.
|
|
166
|
+
|
|
167
|
+
A large tree is trimmed by priority (ancestors kept), the header says so, and
|
|
168
|
+
indices then skip numbers. `app.elements()` returns every element with its index,
|
|
169
|
+
trimmed ones included — filter it in JS, never guess an index.
|
|
170
|
+
|
|
171
|
+
A capture is scoped to one window: without a `window_id` the main/key window is
|
|
172
|
+
re-resolved every observation, so a modal that just opened becomes the captured
|
|
173
|
+
window. When an action fails or the tree reads like another part of the app, check
|
|
174
|
+
the observation's `window` — a dialog shows up there. Read it, then act on it, or
|
|
175
|
+
dismiss it (Escape or its own cancel) and observe again. A missing element is not
|
|
176
|
+
proof the action worked. Pin one with `getApp({pid, window_id})`; `list_windows`
|
|
177
|
+
has the id.
|
|
178
|
+
|
|
179
|
+
## Output
|
|
180
|
+
|
|
181
|
+
Observations display themselves: `getAXState`, `getScreenshot`,
|
|
182
|
+
`getAXStateAndScreenshot`, `getState` and `listApps` emit their own result.
|
|
183
|
+
Never pass their return value to `nodeRepl.write(...)` or
|
|
184
|
+
`nodeRepl.emitImage(...)`: a second raster in one result breaks the one-raster
|
|
185
|
+
rule and the frame is removed entirely, so you end up with no picture at all.
|
|
186
|
+
Pass `{ emit: false }` to suppress the display and still receive the value.
|
|
187
|
+
|
|
188
|
+
Action methods display nothing.
|
|
189
|
+
|
|
190
|
+
## Coordinates
|
|
191
|
+
|
|
192
|
+
Choose `x` and `y` only by looking at the current returned raster, with
|
|
193
|
+
`0 <= x < width` and `0 <= y < height` for that raster. Submit those integers
|
|
194
|
+
unchanged; CUA binds the current raster internally and owns every transform from
|
|
195
|
+
the returned raster to native dispatch. Element and window bounds are diagnostic
|
|
196
|
+
global screen points and must never be copied into a coordinate. When visual
|
|
197
|
+
fallback begins, discard coordinate-like numbers from earlier text or
|
|
198
|
+
accessibility results.
|
|
199
|
+
|
|
200
|
+
Act on the current raster when the target is clear; if it is too small or ambiguous,
|
|
201
|
+
re-observe rather than guessing at geometry.
|
|
202
|
+
|
|
203
|
+
A pointer action accepted with no change usually means the app acted where the real
|
|
204
|
+
pointer sits: repeating it will not help — use an element index or the keyboard.
|
|
205
|
+
|
|
206
|
+
A coordinate refusal may name an unexpected owner, or say the frame is stale — the
|
|
207
|
+
window moved, resized or was replaced. Observe again for a current raster, or act on
|
|
208
|
+
an element index, which does not depend on window geometry. Never move, resize or
|
|
209
|
+
close a window to make a coordinate land.
|
|
210
|
+
|
|
211
|
+
## Keyboard
|
|
212
|
+
|
|
213
|
+
`pressKey` takes a key or a `+`-separated chord and accepts both short names and
|
|
214
|
+
X keysym style: `"a"`, `"Return"`, `"Tab"`, `"Control_L+a"`, `"super+c"`, `"Up"`.
|
|
215
|
+
macOS uses `cmd`; Linux and Windows use `ctrl`. Use `{ holdSeconds }` to hold a
|
|
216
|
+
key or chord for a duration rather than simulating repeated presses.
|
|
217
|
+
|
|
218
|
+
Bind keyboard input to an app or element; never send it unbound. `paste` is for
|
|
219
|
+
rich text, or a target `setValue` cannot set — not for plain text a settable
|
|
220
|
+
element accepts.
|
|
221
|
+
|
|
222
|
+
`performSecondaryAction` accepts only an action the element advertises in the
|
|
223
|
+
current tree. Do not guess an action name.
|
|
224
|
+
|
|
225
|
+
At most one element is `focused` — the one holding keyboard focus; absent means
|
|
226
|
+
undetermined.
|
|
227
|
+
|
|
228
|
+
`selectText` locates text inside an editable element. Use `prefix`/`suffix` to
|
|
229
|
+
disambiguate repeated matches and `selectionType` to place the cursor instead of
|
|
230
|
+
selecting. An ambiguous match is refused rather than resolved to the first hit.
|
|
231
|
+
|
|
232
|
+
## Waiting
|
|
233
|
+
|
|
234
|
+
Observations wait for the UI to settle before capturing. Do not pause or delay
|
|
235
|
+
before reading state — no `setTimeout`, no polling loop.
|
|
236
|
+
|
|
237
|
+
## Errors and stopping
|
|
238
|
+
|
|
239
|
+
Actions resolve to `undefined` on success and throw `ComputerUseError` otherwise:
|
|
240
|
+
|
|
241
|
+
- `code` — `PERMISSION_DENIED`, `NOT_AUTHORIZED`, `APP_NOT_FOUND`,
|
|
242
|
+
`AMBIGUOUS_APP`, `LAUNCH_FAILED`, `INVALID_APP`, `ELEMENT_UNAVAILABLE`,
|
|
243
|
+
`STALE_STATE`, `NOT_SETTABLE`, `NOT_SELECTABLE`, `ACTION_UNAVAILABLE`,
|
|
244
|
+
`FOREGROUND_REQUIRED`, `CONTROLLER_BUSY`, `CONTROL_STOPPED`, `SCREEN_LOCKED`,
|
|
245
|
+
`HELPER_UNAVAILABLE`, `VERSION_MISMATCH`, `TIMEOUT`,
|
|
246
|
+
`STRUCTURED_STATE_UNAVAILABLE`, `INTERNAL`.
|
|
247
|
+
- `actionSent` — whether the action may already have reached the app. Retry a
|
|
248
|
+
non-idempotent action only when this is `false`; otherwise observe first and
|
|
249
|
+
decide from what you see.
|
|
250
|
+
- `retry` — `"reobserve"`, `"retry"` or `"never"`.
|
|
251
|
+
|
|
252
|
+
`CONTROLLER_BUSY` means another live DCode Computer Use session owns input. It is
|
|
253
|
+
never retryable: report the owner from the error and ask the user to close that
|
|
254
|
+
session.
|
|
255
|
+
|
|
256
|
+
Stop immediately after `agent.computerUse.stop()`, a kill switch, a permission
|
|
257
|
+
refusal, or a non-retryable error. Do not switch to a different UI-automation
|
|
258
|
+
technology after an access refusal.
|
|
259
|
+
|
|
260
|
+
Persist until the request is actually complete. Attempting an action is not
|
|
261
|
+
completion: verify the returned state visibly shows the result. If it is unchanged
|
|
262
|
+
or only intermediate, try another approach. Respond only when the requested state
|
|
263
|
+
is visibly present, or explain a concrete blocker you cannot resolve.
|
|
264
|
+
|
|
265
|
+
## Tool surface
|
|
266
|
+
|
|
267
|
+
`agent.computerUse.computer.<tool>(args)` is the low-level surface. Prefer the
|
|
268
|
+
bound-object API above; reach for a tool only for what the API does not
|
|
269
|
+
express — window enumeration, key repeat, or reading state back in the same
|
|
270
|
+
call. Arguments are strict: an undeclared key is refused. Each tool takes **one**
|
|
271
|
+
arguments object — the names below are its keys, not positional parameters:
|
|
272
|
+
`get_app_state({ app_ref: { bundle_id: "com.apple.Notes" }, include_screenshot: true })`.
|
|
273
|
+
|
|
274
|
+
Name an app as the OS lists it (Windows: the Start-menu name, not a window title).
|
|
275
|
+
Nothing takes the user's focus, except on Windows: launching a non-packaged app
|
|
276
|
+
does, and `include_screenshot=true` un-minimizes — a minimized tree is fully
|
|
277
|
+
usable, so keep the default.
|
|
278
|
+
|
|
279
|
+
```
|
|
280
|
+
list_apps({})
|
|
281
|
+
list_windows({app_ref})
|
|
282
|
+
get_app_state({app_ref, include_screenshot?=false, disable_diffing?=false})
|
|
283
|
+
|
|
284
|
+
left_click({target, mouse_button?="left", click_count?=1, modifiers?="",
|
|
285
|
+
strategy?, app_ref?, return_state?})
|
|
286
|
+
left_click_drag({from_target, to, modifiers?="", app_ref?, return_state?})
|
|
287
|
+
scroll({target, scroll_direction, scroll_amount, strategy?, app_ref?,
|
|
288
|
+
return_state?})
|
|
289
|
+
|
|
290
|
+
type({text, target?, app_ref?, strategy?, return_state?})
|
|
291
|
+
set_value({target, value, strategy?, app_ref?, return_state?})
|
|
292
|
+
select_text({target, text_range?, app_ref?, return_state?})
|
|
293
|
+
key({text, repeat?, hold_seconds?, app_ref?, strategy?, return_state?})
|
|
294
|
+
paste({text, format?="text", app_ref?, return_state?})
|
|
295
|
+
perform_action({target, action, app_ref?, return_state?})
|
|
296
|
+
|
|
297
|
+
request_access({capabilities?})
|
|
298
|
+
stop_computer_control({reason?})
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Argument shapes: `app_ref` / `app` is an `AppRef`, but a bare string here is
|
|
302
|
+
read as a bundle id, so pass `{name: "Notes"}` for a display name.
|
|
303
|
+
`scroll_direction` is `up|down|left|right`, `scroll_amount` is pages, `strategy`
|
|
304
|
+
is `auto|a11y|event`, and `return_state` is `compact|full|none` to return the app
|
|
305
|
+
state in the same call. For `target`, `text_range`, `modifiers` and the response
|
|
306
|
+
shapes, see `nodeRepl.write(await agent.documentation.get("computer-use"))`.
|
|
@@ -52622,14 +52622,14 @@ function rewriteDynamicImportsForRepl(code) {
|
|
|
52622
52622
|
function nodeEnd(node2) {
|
|
52623
52623
|
const end = node2.end ?? node2.range?.[1];
|
|
52624
52624
|
if (end === void 0) {
|
|
52625
|
-
throw new Error("instrument
|
|
52625
|
+
throw new Error("instrument requires a node end (parseReplCode must be called with ranges:true)");
|
|
52626
52626
|
}
|
|
52627
52627
|
return end;
|
|
52628
52628
|
}
|
|
52629
52629
|
function nodeStart(node2) {
|
|
52630
52630
|
const start = node2.start ?? node2.range?.[0];
|
|
52631
52631
|
if (start === void 0) {
|
|
52632
|
-
throw new Error("instrument
|
|
52632
|
+
throw new Error("instrument requires a node start (parseReplCode must be called with ranges:true)");
|
|
52633
52633
|
}
|
|
52634
52634
|
return start;
|
|
52635
52635
|
}
|
|
@@ -52991,7 +52991,7 @@ var NodeReplSession = class {
|
|
|
52991
52991
|
if (this.disposed) {
|
|
52992
52992
|
return {
|
|
52993
52993
|
logs: "",
|
|
52994
|
-
error: { name: "DisposedError", message: "REPL session
|
|
52994
|
+
error: { name: "DisposedError", message: "REPL session is disposed" }
|
|
52995
52995
|
};
|
|
52996
52996
|
}
|
|
52997
52997
|
let buffer = "";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dcode/node-repl-host",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Shared node_repl MCP host. Owns the JS execution surface and the domain bridges (Browser Use, Computer Use) that ride on it; the official plugins contribute only their own skills, docs and runtime assets.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|