@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,59 @@
1
+ // Probe actual packaged bits, not Swift source or an injected foreground.json.
2
+ // Optional first argument checks a different installed/packaged executable.
3
+ import assert from 'node:assert/strict';
4
+ import { execFileSync, spawn } from 'node:child_process';
5
+ import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs';
6
+ import net from 'node:net';
7
+ import os from 'node:os';
8
+ import path from 'node:path';
9
+ import { appName, installMacosApp } from './macos-bundle.mjs';
10
+
11
+ if (process.platform !== 'darwin') { console.log('SKIP macOS native helper'); process.exit(0); }
12
+ const root = path.resolve(import.meta.dirname, '..');
13
+ const temp = mkdtempSync(path.join(os.tmpdir(), 'cu-signal-')), socket = path.join(temp, 'bridge.sock');
14
+ const app = process.argv[2] ? undefined : path.join(temp, appName);
15
+ const executable = process.argv[2] || path.join(app, 'Contents/MacOS/bridge');
16
+ const pause = ms => new Promise(r => setTimeout(r, ms));
17
+ function request(cmd) {
18
+ return new Promise((resolve, reject) => {
19
+ const s = net.createConnection(socket); let data = '';
20
+ const finish = (error, result) => { clearTimeout(timer); s.destroy(); error ? reject(error) : resolve(result); };
21
+ const timer = setTimeout(() => finish(new Error('helper timeout')), 2000);
22
+ s.on('connect', () => s.write(JSON.stringify({ id: 'signal-probe', cmd }) + '\n'));
23
+ s.on('error', error => finish(error));
24
+ s.on('data', chunk => { data += chunk; if (data.includes('\n')) { try { finish(null, JSON.parse(data.split('\n')[0])); } catch(error) { finish(error); } } });
25
+ });
26
+ }
27
+ let child, spawnError;
28
+ try {
29
+ if (app) await installMacosApp(root, app, { register: false });
30
+ assert.ok(existsSync(executable), 'bundle must ship a macOS helper');
31
+ child = app
32
+ ? spawn('/usr/bin/open', ['-n', '-g', app, '--args', 'serve', '--socket', socket], { stdio: 'ignore' })
33
+ : spawn(executable, ['serve', '--socket', socket], { stdio: 'ignore' });
34
+ child.on('error', error => { spawnError = error; });
35
+ let diagnostics;
36
+ for (let i = 0; i < 50; i++) {
37
+ if (spawnError) throw spawnError;
38
+ try { diagnostics = await request('diagnostics'); break; } catch { await pause(100); }
39
+ }
40
+ assert.equal(diagnostics?.ok, true, 'native helper must start');
41
+ const status = await request('permissionStatus');
42
+ assert.equal(status.ok, true);
43
+ if (app) {
44
+ assert.equal(status.result.source.attribution, 'helper-app', 'LaunchServices must attribute permissions to the installed helper app');
45
+ assert.equal(realpathSync(status.result.source.executablePath), realpathSync(executable));
46
+ }
47
+ const signal = JSON.parse(readFileSync(path.join(temp, 'foreground.json'), 'utf8'));
48
+ assert.equal(signal.v, 1); assert.equal(signal.active, false); assert.equal(signal.helperPid, status.result.source.pid);
49
+ assert.ok(Math.abs(Date.now() - signal.updatedAt) < 10000, 'signal belongs to this startup');
50
+ assert.ok(signal.expiresAt <= Date.now(), 'idle startup cannot open Mini');
51
+ console.log(`PASS native helper publishes an idle Mini signal (protocol ${diagnostics.result.protocolVersion})`);
52
+ } finally {
53
+ await request('shutdown').catch(() => {}); await pause(300);
54
+ if (child && child.exitCode === null) child.kill('SIGTERM'); // Only our isolated probe/launcher, never a user daemon.
55
+ if (app) {
56
+ try { execFileSync('/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister', ['-u', app], { stdio: 'ignore' }); } catch {}
57
+ }
58
+ rmSync(temp, { recursive: true, force: true });
59
+ }
@@ -0,0 +1,32 @@
1
+ #!/usr/bin/env node
2
+ // 边缘光效的几何单测(照上游 INV-17 跑 agent_cursor_tests 的做法:临时编一个 @main 可执行文件跑一遍)。
3
+ // 非 macOS 直接跳过 —— 光效本来就只有 mac 有。
4
+ import { execFileSync } from 'node:child_process';
5
+ import fs from 'node:fs';
6
+ import os from 'node:os';
7
+ import path from 'node:path';
8
+ import { fileURLToPath } from 'node:url';
9
+
10
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
11
+
12
+ if (process.platform !== 'darwin') {
13
+ console.log('highlight check skipped (not macOS)');
14
+ process.exit(0);
15
+ }
16
+
17
+ const triple = process.arch === 'x64' ? 'x86_64-apple-macosx14.0' : 'arm64-apple-macosx14.0';
18
+ const binary = path.join(os.tmpdir(), `tangu-cu-highlight-tests-${process.pid}`);
19
+ try {
20
+ execFileSync('xcrun', [
21
+ 'swiftc', '-target', triple, '-parse-as-library',
22
+ '-module-cache-path', path.join(os.tmpdir(), `tangu-cu-highlight-test-cache-${process.arch}`),
23
+ '-framework', 'AppKit',
24
+ '-framework', 'SwiftUI',
25
+ 'native/macos/agent_highlight.swift',
26
+ 'native/macos/agent_highlight_tests.swift',
27
+ '-o', binary,
28
+ ], { cwd: root, stdio: 'inherit' });
29
+ execFileSync(binary, [], { cwd: root, stdio: 'inherit' });
30
+ } finally {
31
+ fs.rmSync(binary, { force: true });
32
+ }
@@ -0,0 +1,140 @@
1
+ #!/usr/bin/env node
2
+ // Real install/repair checks in disposable directories. A Node preload traps all
3
+ // process launches, including absolute paths, before production modules load.
4
+ // No keychain access, LS registration, system permission prompt or user app.
5
+ import assert from 'node:assert/strict';
6
+ import { spawnSync, spawn } from 'node:child_process';
7
+ import fs from 'node:fs/promises';
8
+ import os from 'node:os';
9
+ import path from 'node:path';
10
+ import { fileURLToPath, pathToFileURL } from 'node:url';
11
+ import { bundledMacosApp, sha256, macosHelperIsCurrent, appName } from './macos-bundle.mjs';
12
+
13
+ if (process.platform !== 'darwin') {
14
+ console.log('SKIP real macOS installer checks');
15
+ process.exit(0);
16
+ }
17
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
18
+ const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'cu-keychain-free-'));
19
+ try {
20
+ const trace = path.join(temp, 'commands.jsonl'), guard = path.join(temp, 'guard.mjs');
21
+ await fs.writeFile(guard, `import cp from 'node:child_process';
22
+ import fs from 'node:fs/promises';
23
+ import {appendFileSync} from 'node:fs';
24
+ import {syncBuiltinESMExports} from 'node:module';
25
+ import path from 'node:path';
26
+ for (const method of ['spawn','spawnSync','exec','execSync','execFile','execFileSync']) {
27
+ const original = cp[method];
28
+ cp[method] = function(command, args, ...rest) {
29
+ const name = path.basename(command);
30
+ appendFileSync(process.env.CU_TEST_TRACE, JSON.stringify({method,command,args})+'\\n');
31
+ if (['security','openssl','xcrun'].includes(name) || method === 'exec' || method === 'execSync'
32
+ || (name === 'codesign' && (!Array.isArray(args) || args[0] !== '--verify'))) {
33
+ throw Error('Forbidden runtime signing/keychain command: '+command);
34
+ }
35
+ if (name === 'lsregister') return Buffer.alloc(0);
36
+ return original.call(this, command, args, ...rest);
37
+ };
38
+ }
39
+ syncBuiltinESMExports();
40
+ if (process.env.CU_TEST_RENAME_FAIL === '1') {
41
+ const rename = fs.rename;
42
+ fs.rename = async (from, to) => {
43
+ if (path.basename(from) === 'tangu-computer-use.app' && to === process.env.PI_COMPUTER_USE_HELPER_APP_PATH) {
44
+ throw Object.assign(Error('fixture replacement failure'), {code:'EACCES'});
45
+ }
46
+ return rename(from, to);
47
+ };
48
+ }
49
+ `);
50
+ const installed = path.join(temp, 'Applications', appName);
51
+ const env = { ...process.env, CU_TEST_TRACE: trace, PI_COMPUTER_USE_HELPER_APP_PATH: installed,
52
+ // These old knobs must not opt runtime back into signing or key discovery.
53
+ PI_COMPUTER_USE_CODESIGN_IDENTITY: 'must-never-use-user-key', PI_COMPUTER_USE_NO_SIGN: '0',
54
+ PI_COMPUTER_USE_ALLOW_BUILD: '1' };
55
+ function run(flags = [], source = root, extraEnv = {}) {
56
+ const result = spawnSync(process.execPath, ['--import', pathToFileURL(guard).href, path.join(source, 'scripts/setup-helper.mjs'), ...flags],
57
+ { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 45_000 });
58
+ assert.ifError(result.error);
59
+ return result;
60
+ }
61
+ const install = () => { const result = run(['--runtime']); assert.equal(result.status, 0, result.stderr); };
62
+ const check = status => { const result = run(['--check']); assert.equal(result.status, status, result.stderr); };
63
+ const image = () => fs.readFile(path.join(installed, 'Contents/MacOS/bridge')).then(sha256);
64
+
65
+ for (const arch of ['arm64', 'x64']) {
66
+ const bundle = bundledMacosApp(root, arch);
67
+ assert.equal(sha256(await fs.readFile(bundle.archive)), bundle.metadata.archiveSha256);
68
+ }
69
+ check(10);
70
+ install();
71
+ check(0);
72
+ assert.ok(macosHelperIsCurrent(root, installed));
73
+ const inode = (await fs.stat(installed)).ino, expected = await image();
74
+ install();
75
+ assert.equal((await fs.stat(installed)).ino, inode, 'current app must not be replaced');
76
+ console.log('PASS first install / idempotent reinstall / current check');
77
+
78
+ await fs.rm(path.join(installed, 'Contents/_CodeSignature'), { recursive: true });
79
+ await fs.writeFile(path.join(installed, 'Contents/MacOS/bridge.cstemp'), 'interrupted old codesign');
80
+ check(10); install(); check(0);
81
+ assert.equal(await fs.access(path.join(installed, 'Contents/MacOS/bridge.cstemp')).then(() => true, () => false), false);
82
+ assert.equal(await image(), expected);
83
+ console.log('PASS rejected-keychain partial app repaired, including signature/temp residue');
84
+
85
+ const info = path.join(installed, 'Contents/Info.plist');
86
+ const packageVersion = JSON.parse(await fs.readFile(path.join(root, 'package.json'), 'utf8')).version;
87
+ await fs.writeFile(info, (await fs.readFile(info, 'utf8')).replaceAll(packageVersion, '0.0.0'));
88
+ check(10); install(); check(0);
89
+ console.log('PASS same-protocol old app upgraded');
90
+
91
+ await fs.appendFile(info, '<!-- old installation -->');
92
+ const previousInfo = await fs.readFile(info, 'utf8');
93
+ const failedReplacement = run(['--runtime'], root, { CU_TEST_RENAME_FAIL: '1' });
94
+ assert.notEqual(failedReplacement.status, 0);
95
+ assert.match(failedReplacement.stderr, /fixture replacement failure/);
96
+ assert.equal(await fs.readFile(info, 'utf8'), previousInfo, 'replacement failure must restore previous app');
97
+ install(); check(0);
98
+ console.log('PASS atomic replacement rollback preserves previous app');
99
+
100
+ // Bad/missing bundled assets must fail without changing an existing app and
101
+ // without falling back to loose-binary signing, keychain discovery or downloads.
102
+ const bad = path.join(temp, 'bad-package');
103
+ await fs.mkdir(bad);
104
+ for (const rel of ['scripts', 'src/vendor/platform/macos', 'prebuilt/macos', 'package.json']) {
105
+ await fs.cp(path.join(root, rel), path.join(bad, rel), { recursive: true });
106
+ }
107
+ const badBundle = bundledMacosApp(bad);
108
+ await fs.appendFile(badBundle.archive, 'corruption');
109
+ const corrupted = run(['--runtime'], bad);
110
+ assert.notEqual(corrupted.status, 0); assert.match(corrupted.stderr, /archive is damaged/);
111
+ assert.equal(await image(), expected); check(0);
112
+ await fs.rm(path.join(bad, 'prebuilt/macos'), { recursive: true });
113
+ await fs.mkdir(path.join(bad, 'prebuilt/macos', process.arch), { recursive: true });
114
+ await fs.copyFile(path.join(root, 'prebuilt/macos', process.arch, 'bridge'), path.join(bad, 'prebuilt/macos', process.arch, 'bridge'));
115
+ const missing = run(['--runtime'], bad);
116
+ assert.notEqual(missing.status, 0); assert.match(missing.stderr, /pre-signed.*missing/);
117
+ assert.equal(await image(), expected); check(0);
118
+ console.log('PASS corrupt/missing archives fail closed and preserve existing app');
119
+
120
+ await fs.rm(path.join(installed, 'Contents/_CodeSignature'), { recursive: true });
121
+ const staleLock = `${installed}.install-lock`;
122
+ await fs.mkdir(staleLock);
123
+ await fs.writeFile(path.join(staleLock, 'pid'), '0');
124
+ await fs.utimes(staleLock, new Date(Date.now() - 60_000), new Date(Date.now() - 60_000));
125
+ const concurrent = () => new Promise((resolve, reject) => {
126
+ const child = spawn(process.execPath, ['--import', pathToFileURL(guard).href, path.join(root, 'scripts/setup-helper.mjs'), '--runtime'], { env, stdio: ['ignore','pipe','pipe'] });
127
+ let stderr = ''; child.stderr.on('data', data => { stderr += data; });
128
+ child.on('error', reject); child.on('close', code => code === 0 ? resolve() : reject(new Error(stderr)));
129
+ });
130
+ await Promise.all([concurrent(), concurrent()]);
131
+ check(0);
132
+ assert.deepEqual(await fs.readdir(path.dirname(installed)), [appName], 'no staging or lock residue');
133
+ const commands = (await fs.readFile(trace, 'utf8')).trim().split('\n').map(line => JSON.parse(line));
134
+ assert.ok(commands.some(cmd => path.basename(cmd.command) === 'codesign'));
135
+ assert.ok(commands.every(cmd => !['security','openssl','xcrun'].includes(path.basename(cmd.command))
136
+ && (path.basename(cmd.command) !== 'codesign' || cmd.args[0] === '--verify')));
137
+ console.log('PASS concurrent repair / stale-lock recovery / no runtime signing or keychain commands');
138
+ } finally {
139
+ await fs.rm(temp, { recursive: true, force: true });
140
+ }
@@ -0,0 +1,124 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * 「实时画面」的真机仪器 —— 证明 liveView 走的是**常驻取景流**而不是每帧现截,而且画面真的在动。
4
+ *
5
+ * 症状是"不是 share screen 的实时画面,是掉帧的"。掉帧有两种完全不同的成因,这个脚本分开钉:
6
+ * ① **取一帧要多久**(source=snapshot 时每帧都要重新协商采集会话,几十到几百毫秒不等)
7
+ * ② **帧到底新不新**(流没起来 / 只推 .idle 空帧的话,拿到的会是同一张,再快也是静止画)
8
+ * 所以既量延迟也量**相邻帧的字节差异** —— 只量其一都能被另一种失败蒙混过去。
9
+ *
10
+ * 跑法:node scripts/live-view.check.mjs
11
+ * 需要:macOS、helper 已装并授权(辅助功能 + 屏幕录制)。会短暂打开「计算器」,跑完关掉。
12
+ */
13
+ import assert from 'node:assert/strict'
14
+ import { createHash } from 'node:crypto'
15
+ import { connect } from 'node:net'
16
+ import { homedir } from 'node:os'
17
+ import { ensureDaemon, freshCalculator, killCalculator } from './calc-fixture.mjs'
18
+
19
+ const SOCK = `${homedir()}/Library/Caches/tangu-computer-use/bridge.sock`
20
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
21
+ const FRAMES = 24
22
+ const TICK_MS = 120 // 与 main.js 的轮询节奏一致
23
+
24
+ let seq = 0
25
+ function ask(payload) {
26
+ return new Promise((resolve, reject) => {
27
+ const sock = connect(SOCK)
28
+ let buf = ''
29
+ const timer = setTimeout(() => { sock.destroy(); reject(new Error(`timeout: ${payload.cmd}`)) }, 20_000)
30
+ sock.on('error', (e) => { clearTimeout(timer); reject(e) })
31
+ sock.on('data', (d) => {
32
+ buf += d
33
+ const nl = buf.indexOf('\n')
34
+ if (nl < 0) return
35
+ clearTimeout(timer)
36
+ sock.end()
37
+ const msg = JSON.parse(buf.slice(0, nl))
38
+ if (!msg.ok) return reject(Object.assign(new Error(msg.error?.message || 'helper error'), { code: msg.error?.code }))
39
+ resolve(msg.result)
40
+ })
41
+ sock.write(`${JSON.stringify({ id: `lv_${++seq}`, ...payload })}\n`)
42
+ })
43
+ }
44
+
45
+ const findNode = (node, pred) => {
46
+ if (!node) return null
47
+ if (pred(node)) return node
48
+ for (const c of node.children || []) { const hit = findNode(c, pred); if (hit) return hit }
49
+ return null
50
+ }
51
+ const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]
52
+
53
+ const diag = await ensureDaemon(ask)
54
+ assert.equal(diag.protocolVersion, 11, `helper 协议应为 11,实测 ${diag.protocolVersion}`)
55
+ assert.ok(diag.screenRecording, '需要「屏幕录制」权限,否则拿不到任何画面')
56
+
57
+ try {
58
+ const { calc, win } = await freshCalculator(ask)
59
+ const look = await ask({ cmd: 'look', pid: calc.pid, windowId: win.windowId, includeImage: true })
60
+ const keyOf = (d) => {
61
+ const n = findNode(look.outline, (x) => x.role === 'AXButton' && (x.description === d || x.title === d))
62
+ assert.ok(n, `没找到数字键 ${d}`)
63
+ return { x: n.rect.x + n.rect.w / 2, y: n.rect.y + n.rect.h / 2 }
64
+ }
65
+ // ⚠️policy 必须是 background:这里只是要把窗口内容点变以便验证帧在更新,
66
+ // 用 foreground/hid 的话仪器每跑一次就抢 12 次用户焦点、还把实物鼠标拽走 ——
67
+ // 测「不抢前台」的项目自己抢前台,荒唐。后台 AX 按压点得中数字键(check:blindclick 已证明)。
68
+ const click = (d) => ask({
69
+ cmd: 'act', lookId: look.lookId, pid: calc.pid, policy: 'background',
70
+ cursorOverlay: true, action: 'click', target: keyOf(d), params: { button: 'left', clickCount: 1 },
71
+ })
72
+
73
+ // liveView 只认"最近被操控过的窗口" —— 先动一下,它才有东西可播
74
+ await click('1')
75
+ await sleep(300)
76
+
77
+ const frames = []
78
+ for (let i = 0; i < FRAMES; i++) {
79
+ // 每帧之间改一下计算器的读数,窗口内容才会**真的**变 —— 否则静止窗口本来就该是同一张,
80
+ // "帧不变"证明不了流没工作。
81
+ if (i % 2 === 0) await click(String((i / 2) % 10))
82
+ const t0 = process.hrtime.bigint()
83
+ const f = await ask({ cmd: 'liveView', maxDimension: 800, quality: 0.55, activeWithinMs: 120_000 })
84
+ const ms = Number(process.hrtime.bigint() - t0) / 1e6
85
+ frames.push({
86
+ ms,
87
+ source: f.source,
88
+ active: f.active,
89
+ error: f.error,
90
+ hash: f.jpegBase64 ? createHash('sha1').update(f.jpegBase64).digest('hex').slice(0, 8) : null,
91
+ size: f.jpegBase64 ? f.jpegBase64.length : 0,
92
+ dims: f.width ? `${f.width}x${f.height}` : null,
93
+ })
94
+ await sleep(TICK_MS)
95
+ }
96
+
97
+ const withImage = frames.filter((f) => f.hash)
98
+ const streamed = frames.filter((f) => f.source === 'stream')
99
+ const distinct = new Set(withImage.map((f) => f.hash)).size
100
+ const lat = median(withImage.map((f) => f.ms))
101
+
102
+ console.log(` 取到画面 ${withImage.length}/${FRAMES} 帧 | 走取景流 ${streamed.length} 帧 | 不重复画面 ${distinct} 种`)
103
+ console.log(` 单帧耗时中位数 ${lat.toFixed(1)}ms | 尺寸 ${withImage[0]?.dims} | 约 ${(withImage[0]?.size / 1024).toFixed(0)}KB/帧`)
104
+ const mark = (f) => ({ stream: 'S', snapshot: 'n', legacy: 'L' }[f.source] || (f.error ? '!' : '·'))
105
+ console.log(` source 序列: ${frames.map(mark).join('')} (S=常驻流, n=单帧兜底, L=上游老路)`)
106
+
107
+ assert.ok(withImage.length >= FRAMES - 2, `应该几乎每次都拿到画面,实得 ${withImage.length}/${FRAMES}`)
108
+ assert.ok(streamed.length >= FRAMES - 3, `除头几帧外都应走常驻取景流,实得 ${streamed.length}/${FRAMES} —— 全是 snapshot 说明流没起来`)
109
+ // 命门:延迟低但画面永远是同一张 = 静止画,照样是用户说的"掉帧"
110
+ assert.ok(distinct >= FRAMES / 3, `画面必须真的在更新,${withImage.length} 帧里只有 ${distinct} 种`)
111
+ assert.ok(lat < 100, `单帧耗时中位数应远低于轮询间隔(${TICK_MS}ms),实测 ${lat.toFixed(1)}ms —— 追不上就是掉帧`)
112
+ // ⚠️"占不满有留白"的回归线:上游 captureWindow 不设宽高,ScreenCaptureKit 会按整块屏幕出图、
113
+ // 把窗口摆在左上角、其余留白 —— 留白是**烤进 JPEG** 的,CSS 那头救不回来。
114
+ // 所以每一帧的宽高比都必须等于窗口自己的宽高比。
115
+ const wantAr = win.framePoints.w / win.framePoints.h
116
+ for (const f of withImage) {
117
+ const [w, h] = f.dims.split('x').map(Number)
118
+ assert.ok(Math.abs(w / h - wantAr) < 0.02, `画面宽高比必须等于窗口(窗口 ${win.framePoints.w}x${win.framePoints.h} → ${wantAr.toFixed(3)}),实得 ${f.dims} → ${(w / h).toFixed(3)}(source=${f.source})`)
119
+ }
120
+ assert.ok(!frames.some((f) => f.source === 'legacy'), '不该退到上游那条会留白的老路')
121
+ console.log(`✅ 实时画面:走常驻取景流、单帧亚百毫秒、画面确实在动、且每帧都是窗口原比例(无留白)`)
122
+ } finally {
123
+ await killCalculator(ask).catch(() => {})
124
+ }
@@ -0,0 +1 @@
1
+ export function macosHelperIsCurrent(root: string, app: string): boolean;
@@ -0,0 +1,119 @@
1
+ // A sealed helper travels as a ZIP: Electron's outer --deep signing must never
2
+ // re-sign it. Runtime operations here only verify, copy and register the app.
3
+ import { createHash } from 'node:crypto';
4
+ import { execFileSync } from 'node:child_process';
5
+ import { existsSync, readFileSync } from 'node:fs';
6
+ import fs from 'node:fs/promises';
7
+ import path from 'node:path';
8
+
9
+ export const appName = 'tangu-computer-use.app';
10
+ export const bundleId = 'com.forsion.tangu-computer-use';
11
+ // Release helpers are signed with one fixed self-signed certificate (SHA-1 below). macOS files the
12
+ // Accessibility and Screen Recording grants under the designated requirement = bundle id + this
13
+ // certificate's hash, so helper updates keep the grants for as long as neither changes. Never replace
14
+ // the certificate or change the bundle id: every user would have to grant access again. The private key
15
+ // lives only in the GitHub secrets CU_SIGNING_P12 / CU_SIGNING_P12_PASSWORD (backup kept by the owner);
16
+ // the release workflow signs with it and scripts/verify-package.mjs refuses a helper signed by anything else.
17
+ export const releaseCertSha1 = 'dab3a30e7568c7e2c021660b49398356a205a191';
18
+ export const sealedFiles = ['Contents/MacOS/bridge', 'Contents/Info.plist', 'Contents/_CodeSignature/CodeResources'];
19
+ export const sha256 = data => createHash('sha256').update(data).digest('hex');
20
+
21
+ export function bundledMacosApp(root, arch = process.arch) {
22
+ for (const candidate of ['universal', arch]) {
23
+ const dir = path.join(root, 'prebuilt/macos', candidate);
24
+ const archive = path.join(dir, `${appName}.zip`);
25
+ if (!existsSync(archive)) continue;
26
+ const metadata = JSON.parse(readFileSync(path.join(dir, `${appName}.json`), 'utf8'));
27
+ const version = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')).version;
28
+ if (metadata.bundleId !== bundleId || metadata.version !== version || !/^[a-f0-9]{64}$/.test(metadata.archiveSha256)
29
+ || sealedFiles.some(file => !/^[a-f0-9]{64}$/.test(metadata.files?.[file]))) {
30
+ throw new Error('The bundled Computer Use helper metadata is invalid. Reinstall or rebuild the plugin.');
31
+ }
32
+ return { archive, metadata };
33
+ }
34
+ throw new Error('The pre-signed Computer Use helper app is missing. Reinstall the plugin. For development, run npm run build:native on macOS.');
35
+ }
36
+
37
+ export function verifyMacosApp(app) {
38
+ // -R pins the helper identity; signature verification never needs a private key.
39
+ execFileSync('/usr/bin/codesign', ['--verify', '--strict', '-R', `=identifier "${bundleId}"`, app],
40
+ { stdio: 'pipe', timeout: 15_000 });
41
+ }
42
+
43
+ export function macosAppMatches(app, metadata) {
44
+ try {
45
+ if (sealedFiles.some(file => sha256(readFileSync(path.join(app, file))) !== metadata.files[file])) return false;
46
+ verifyMacosApp(app);
47
+ return true;
48
+ } catch { return false; }
49
+ }
50
+
51
+ export function macosHelperIsCurrent(root, app) {
52
+ return macosAppMatches(app, bundledMacosApp(root).metadata);
53
+ }
54
+
55
+ async function withInstallLock(app, work) {
56
+ const lock = `${app}.install-lock`;
57
+ async function abandoned() {
58
+ const owner = Number(await fs.readFile(path.join(lock, 'pid'), 'utf8').catch(() => ''));
59
+ const age = Date.now() - (await fs.stat(lock).catch(() => ({ mtimeMs: Date.now() }))).mtimeMs;
60
+ if (age <= 30_000) return false;
61
+ if (!owner) return true;
62
+ try { process.kill(owner, 0); return false; } catch (error) { return error.code === 'ESRCH'; }
63
+ }
64
+ const deadline = Date.now() + 30_000;
65
+ while (true) {
66
+ try { await fs.mkdir(lock); break; }
67
+ catch (error) {
68
+ if (error.code !== 'EEXIST') throw error;
69
+ // A live owner, regardless of age, keeps its lock until replacement finishes.
70
+ if (await abandoned()) {
71
+ // Serialize stale-lock recovery and recheck inside it. Two recovering
72
+ // installers must not delete a lock that the first has just acquired.
73
+ const recovery = `${lock}.recovery`;
74
+ const ownsRecovery = await fs.mkdir(recovery).then(() => true, error => { if (error.code === 'EEXIST') return false; throw error; });
75
+ if (ownsRecovery) {
76
+ try { if (await abandoned()) await fs.rm(lock, { recursive: true, force: true }); }
77
+ finally { await fs.rm(recovery, { recursive: true, force: true }); }
78
+ continue;
79
+ }
80
+ }
81
+ if (Date.now() > deadline) throw new Error('Another Computer Use installation is still running. Retry when it finishes.');
82
+ await new Promise(resolve => setTimeout(resolve, 100));
83
+ }
84
+ }
85
+ try { await fs.writeFile(path.join(lock, 'pid'), String(process.pid)); return await work(); }
86
+ finally { await fs.rm(lock, { recursive: true, force: true }); }
87
+ }
88
+
89
+ export async function installMacosApp(root, app, { register = true } = {}) {
90
+ if (!path.basename(app).endsWith('.app')) throw new Error('Helper destination must be an .app bundle.');
91
+ const parent = path.dirname(app);
92
+ await fs.mkdir(parent, { recursive: true });
93
+ return withInstallLock(app, async () => {
94
+ const { archive, metadata } = bundledMacosApp(root);
95
+ if (sha256(await fs.readFile(archive)) !== metadata.archiveSha256) throw new Error('The bundled Computer Use helper archive is damaged. Reinstall the plugin.');
96
+ let changed = false;
97
+ if (!macosAppMatches(app, metadata)) {
98
+ const staging = await fs.mkdtemp(path.join(parent, '.tangu-computer-use-install-'));
99
+ const next = path.join(staging, appName), old = path.join(staging, 'previous.app');
100
+ let movedOld = false;
101
+ try {
102
+ execFileSync('/usr/bin/ditto', ['-x', '-k', archive, staging], { stdio: 'pipe', timeout: 30_000 });
103
+ if (!macosAppMatches(next, metadata)) throw new Error('The bundled Computer Use helper signature or contents are invalid.');
104
+ if (existsSync(app)) { await fs.rename(app, old); movedOld = true; }
105
+ try { await fs.rename(next, app); }
106
+ catch (error) { if (movedOld) await fs.rename(old, app); throw error; }
107
+ changed = true;
108
+ } finally {
109
+ // If rollback itself failed, preserve the previous app for recovery.
110
+ if (!movedOld || existsSync(app)) await fs.rm(staging, { recursive: true, force: true });
111
+ }
112
+ }
113
+ if (register) {
114
+ const command = '/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister';
115
+ try { execFileSync(command, ['-f', app], { stdio: 'pipe', timeout: 10_000 }); } catch { /* open will register it as well */ }
116
+ }
117
+ return changed;
118
+ });
119
+ }
@@ -0,0 +1,57 @@
1
+ #!/usr/bin/env bash
2
+ # Create a stable self-signed code-signing certificate for release signing,
3
+ # exported as a .p12 ready for the release workflow secrets.
4
+ #
5
+ # This certificate makes local signing repeatable, but macOS may still require
6
+ # permission review after native code changes. Persistent update identity is a
7
+ # property of Developer ID-signed release bundles, not a guarantee of this
8
+ # self-signed development certificate.
9
+ #
10
+ # Keep the generated key.pem + cert.pem somewhere safe and permanent: losing
11
+ # them means the next release signs under a new leaf and every user
12
+ # re-grants once. If you later enroll in the Apple Developer Program, swap
13
+ # the workflow secrets for a Developer ID cert (users re-grant that once).
14
+ #
15
+ # Usage: ./scripts/make-signing-cert.sh [output-dir]
16
+ # Then, for the Release workflow:
17
+ # base64 -i <dir>/id.p12 | pbcopy -> secret APPLICATION_CERT_BASE64
18
+ # secret CERT_PASSWORD = the password you enter below
19
+ # secret SIGN_IDENTITY = "tangu-computer-use Self Signed"
20
+ set -euo pipefail
21
+
22
+ OUT_DIR="${1:-./signing}"
23
+ CN="tangu-computer-use Self Signed"
24
+ mkdir -p "$OUT_DIR"
25
+ cd "$OUT_DIR"
26
+
27
+ read -r -s -p "Password for the exported .p12: " P12_PASS
28
+ echo
29
+
30
+ cat > cert.conf <<'EOF'
31
+ [req]
32
+ distinguished_name = dn
33
+ x509_extensions = v3
34
+ prompt = no
35
+ [dn]
36
+ CN = tangu-computer-use Self Signed
37
+ [v3]
38
+ keyUsage = critical, digitalSignature
39
+ extendedKeyUsage = critical, codeSigning
40
+ basicConstraints = critical, CA:false
41
+ EOF
42
+
43
+ # 10-year validity so the signing identity does not silently expire between
44
+ # releases (an expired leaf would change grant behavior).
45
+ openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem \
46
+ -days 3650 -nodes -config cert.conf
47
+
48
+ # macOS `security`/Keychain cannot read OpenSSL 3's default PKCS12 MAC; the
49
+ # legacy PBE + SHA1 MAC flags are required for `security import` to accept it.
50
+ openssl pkcs12 -export -out id.p12 -inkey key.pem -in cert.pem \
51
+ -passout "pass:${P12_PASS}" -name "$CN" \
52
+ -legacy -certpbe PBE-SHA1-3DES -keypbe PBE-SHA1-3DES -macalg sha1
53
+
54
+ rm -f cert.conf
55
+ echo
56
+ echo "Wrote $OUT_DIR/{key.pem,cert.pem,id.p12}. Back these up permanently."
57
+ echo "SIGN_IDENTITY secret value: $CN"
@@ -0,0 +1,11 @@
1
+ import { execFileSync } from 'node:child_process';
2
+ import { mkdtempSync, rmSync } from 'node:fs';
3
+ import os from 'node:os';
4
+ import path from 'node:path';
5
+ const root = path.resolve(import.meta.dirname, '..');
6
+ const temp = mkdtempSync(path.join(os.tmpdir(), 'cu-mini-signal-'));
7
+ try {
8
+ execFileSync('xcrun', ['swiftc', '-module-cache-path', path.join(temp, 'cache'), '-parse-as-library',
9
+ path.join(root, 'native/macos/foreground_activity.swift'), path.join(root, 'native/macos/foreground_activity_tests.swift'), '-o', path.join(temp, 'test')], { stdio: 'inherit' });
10
+ execFileSync(path.join(temp, 'test'), [], { stdio: 'inherit' });
11
+ } finally { rmSync(temp, { recursive: true, force: true }); }
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * 前台行为的回归仪器。两部分:
4
+ * 1) 默认:foregroundNote 判据的纯单测(免 TCC/免真机,CI 可跑)。钉住「什么算夺取前台」的定义,
5
+ * 防上游同步或重构悄悄让透明化提示失效。需先 `npm run build`(读 dist/foregroundNote.js)。
6
+ * 2) --live:ensure_app 的「后台启动不抢前台」不变量——open -g 一个临时文件到 TextEdit,断言终端仍是
7
+ * 前台(TextEdit 没跳到前台)。需 macOS。这是我新增行为(ensure_app/透明化)的真机绿灯,不重复测 vendor。
8
+ *
9
+ * 用法: node scripts/no-foreground.check.mjs [--live]
10
+ */
11
+ import assert from 'node:assert/strict';
12
+ import { execFile } from 'node:child_process';
13
+ import { promisify } from 'node:util';
14
+ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
15
+ import { tmpdir } from 'node:os';
16
+ import path from 'node:path';
17
+ import { fileURLToPath } from 'node:url';
18
+
19
+ const execFileP = promisify(execFile);
20
+ const here = path.dirname(fileURLToPath(import.meta.url));
21
+
22
+ async function unitTests() {
23
+ const { foregroundNote } = await import(path.join(here, '..', 'tangu-plugins', 'computer-use', 'dist', 'foregroundNote.js'));
24
+ const has = (s) => s.includes('[foreground]');
25
+
26
+ // 无信号 → 空串
27
+ assert.equal(foregroundNote(undefined), '', 'undefined → empty');
28
+ assert.equal(foregroundNote(null), '', 'null → empty');
29
+ assert.equal(foregroundNote({}), '', 'no signals → empty');
30
+ assert.equal(foregroundNote({ delivery: 'ax', deliveryPolicy: 'background' }), '', 'pure background → empty');
31
+ assert.equal(foregroundNote({ steps: [{ delivery: 'ax' }, { delivery: 'pid', deliveryPolicy: 'background' }] }), '', 'all-background steps → empty');
32
+
33
+ // 各前台信号 → 提示
34
+ assert.ok(has(foregroundNote({ escalatedToForeground: true })), 'escalated → note');
35
+ assert.ok(has(foregroundNote({ delivery: 'hid' })), 'hid delivery → note');
36
+ assert.ok(has(foregroundNote({ deliveryPolicy: 'foreground' })), 'foreground policy → note');
37
+
38
+ // ⚠️坐标点击在 TS 层恒被判 needsForeground → policy 恒为 foreground;helper 却可能用 AX 命中测试
39
+ // 在后台完成它。这时报「抢了前台」是**反的**,比不报更坏。
40
+ assert.equal(foregroundNote({ delivery: 'ax', deliveryPolicy: 'foreground' }), '', 'ax delivery under a foreground policy → no note');
41
+ assert.ok(has(foregroundNote({ delivery: 'ax', deliveryPolicy: 'foreground', escalatedToForeground: true })), 'ax + escalated → still a note');
42
+ assert.ok(has(foregroundNote({ steps: [{ delivery: 'ax', deliveryPolicy: 'foreground' }, { delivery: 'hid' }] })), 'any hid step → note');
43
+ assert.ok(has(foregroundNote({ steps: [{ delivery: 'ax' }, { escalatedToForeground: true }] })), 'any escalated step → note');
44
+
45
+ // 升级原因透传
46
+ const withReason = foregroundNote({ escalatedToForeground: true, escalationReason: 'foreground_required' });
47
+ assert.ok(withReason.includes('foreground_required'), 'reason surfaced');
48
+
49
+ console.log('✓ unit: foregroundNote 判据 9 项全过');
50
+ }
51
+
52
+ async function frontmostBundleId() {
53
+ const { stdout } = await execFileP('osascript', ['-e',
54
+ 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true']);
55
+ return stdout.trim();
56
+ }
57
+
58
+ async function liveTest() {
59
+ if (process.platform !== 'darwin') { console.log('· live: 跳过(非 macOS)'); return; }
60
+ const before = await frontmostBundleId();
61
+ const dir = mkdtempSync(path.join(tmpdir(), 'cu-nofg-'));
62
+ const file = path.join(dir, 'cu-no-foreground-probe.txt');
63
+ writeFileSync(file, 'probe\n');
64
+ try {
65
+ // -g:后台打开(TextEdit 承接 .txt),不应置前台。
66
+ await execFileP('open', ['-g', '-e', file], { timeout: 8_000 });
67
+ await new Promise((r) => setTimeout(r, 1500));
68
+ const after = await frontmostBundleId();
69
+ assert.equal(after, before, `frontmost changed: ${before} → ${after}(后台启动却抢了前台)`);
70
+ assert.notEqual(after, 'com.apple.TextEdit', 'TextEdit 跳到了前台');
71
+ console.log(`✓ live: 后台启动 TextEdit,前台仍是 ${after}(未被抢)`);
72
+ } finally {
73
+ await execFileP('osascript', ['-e', 'tell application "TextEdit" to close (every document whose path is "' + file + '")']).catch(() => {});
74
+ rmSync(dir, { recursive: true, force: true });
75
+ }
76
+ }
77
+
78
+ const live = process.argv.includes('--live');
79
+ await unitTests();
80
+ if (live) await liveTest();
81
+ console.log(`\n✅ no-foreground check 通过${live ? '(含 --live)' : ''}`);