@forsion/tangu-computer-use 0.5.8 → 0.6.1

package/scripts/recorder-logic.check.mjs

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+// 电脑历史采集器(activity_recorder.swift)的单测:照 highlight / mini-foreground 的做法,临时编一个 @main 可执行文件跑一遍。
+// 不需要任何授权;非 macOS 直接跳过。大部分是纯逻辑;订阅流那一组走真实订阅路径 —— 会装上 NSWorkspace 观察者与
+// NSEvent 全局监听、对前台 App 发 AX 读取(没有辅助功能授权时读取失败、键盘监听收不到东西),
+// 并断言最后一个订阅者走后这些全部拆掉。只断言形状,不打印事件内容。
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+if (process.platform !== 'darwin') {
+  console.log('recorder logic check skipped (not macOS)');
+  process.exit(0);
+}
+
+const triple = process.arch === 'x64' ? 'x86_64-apple-macosx14.0' : 'arm64-apple-macosx14.0';
+const temp = mkdtempSync(path.join(os.tmpdir(), 'cu-recorder-logic-'));
+try {
+  execFileSync('xcrun', [
+    'swiftc', '-target', triple, '-parse-as-library',
+    '-module-cache-path', path.join(os.tmpdir(), `tangu-cu-recorder-test-cache-${process.arch}`),
+    '-framework', 'AppKit',
+    '-framework', 'ApplicationServices',
+    path.join(root, 'native/macos/activity_recorder.swift'),
+    path.join(root, 'native/macos/activity_recorder_tests.swift'),
+    '-o', path.join(temp, 'test'),
+  ], { stdio: 'inherit' });
+  execFileSync(path.join(temp, 'test'), [], { stdio: 'inherit' });
+} finally {
+  rmSync(temp, { recursive: true, force: true });
+}

package/scripts/recorder.check.mjs

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+// 电脑历史 recordSubscribe(协议 13)的接线检查,不需要任何授权:直接起刚编好的 helper 二进制
+// (serve 模式、临时 socket;不走 open,因为这里只查接线,不查 TCC 归属),然后断言:
+//   1. diagnostics 报协议 13、订阅者 0,无痕扫描计数(recorderPrivateScan*)在且为 0;
+//   2. recordSubscribe 的首行回包形状正确 —— ok:true + subscribed,或(dev 构建没授权时)ok:false + accessibility_denied;
+//   3. 这条连接保持打开 1 秒;之后收到的每一行都是 {"ev":{t,kind}};
+//   4. 订阅成功时 diagnostics 报 1 个订阅者,客户端关掉后回到 0(这里只看注册表;观察者 / 全局监听真的拆掉了
+//      由 check:recorder-logic 的 debugState 断言覆盖);
+//      被拒时这条连接仍是普通连接,照常应答 diagnostics;
+//   5. 最后 shutdown(兜底 SIGKILL),删临时目录。
+// 若本机终端恰好有辅助功能授权,订阅会真的开始记录:这里只打印事件的 kind 计数,绝不打印内容。
+// 需要先 `npm run build:native`。
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import net from 'node:net';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+if (process.platform !== 'darwin') {
+  console.log('recorder check skipped (not macOS)');
+  process.exit(0);
+}
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const arch = process.arch === 'x64' ? 'x64' : 'arm64';
+const binary = process.argv[2] ?? path.join(root, 'prebuilt', 'macos', arch, 'bridge');
+if (!existsSync(binary)) {
+  console.error(`FAIL missing helper binary ${binary} — run \`npm run build:native\` first`);
+  process.exit(1);
+}
+
+const temp = mkdtempSync(path.join(os.tmpdir(), 'cu-rec-'));
+const socketPath = path.join(temp, 'bridge.sock');
+const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const EVENT_KINDS = new Set(['app', 'window', 'text', 'click', 'key', 'system']);
+
+/** 一条连接:按行收 JSON。 */
+function connect() {
+  return new Promise((resolve, reject) => {
+    const socket = net.createConnection(socketPath);
+    const lines = [];
+    const waiters = [];
+    let buffer = '';
+    let closed = false;
+    socket.setEncoding('utf8');
+    socket.on('data', (chunk) => {
+      buffer += chunk;
+      let index;
+      while ((index = buffer.indexOf('\n')) >= 0) {
+        const line = buffer.slice(0, index);
+        buffer = buffer.slice(index + 1);
+        if (!line) continue;
+        lines.push(JSON.parse(line));
+        waiters.splice(0).forEach((wake) => wake());
+      }
+    });
+    socket.on('close', () => { closed = true; waiters.splice(0).forEach((wake) => wake()); });
+    socket.once('error', reject);
+    socket.once('connect', () => {
+      socket.removeListener('error', reject);
+      socket.on('error', () => {});
+      resolve({
+        socket,
+        lines,
+        get closed() { return closed; },
+        send(object) { socket.write(`${JSON.stringify(object)}\n`); },
+        async next(timeoutMs = 3000) {
+          const deadline = Date.now() + timeoutMs;
+          while (lines.length === 0) {
+            if (closed) throw new Error('connection closed before a reply');
+            if (Date.now() > deadline) throw new Error('timed out waiting for a line');
+            await new Promise((wake) => { waiters.push(wake); setTimeout(wake, 100); });
+          }
+          return lines.shift();
+        },
+      });
+    });
+  });
+}
+
+async function request(object) {
+  const connection = await connect();
+  try {
+    connection.send(object);
+    return await connection.next();
+  } finally {
+    connection.socket.destroy();
+  }
+}
+
+let child;
+try {
+  child = spawn(binary, ['serve', '--socket', socketPath], { stdio: ['ignore', 'ignore', 'inherit'] });
+  let diagnostics;
+  for (let attempt = 0; attempt < 100 && !diagnostics; attempt++) {
+    if (child.exitCode !== null) throw new Error(`helper exited early (code ${child.exitCode})`);
+    try { diagnostics = await request({ id: 'diag-0', cmd: 'diagnostics' }); } catch { await pause(100); }
+  }
+  assert.equal(diagnostics?.ok, true, 'helper must answer diagnostics');
+  assert.equal(diagnostics.result.protocolVersion, 13, `protocol must be 13, got ${diagnostics.result.protocolVersion}`);
+  assert.equal(diagnostics.result.recorderSubscribers, 0, 'no subscribers at startup');
+  console.log('PASS helper answers diagnostics with protocol 13 and no subscribers');
+  // 无痕扫描计数(只有计数):真机探针据此判断「扫描不完整时仍按普通窗口记」要不要改。刚起的 helper 全是 0。
+  assert.equal(diagnostics.result.recorderPrivateScans, 0, 'recorderPrivateScans is a number, 0 at startup');
+  assert.equal(diagnostics.result.recorderPrivateScanIncomplete, 0, 'recorderPrivateScanIncomplete is a number, 0 at startup');
+  assert.deepEqual(diagnostics.result.recorderPrivateScanStops, { afterWebArea: 0, depthCap: 0, nodeCap: 0, deadline: 0 }, 'recorderPrivateScanStops lists every stop reason');
+  console.log('PASS diagnostics reports private-scan counters');
+
+  const subscription = await connect();
+  subscription.send({
+    id: 'rec-1',
+    cmd: 'recordSubscribe',
+    policy: { excludeBundleIds: ['com.example.never'], titleOnlyBundleIds: ['com.forsion.*'], excludeDomains: ['example.com'], text: true, clicks: true, keys: true },
+  });
+  const reply = await subscription.next();
+  assert.equal(reply.id, 'rec-1', 'reply echoes the request id');
+  let subscribed = false;
+  if (reply.ok) {
+    assert.equal(reply.result?.subscribed, true, 'ok reply says subscribed');
+    assert.equal(reply.result?.protocolVersion, 13, 'ok reply carries protocol 13');
+    assert.equal(typeof reply.result?.axTrusted, 'boolean', 'ok reply carries axTrusted');
+    subscribed = true;
+    console.log('PASS recordSubscribe replied ok:true subscribed (this process tree has an Accessibility grant)');
+  } else {
+    assert.equal(reply.error?.code, 'accessibility_denied', `denied reply must use accessibility_denied, got ${reply.error?.code}`);
+    assert.equal(typeof reply.error?.message, 'string', 'denied reply carries a message');
+    console.log('PASS recordSubscribe replied ok:false accessibility_denied (expected for an ungranted dev build)');
+  }
+
+  await pause(1000);
+  assert.equal(subscription.closed, false, 'the subscription connection stays open for 1s');
+  console.log('PASS connection stays open for 1s after the reply');
+
+  if (subscribed) {
+    const kinds = {};
+    for (const line of subscription.lines.splice(0)) {
+      assert.equal(typeof line.ev?.t, 'number', 'every streamed line is {"ev":{t,...}}');
+      assert.ok(EVENT_KINDS.has(line.ev.kind), `unknown event kind ${line.ev?.kind}`);
+      kinds[line.ev.kind] = (kinds[line.ev.kind] ?? 0) + 1;
+    }
+    console.log(`PASS streamed lines are well-formed events (kinds only: ${JSON.stringify(kinds)})`);
+    const during = await request({ id: 'diag-1', cmd: 'diagnostics' });
+    assert.equal(during.result.recorderSubscribers, 1, 'one subscriber while the connection is open');
+    subscription.socket.destroy();
+    let after;
+    for (let attempt = 0; attempt < 30; attempt++) {
+      after = await request({ id: 'diag-2', cmd: 'diagnostics' });
+      if (after.result.recorderSubscribers === 0) break;
+      await pause(100);
+    }
+    assert.equal(after.result.recorderSubscribers, 0, 'closing the client unsubscribes it');
+    console.log('PASS closing the client drops the subscriber back to 0');
+  } else {
+    assert.equal(subscription.lines.length, 0, 'a denied connection streams no events');
+    subscription.send({ id: 'diag-3', cmd: 'diagnostics' });
+    const same = await subscription.next();
+    assert.equal(same.id, 'diag-3', 'a denied connection keeps serving ordinary requests');
+    assert.equal(same.result.recorderSubscribers, 0, 'denied subscription registers nothing');
+    subscription.socket.destroy();
+    console.log('PASS denied connection stays a normal request connection with no subscriber');
+  }
+} finally {
+  await request({ id: 'bye', cmd: 'shutdown' }).catch(() => {});
+  for (let attempt = 0; attempt < 30 && child && child.exitCode === null && child.signalCode === null; attempt++) await pause(100);
+  if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); // 只杀我们自己起的隔离实例,绝不碰用户的常驻 helper
+  rmSync(temp, { recursive: true, force: true });
+}

package/src/index.ts

@@ -11,6 +11,14 @@ import { SETTINGS } from './settings.ts';
 import { cliMain } from './setup.ts';
 import { shutdownComputerUseSession } from './vendor/bridge.ts';
 
+// helper 惰性启动后常驻;进程退出时收口(无 pi 的 session_shutdown 钩子)。
+// stop 放模块级、引用唯一:宿主热插拔时会在同一个模块对象上反复 activate / deactivate(启动时每个插件先 activate、
+// 未启用的随即 deactivate;停用后再启用;activate 抛错也补调一次 deactivate)。每次 activate 新建 stop 的话 deactivate
+// 摘不掉,每拨一次开关叠一组监听 → MaxListenersExceededWarning、退出时 shutdown 跑 N 遍。
+// 同 id 升级 import 的是新一代模块,自带一个 stop:各代只摘自己的,互不干扰。
+const EXIT_EVENTS = ['exit', 'SIGTERM', 'SIGINT'] as const;
+const stop = (): void => { void shutdownComputerUseSession(); };
+
 const plugin: TanguPlugin = {
   activate(ctx) {
     ctx.registerPlugin({
@@ -47,15 +55,17 @@ const plugin: TanguPlugin = {
       ctx.registerCommand({ name: 'computer-use', summary: 'Computer Use: setup / doctor / stop', run: cliMain });
     }
 
-    // helper 惰性启动后常驻;进程退出时收口(无 pi 的 session_shutdown 钩子)。
-    const stop = (): void => { void shutdownComputerUseSession(); };
-    process.once('exit', stop);
-    process.once('SIGTERM', stop);
-    process.once('SIGINT', stop);
+    // 先摘再挂:同一模块对象上再 activate(停用后再启用、抛错后再次激活)也只挂一组。
+    for (const ev of EXIT_EVENTS) {
+      process.off(ev, stop);
+      process.once(ev, stop);
+    }
 
     ctx.log('computer-use plugin registered');
   },
   deactivate() {
+    // 只摘本模块自己的 stop(新一代模块与宿主的监听不动),helper 照旧收口。
+    for (const ev of EXIT_EVENTS) process.off(ev, stop);
     void shutdownComputerUseSession();
   },
 };

package/src/vendor/platform/macos/helper.ts

@@ -10,7 +10,7 @@ import type { PlatformDiagnostics } from "../types.ts";
 import { resolveMacosHelperAppPath } from "./helper-path.mjs";
 
 const COMMAND_TIMEOUT_MS = 15_000;
-const HELPER_PROTOCOL_VERSION = 12;
+const HELPER_PROTOCOL_VERSION = 13;
 const HELPER_SETUP_TIMEOUT_MS = 60_000;
 
 export const HELPER_BUNDLE_ID = "com.forsion.tangu-computer-use";

package/tangu-plugins/computer-use/dist/index.js

@@ -171,7 +171,7 @@ var init_helper = __esm({
     init_coerce();
     init_helper_path();
     COMMAND_TIMEOUT_MS2 = 15e3;
-    HELPER_PROTOCOL_VERSION = 12;
+    HELPER_PROTOCOL_VERSION = 13;
     HELPER_SETUP_TIMEOUT_MS = 6e4;
     HELPER_BUNDLE_ID = "com.forsion.tangu-computer-use";
     HELPER_APP_PATH = resolveMacosHelperAppPath();
@@ -5191,9 +5191,9 @@ function waitFor(work, signal) {
         fn();
       }
     };
-    const stop = (why) => () => settle(() => reject(new Error(`${why}; the installer may still finish in the background`)));
-    const timer = setTimeout(stop("helper setup is taking longer than expected"), SETUP_TIMEOUT_MS);
-    const onAbort = stop("helper setup wait was interrupted");
+    const stop2 = (why) => () => settle(() => reject(new Error(`${why}; the installer may still finish in the background`)));
+    const timer = setTimeout(stop2("helper setup is taking longer than expected"), SETUP_TIMEOUT_MS);
+    const onAbort = stop2("helper setup wait was interrupted");
     signal?.addEventListener("abort", onAbort, { once: true });
     work.then((v) => settle(() => resolve(v)), (e) => settle(() => reject(e instanceof Error ? e : new Error(String(e)))));
   });
@@ -5626,6 +5626,10 @@ async function doctor() {
 }
 
 // src/index.ts
+var EXIT_EVENTS = ["exit", "SIGTERM", "SIGINT"];
+var stop = () => {
+  void shutdownComputerUseSession();
+};
 var plugin = {
   activate(ctx) {
     ctx.registerPlugin({
@@ -5654,15 +5658,14 @@ var plugin = {
     if (typeof ctx.registerCommand === "function") {
       ctx.registerCommand({ name: "computer-use", summary: "Computer Use: setup / doctor / stop", run: cliMain });
     }
-    const stop = () => {
-      void shutdownComputerUseSession();
-    };
-    process.once("exit", stop);
-    process.once("SIGTERM", stop);
-    process.once("SIGINT", stop);
+    for (const ev of EXIT_EVENTS) {
+      process.off(ev, stop);
+      process.once(ev, stop);
+    }
     ctx.log("computer-use plugin registered");
   },
   deactivate() {
+    for (const ev of EXIT_EVENTS) process.off(ev, stop);
     void shutdownComputerUseSession();
   }
 };

package/tangu-plugins/computer-use/tangu-plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "computer-use",
   "name": "Computer Use",
-  "version": "0.5.8",
+  "version": "0.6.1",
   "apiVersion": 1,
   "entry": "dist/index.js",
   "commands": ["computer-use"],

package/types/tangu-agent.d.ts

@@ -35,6 +35,44 @@ export interface ToolContext {
    *  缺省(未装配此闸)时工具应优雅降级,不要假定一定可用。 */
   collectImage?: (img: { url: string; name?: string }) => void;
   displayFile?: (item: { name: string; mime?: string; path?: string; dataUrl?: string }) => void;
+  /** 发起端(手机)自报的客户端能力,如 'phone.intents'(run 内冻结)。契约 tangu-agent/docs/phone-control.md §2。 */
+  clientCapabilities?: readonly string[];
+  /** 让发起端原生层执行一个动作:发 `client_cmd` → 原生 claim → 执行 → 回执。闭包已绑定本 run,只在
+   *  clientCapabilities 非空时存在(缺省时工具应优雅降级),且**只给声明了 `clientCapability` 的工具**:
+   *  没声明的工具(以及任何工具的 isEnabledFor)拿到 undefined;声明了的只能发自己能力的 ns
+   *  ('phone.intents' → 'phone'),别的 ns 立即 {ok:false, code:'undeclared'}、什么都不发。
+   *  `opts.signal` 传 ctx.signal;run 级中止总会一并监听。
+   *  ⚠️ 若工具声明了 capabilities.defaultTimeoutMs,它必须 ≥ claimMs + execMs —— 否则工具先被判超时,
+   *  手机那边却可能在稍后照样执行。 */
+  requestClientAction?: (req: ClientActionRequest, opts?: ClientActionOptions) => Promise<ClientActionResult>;
+}
+
+/** 客户端动作。ns 必须命中本 run 已声明的 `<ns>.*` 能力(否则立即 {ok:false, code:'undeclared'});op 由原生 verb 表解释。 */
+export interface ClientActionRequest {
+  ns: string;
+  op: string;
+  args?: Record<string, unknown>;
+}
+
+/** claimMs:等原生 claim(缺省 15s,钳 3–30s);execMs:claim 之后等结果(缺省 20s,钳 5–120s)。 */
+export interface ClientActionOptions {
+  claimMs?: number;
+  execMs?: number;
+  signal?: AbortSignal;
+}
+
+/** 原生回执(已消毒)或引擎自产的失败。code 恒匹配 /^[a-z_]{1,32}$/:原生码见契约 §3.4,
+ *  引擎自产 undeclared / not_picked_up(没接,什么都没发生)/ no_report(接了没回,可能发生了)/
+ *  aborted(没接就被中止,什么都没发生)/ aborted_claimed(接了之后被中止,可能发生了)。 */
+export interface ClientActionResult {
+  ok: boolean;
+  code?: string;
+  error?: string;
+  text?: string;
+  image?: string;
+  app?: string;
+  handoff?: boolean;
+  verified?: boolean;
 }
 
 export interface ToolCapabilities {
@@ -58,6 +96,16 @@ export interface ToolDef {
   mode?: 'sandbox' | 'host' | 'both';
   capabilities?: ToolCapabilities;
   isEnabledFor?(profile: AppProfile, ctx: ToolContext): boolean;
+  /** 按需装载:true = 定义默认不进工具面,系统提示「Additional Tools」目录里只留一行,模型经 load_tools 解锁。 */
+  deferred?: boolean;
+  /** 同组连坐解锁:解锁组内任一即整组解锁。 */
+  deferGroup?: string;
+  /** 目录行文案(英文一句话,带典型触发意图;缺省取 description 首行截断)。 */
+  deferHint?: string;
+  /** 客户端能力(如 'phone.intents'):本工具经 ctx.requestClientAction 让发起端原生层执行。声明后由核心中央闸
+   *  default-deny(手机端发起 + 本 run 声明了该能力 + 非子代理 / 计划 / 通道 / 讨论),且工具名必须以 `<ns>_` 开头
+   *  ('phone.intents' → phone_*),否则永不可见。chat 预设按能力放行,内置与插件等价。 */
+  clientCapability?: string;
 }
 
 export interface ToolProvider {
@@ -135,8 +183,19 @@ export interface TanguPluginContext {
   activity: { append(event: string, detail?: Record<string, unknown>): void };
 }
 
+/**
+ * 生命周期:启动时每个插件都 activate 一次,未启用 / 前置没齐的随即 deactivate 休眠(仍列在设置页)。
+ * 停用、卸载、原地升级、前置消失时宿主调 `deactivate()`(限时 5s),再撤掉本次激活注册的一切(工具、命令、路由;meta 留着休眠)。
+ * 再启用会在**同一个模块对象**上再调一次 `activate(ctx)`:别依赖模块级「只做一次」的状态,activate 里起的定时器 /
+ * 子进程 / 监听都要在 deactivate 里收掉。activate 抛错则本次激活作废(回滚 + 调一次 deactivate 收尾),插件休眠不自动重试。
+ */
 export interface TanguPlugin {
-  manifest?: { id: string; name: string; version: string; apiVersion: number; entry: string; commands?: string[]; description?: string };
+  manifest?: {
+    id: string; name: string; version: string; apiVersion: number; entry: string; commands?: string[]; description?: string;
+    /** 前置插件:`"id"` 或 `{ id, minVersion? }`。前置已启用且在跑(版本 ≥ minVersion)时本插件才激活;否则休眠,就位后自动激活。 */
+    requiresPlugins?: Array<string | { id: string; minVersion?: string }>;
+  };
   activate(ctx: TanguPluginContext): void | Promise<void>;
+  /** 停用 / 卸载 / 原地升级时调用(限时 5s):停掉 activate 里起的后台工作。启动期那次早于引擎装配完成,只做自身收尾。 */
   deactivate?(): void | Promise<void>;
 }