@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "computer-use",
3
+ "name": "Computer Use",
4
+ "version": "0.5.8",
5
+ "apiVersion": 1,
6
+ "entry": "dist/index.js",
7
+ "commands": ["computer-use"],
8
+ "description": "Let the agent observe and control desktop apps (macOS 14+ / Windows / Linux)."
9
+ }
package/tsconfig.json ADDED
@@ -0,0 +1,22 @@
1
+ {
2
+ "compilerOptions": {
3
+ "target": "ES2022",
4
+ "module": "ESNext",
5
+ "moduleResolution": "Bundler",
6
+ "lib": ["ES2023"],
7
+ "types": ["node"],
8
+ "strict": true,
9
+ "verbatimModuleSyntax": true,
10
+ "allowImportingTsExtensions": true,
11
+ "noEmit": true,
12
+ "esModuleInterop": true,
13
+ "skipLibCheck": true,
14
+ "forceConsistentCasingInFileNames": true,
15
+ "baseUrl": ".",
16
+ "paths": {
17
+ "@earendil-works/pi-coding-agent": ["./src/pi-compat.ts"],
18
+ "@forsion/tangu-agent": ["./types/tangu-agent.d.ts"]
19
+ }
20
+ },
21
+ "include": ["src", "types"]
22
+ }
@@ -0,0 +1,142 @@
1
+ /**
2
+ * Tangu 插件公开 API 正典(apiVersion 1)。这是「稳定公共子集」,不是核心全量类型——
3
+ * 面越小,兼容承诺越小;插件需要新能力时按需扩这里并升 apiVersion。
4
+ *
5
+ * 单源规则:只编辑本文件,然后 `npm run sync:plugin-api` 同步到 plugins/<dir>/types/(逐字节拷贝);
6
+ * 与核心真类型的兼容由 src/plugins/apiContract.ts 双向断言,随 `npm run typecheck` 跑,漂移即编译错误。
7
+ * 插件侧用法:tsconfig `paths` 把裸标识符 @forsion/tangu-agent 映射到本文件拷贝,一律 `import type`
8
+ * (verbatimModuleSyntax 把值导入变成编译错误);运行时能力全经 activate(ctx) 按引用传入的 ctx.sdk。
9
+ */
10
+
11
+ /** core/types 的 Tool 最小投影(function 工具)。 */
12
+ export interface Tool {
13
+ type: 'function';
14
+ function: { name: string; description: string; parameters: Record<string, unknown> };
15
+ }
16
+
17
+ /** 设置/数据作用域:全局 或 按 agent。 */
18
+ export type Scope = 'global' | { agentSlug: string };
19
+
20
+ /** 能力门禁(只投影工具门禁读到的 capabilities)。 */
21
+ export interface AppProfile {
22
+ capabilities: { hostExec: boolean; groupChat: boolean; memory: boolean; log: boolean };
23
+ }
24
+
25
+ export interface ToolContext {
26
+ userId: string;
27
+ sessionId: string;
28
+ appId: string;
29
+ signal?: AbortSignal;
30
+ execMode?: 'sandbox' | 'host';
31
+ cwd?: string;
32
+ agentSlug?: string;
33
+ /** 工具产出图片的回流闸(view_image / computer-use observe 用):把图片 data URL 交回 loop,
34
+ * loop 在本轮工具执行完后物化成一条 user 图像消息追加到对话尾部,让模型"看见"图片。
35
+ * 缺省(未装配此闸)时工具应优雅降级,不要假定一定可用。 */
36
+ collectImage?: (img: { url: string; name?: string }) => void;
37
+ displayFile?: (item: { name: string; mime?: string; path?: string; dataUrl?: string }) => void;
38
+ }
39
+
40
+ export interface ToolCapabilities {
41
+ sideEffect?: 'none' | 'read' | 'network' | 'browser' | 'write' | 'system' | 'unknown';
42
+ parallel?: boolean;
43
+ concurrencyKey?: string;
44
+ defaultTimeoutMs?: number;
45
+ /** 声明本工具的审批档:'command' = 与 run_bash 同档(readonly/auto-edit 下需用户批准)。
46
+ * 缺省=只读语义,不触发审批。核心据此把插件工具并入审批,无需硬编码工具名。 */
47
+ approval?: 'command';
48
+ /** 正向声明:允许作为自动化 tool_call 动作(不经 LLM、参数在规则里冻结、full-auto 直执行)。
49
+ * 缺省 false——不声明就不进桌面自动化构建器的动作目录。只给参数可完整预填、
50
+ * 无会话交互依赖、副作用可控的工具声明。 */
51
+ automationSafe?: boolean;
52
+ }
53
+
54
+ export interface ToolDef {
55
+ name: string;
56
+ definition: Tool;
57
+ execute: (args: Record<string, any>, ctx: ToolContext) => Promise<string> | string;
58
+ mode?: 'sandbox' | 'host' | 'both';
59
+ capabilities?: ToolCapabilities;
60
+ isEnabledFor?(profile: AppProfile, ctx: ToolContext): boolean;
61
+ }
62
+
63
+ export interface ToolProvider {
64
+ id: string;
65
+ tools(): ToolDef[];
66
+ }
67
+
68
+ export type SettingsScope = 'global' | 'agent';
69
+
70
+ export type SettingsField =
71
+ | { key: string; type: 'toggle'; label: string; labelEn?: string; help?: string; helpEn?: string; default?: boolean }
72
+ | { key: string; type: 'text' | 'textarea'; label: string; labelEn?: string; help?: string; helpEn?: string; default?: string; placeholder?: string }
73
+ | { key: string; type: 'number'; label: string; labelEn?: string; help?: string; helpEn?: string; default?: number; min?: number; max?: number }
74
+ | { key: string; type: 'select'; label: string; labelEn?: string; help?: string; helpEn?: string; default?: string; options: Array<{ value: string; label: string; labelEn?: string }> }
75
+ | { key: string; type: 'image-list'; label: string; labelEn?: string; help?: string; helpEn?: string; itemFields: SettingsField[] };
76
+
77
+ export interface PluginSettingsSchema { fields: SettingsField[] }
78
+
79
+ export interface PluginPromptCtx { slug: string; userId: string; execMode: 'host' | 'sandbox' }
80
+
81
+ export interface PluginMeta {
82
+ id: string;
83
+ name: string;
84
+ nameEn?: string;
85
+ description: string;
86
+ descriptionEn?: string;
87
+ scopes?: SettingsScope[];
88
+ settings?: PluginSettingsSchema;
89
+ defaultEnabled?: boolean;
90
+ source?: 'builtin' | 'folder';
91
+ toolProvider?: ToolProvider;
92
+ promptSection?(ctx: PluginPromptCtx): Promise<string> | string;
93
+ }
94
+
95
+ /** ctx.sdk.pluginStore —— 只列插件常用成员(读写自身设置 + image-list blob)。 */
96
+ export interface PluginStore {
97
+ isPluginEnabledSync(id: string): boolean;
98
+ resolveImageListScope(id: string, field: string, agentSlug?: string): Scope;
99
+ getScopeSettings(id: string, scope: Scope): Record<string, any>;
100
+ setScopeSettings(id: string, scope: Scope, patch: Record<string, any>): Promise<Record<string, any>>;
101
+ readPluginFile(id: string, scope: Scope, name: string): Promise<{ buffer: Buffer; mimeType: string } | null>;
102
+ writePluginFile(id: string, scope: Scope, name: string, buf: Buffer): Promise<string>;
103
+ deletePluginFile(id: string, scope: Scope, name: string): Promise<void>;
104
+ }
105
+
106
+ /** 运行时句柄(按引用传入,核心同一模块实例)。只列稳定公开成员。 */
107
+ export interface TanguSdk {
108
+ pluginStore: PluginStore;
109
+ sendWechatMedia(
110
+ userId: string,
111
+ sessionId: string,
112
+ buffer: Buffer,
113
+ opts: { kind: 'image' | 'file'; fileName: string },
114
+ signal?: AbortSignal,
115
+ ): Promise<{ ok: boolean; error?: string }>;
116
+ }
117
+
118
+ /** 插件注册的 CLI 子命令(`tangu <name> ...`)。 */
119
+ export interface PluginCommand {
120
+ name: string;
121
+ summary: string;
122
+ /** 命令名之后的 argv;返回退出码(或 void=保持进程存活,由其打开的句柄决定)。 */
123
+ run(argv: string[]): Promise<number | void>;
124
+ }
125
+
126
+ export interface TanguPluginContext {
127
+ registerPlugin(meta: PluginMeta): void;
128
+ registerToolProvider(p: ToolProvider): void;
129
+ /** 注册 `tangu <name>` 子命令(如 computer-use 的 doctor/setup)。 */
130
+ registerCommand(cmd: PluginCommand): void;
131
+ sdk: TanguSdk;
132
+ log(msg: string): void;
133
+ paths: { pluginDir: string };
134
+ /** 发用户活动事件(宿主强制加 `plugin:<id>:` 前缀)——自动化 event_seen 可盯;云端 worker no-op。 */
135
+ activity: { append(event: string, detail?: Record<string, unknown>): void };
136
+ }
137
+
138
+ export interface TanguPlugin {
139
+ manifest?: { id: string; name: string; version: string; apiVersion: number; entry: string; commands?: string[]; description?: string };
140
+ activate(ctx: TanguPluginContext): void | Promise<void>;
141
+ deactivate?(): void | Promise<void>;
142
+ }