@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,134 @@
1
+ import { closeSync, mkdtempSync, openSync, readSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
5
+
6
+ export const MODEL_TEXT_MAX_BYTES = 48 * 1024;
7
+ export const MODEL_TEXT_MAX_LINES = 2_000;
8
+ export const OUTPUT_PAGE_BYTES = 16 * 1024;
9
+ export const UI_TEXT_PAGE_CHARS = 12 * 1024;
10
+ export const MODEL_PREVIEW_BYTES = 16 * 1024;
11
+ const OUTPUT_ENTRY_MAX_BYTES = 16 * 1024 * 1024;
12
+ const OUTPUT_STORE_MAX_BYTES = 64 * 1024 * 1024;
13
+
14
+ interface StoredOutput {
15
+ ref: string;
16
+ filePath: string;
17
+ storedBytes: number;
18
+ totalBytes: number;
19
+ complete: boolean;
20
+ }
21
+
22
+ const outputs = new Map<string, StoredOutput>();
23
+ let outputDirectory: string | undefined;
24
+ let outputBytes = 0;
25
+ let nextOutputId = 1;
26
+
27
+ function utf8Prefix(bytes: Uint8Array, maxBytes: number): Uint8Array {
28
+ if (bytes.byteLength <= maxBytes) return bytes;
29
+ let end = Math.max(0, maxBytes);
30
+ while (end > 0 && (bytes[end] & 0xc0) === 0x80) end -= 1;
31
+ return bytes.subarray(0, end);
32
+ }
33
+
34
+ function boundedPrefix(value: string, maxBytes: number, maxLines: number): string {
35
+ const lines = value.split("\n", maxLines + 1);
36
+ const lineBounded = lines.length > maxLines ? lines.slice(0, maxLines).join("\n") : value;
37
+ return new TextDecoder().decode(utf8Prefix(new TextEncoder().encode(lineBounded), maxBytes));
38
+ }
39
+
40
+ function storeOutput(value: string): StoredOutput {
41
+ const encoded = new TextEncoder().encode(value);
42
+ const stored = encoded.byteLength > OUTPUT_ENTRY_MAX_BYTES ? utf8Prefix(encoded, OUTPUT_ENTRY_MAX_BYTES) : encoded;
43
+ outputDirectory ??= mkdtempSync(path.join(os.tmpdir(), "tangu-computer-use-output-"));
44
+ const ref = `@o${nextOutputId++}`;
45
+ const filePath = path.join(outputDirectory, `${ref.slice(2)}.txt`);
46
+ writeFileSync(filePath, stored, { mode: 0o600 });
47
+ const entry: StoredOutput = { ref, filePath, storedBytes: stored.byteLength, totalBytes: encoded.byteLength, complete: stored.byteLength === encoded.byteLength };
48
+ outputs.set(entry.ref, entry);
49
+ outputBytes += entry.storedBytes;
50
+ while (outputBytes > OUTPUT_STORE_MAX_BYTES && outputs.size > 1) {
51
+ const oldestRef = outputs.keys().next().value as string | undefined;
52
+ if (!oldestRef) break;
53
+ const oldest = outputs.get(oldestRef)!;
54
+ outputs.delete(oldestRef);
55
+ outputBytes -= oldest.storedBytes;
56
+ try { unlinkSync(oldest.filePath); } catch { /* already removed */ }
57
+ }
58
+ return entry;
59
+ }
60
+
61
+ function refinementFor(tool: string): string {
62
+ if (tool === "search_ui") return "use a more selective text, role, or capability predicate";
63
+ if (tool === "find_roots") return "use a more selective text, app, bundleId, pid, or kind filter";
64
+ if (tool === "evaluate_browser") return "return selected fields, an aggregate, or a smaller slice";
65
+ if (tool === "observe_ui" || tool === "expand_ui") return "use search_ui or expand a more specific ref";
66
+ return "request a smaller or more focused result";
67
+ }
68
+
69
+ export function applyOutputEnvelope<T>(tool: string, result: AgentToolResult<T>): AgentToolResult<T> {
70
+ const textParts = result.content.filter((part): part is Extract<(typeof result.content)[number], { type: "text" }> => part.type === "text");
71
+ const combined = textParts.map((part) => part.text).join("\n");
72
+ const bytes = new TextEncoder().encode(combined).byteLength;
73
+ const lines = combined === "" ? 0 : combined.split("\n").length;
74
+ if (bytes <= MODEL_TEXT_MAX_BYTES && lines <= MODEL_TEXT_MAX_LINES) return result;
75
+
76
+ const entry = storeOutput(combined);
77
+ const preview = boundedPrefix(combined, MODEL_PREVIEW_BYTES, MODEL_TEXT_MAX_LINES - 4);
78
+ const returnedBytes = new TextEncoder().encode(preview).byteLength;
79
+ const availability = entry.complete
80
+ ? `continue: read_text({ ref: "${entry.ref}", offset: ${returnedBytes} })`
81
+ : `continue: read_text({ ref: "${entry.ref}", offset: ${returnedBytes} }); only the first ${entry.storedBytes} bytes were stored, so refine for the remainder`;
82
+ const trailer = [
83
+ `output truncated: returned ${returnedBytes} of ${entry.totalBytes} utf-8 bytes`,
84
+ `refine: ${refinementFor(tool)}`,
85
+ availability,
86
+ ].join("\n");
87
+ const images = result.content.filter((part) => part.type === "image");
88
+ return { ...result, content: [{ type: "text", text: `${preview}\n\n${trailer}` }, ...images] };
89
+ }
90
+
91
+ export function boundToolError(tool: string, error: unknown): Error {
92
+ const message = error instanceof Error ? error.message : String(error);
93
+ const bytes = new TextEncoder().encode(message).byteLength;
94
+ const lines = message.split("\n").length;
95
+ if (bytes <= MODEL_TEXT_MAX_BYTES && lines <= MODEL_TEXT_MAX_LINES) return error instanceof Error ? error : new Error(message);
96
+ const entry = storeOutput(message);
97
+ const preview = boundedPrefix(message, MODEL_PREVIEW_BYTES, MODEL_TEXT_MAX_LINES - 4);
98
+ const returnedBytes = new TextEncoder().encode(preview).byteLength;
99
+ const storageNote = entry.complete ? "" : `; only the first ${entry.storedBytes} bytes were stored`;
100
+ return new Error(`${preview}\n\nerror truncated: returned ${returnedBytes} of ${entry.totalBytes} utf-8 bytes\nrefine: ${refinementFor(tool)}\ncontinue: read_text({ ref: "${entry.ref}", offset: ${returnedBytes} })${storageNote}`);
101
+ }
102
+
103
+ export function readStoredOutput(ref: string, offsetValue: unknown): { text: string; offset: number; limit: number; totalBytes: number; hasMore: boolean; complete: boolean } | undefined {
104
+ const entry = outputs.get(ref);
105
+ if (!entry) return undefined;
106
+ let offset = Math.min(entry.storedBytes, Math.max(0, Math.trunc(typeof offsetValue === "number" && Number.isFinite(offsetValue) ? offsetValue : 0)));
107
+ const requestedBytes = Math.min(entry.storedBytes - offset, OUTPUT_PAGE_BYTES + 4);
108
+ const buffer = new Uint8Array(Math.max(0, requestedBytes));
109
+ const fd = openSync(entry.filePath, "r");
110
+ try { if (buffer.byteLength > 0) readSync(fd, buffer, 0, buffer.byteLength, offset); } finally { closeSync(fd); }
111
+ let start = 0;
112
+ while (start < buffer.byteLength && (buffer[start] & 0xc0) === 0x80) start += 1;
113
+ offset += start;
114
+ const slice = utf8Prefix(buffer.subarray(start), OUTPUT_PAGE_BYTES);
115
+ const actualEnd = offset + slice.byteLength;
116
+ return {
117
+ text: new TextDecoder().decode(slice),
118
+ offset,
119
+ limit: slice.byteLength,
120
+ totalBytes: entry.totalBytes,
121
+ hasMore: actualEnd < entry.storedBytes,
122
+ complete: entry.complete,
123
+ };
124
+ }
125
+
126
+ export function clearStoredOutputs(): void {
127
+ outputs.clear();
128
+ if (outputDirectory) {
129
+ try { rmSync(outputDirectory, { recursive: true, force: true }); } catch { /* best-effort session cleanup */ }
130
+ }
131
+ outputDirectory = undefined;
132
+ outputBytes = 0;
133
+ nextOutputId = 1;
134
+ }
@@ -0,0 +1,111 @@
1
+ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
2
+
3
+ // Which process identity the permission answers reflect. Platforms may
4
+ // attribute grants to a responsible parent process rather than the helper
5
+ // executable itself.
6
+ export type PermissionAttribution = "helper-app" | "caller";
7
+ export type PermissionKind = "accessibility" | "screenRecording";
8
+
9
+ export interface PermissionSource {
10
+ attribution: PermissionAttribution;
11
+ pid?: number;
12
+ parentPid?: number;
13
+ executablePath?: string;
14
+ parentPath?: string;
15
+ parentBundleId?: string;
16
+ os?: string;
17
+ }
18
+
19
+ export interface PermissionStatus {
20
+ accessibility: boolean;
21
+ screenRecording: boolean;
22
+ screenRecordingPreflight?: boolean;
23
+ source?: PermissionSource;
24
+ }
25
+
26
+ interface PermissionKindCopy {
27
+ kind: PermissionKind;
28
+ openOption: string;
29
+ }
30
+
31
+ interface PermissionFlowCopy {
32
+ nonInteractiveError(helperPath: string): string;
33
+ prompt(status: PermissionStatus, helperPath: string, hint?: string): string;
34
+ incompleteError(helperPath: string): string;
35
+ readyMessage: string;
36
+ stillMissing(kinds: PermissionKind[]): string;
37
+ }
38
+
39
+ export interface PermissionBridge {
40
+ kinds: PermissionKindCopy[];
41
+ copy: PermissionFlowCopy;
42
+ checkPermissions(signal?: AbortSignal): Promise<PermissionStatus>;
43
+ registerPermissions(signal?: AbortSignal): Promise<void>;
44
+ openPermissionPane(kind: PermissionKind, signal?: AbortSignal): Promise<void>;
45
+ // Platforms may cache permission answers per process, so restart before
46
+ // recheck lets a new grant become visible to the helper.
47
+ restartHelper(signal?: AbortSignal): Promise<void>;
48
+ permissionHint?: string;
49
+ }
50
+
51
+ function throwIfAborted(signal?: AbortSignal): void {
52
+ if (signal?.aborted) throw new Error("Operation aborted.");
53
+ }
54
+
55
+ function granted(status: PermissionStatus, kind: PermissionKind): boolean {
56
+ return status[kind] === true;
57
+ }
58
+
59
+ function allGranted(status: PermissionStatus, kinds: PermissionKindCopy[]): boolean {
60
+ return kinds.every(({ kind }) => granted(status, kind));
61
+ }
62
+
63
+ function missingKinds(status: PermissionStatus, kinds: PermissionKindCopy[]): PermissionKind[] {
64
+ return kinds.flatMap(({ kind }) => granted(status, kind) ? [] : [kind]);
65
+ }
66
+
67
+ export async function ensurePermissions(
68
+ ctx: ExtensionContext,
69
+ bridge: PermissionBridge,
70
+ helperPath: string,
71
+ signal?: AbortSignal,
72
+ ): Promise<PermissionStatus> {
73
+ let status = await bridge.checkPermissions(signal);
74
+ if (allGranted(status, bridge.kinds)) return status;
75
+
76
+ if (!ctx.hasUI) throw new Error(bridge.copy.nonInteractiveError(helperPath));
77
+
78
+ // Register before prompting so platform settings panes can already list
79
+ // the helper and the user only has to enable existing entries.
80
+ await bridge.registerPermissions(signal).catch(() => undefined);
81
+
82
+ while (!allGranted(status, bridge.kinds)) {
83
+ throwIfAborted(signal);
84
+
85
+ const missing = missingKinds(status, bridge.kinds);
86
+ const options = bridge.kinds
87
+ .filter(({ kind }) => missing.includes(kind))
88
+ .map(({ openOption }) => openOption);
89
+ options.push("Recheck (restarts helper)", "Cancel");
90
+
91
+ const choice = await ctx.ui.select(bridge.copy.prompt(status, helperPath, bridge.permissionHint), options, { signal });
92
+ if (!choice || choice === "Cancel") throw new Error(bridge.copy.incompleteError(helperPath));
93
+
94
+ const selected = bridge.kinds.find(({ openOption }) => choice === openOption);
95
+ if (selected) await bridge.openPermissionPane(selected.kind, signal);
96
+
97
+ if (choice.startsWith("Recheck")) {
98
+ // Restart first: permission decisions can be cached by a running
99
+ // helper process and remain stale after the user grants access.
100
+ await bridge.restartHelper(signal);
101
+ status = await bridge.checkPermissions(signal);
102
+ if (allGranted(status, bridge.kinds)) {
103
+ ctx.ui.notify(bridge.copy.readyMessage, "info");
104
+ } else {
105
+ ctx.ui.notify(bridge.copy.stillMissing(missingKinds(status, bridge.kinds)), "warning");
106
+ }
107
+ }
108
+ }
109
+
110
+ return status;
111
+ }
@@ -0,0 +1,23 @@
1
+ import type { PlatformDiagnostics } from "./types.ts";
2
+
3
+ export const PLATFORM_ARCHITECTURE_VERSION = 1;
4
+ export const REQUIRED_PLATFORM_INVARIANTS = [
5
+ "state-scoped-observations",
6
+ "bounded-observation-history",
7
+ "multi-root-forest",
8
+ "progressive-disclosure",
9
+ "atomic-physical-input",
10
+ "concurrent-requests",
11
+ "transactional-batching",
12
+ ] as const;
13
+
14
+ export function assertPlatformArchitecture(platform: string, diagnostics: PlatformDiagnostics): void {
15
+ if (diagnostics.architectureVersion !== PLATFORM_ARCHITECTURE_VERSION) {
16
+ throw new Error(`${platform} helper architecture mismatch: expected ${PLATFORM_ARCHITECTURE_VERSION}, got ${diagnostics.architectureVersion ?? "unknown"}. Rebuild and restart the helper.`);
17
+ }
18
+ const reported = new Set(diagnostics.invariants ?? []);
19
+ const missing = REQUIRED_PLATFORM_INVARIANTS.filter((invariant) => !reported.has(invariant));
20
+ if (missing.length > 0) {
21
+ throw new Error(`${platform} helper does not satisfy the shared computer-use contract: ${missing.join(", ")}.`);
22
+ }
23
+ }
@@ -0,0 +1,16 @@
1
+ export function toBoolean(value: unknown): boolean {
2
+ return value === true || value === "true" || value === 1;
3
+ }
4
+
5
+ export function toFiniteNumber(value: unknown, fallback: number): number {
6
+ if (typeof value === "number" && Number.isFinite(value)) return value;
7
+ if (typeof value === "string") {
8
+ const parsed = Number(value);
9
+ if (Number.isFinite(parsed)) return parsed;
10
+ }
11
+ return fallback;
12
+ }
13
+
14
+ export function toOptionalString(value: unknown): string | undefined {
15
+ return typeof value === "string" && value.length > 0 ? value : undefined;
16
+ }
@@ -0,0 +1,59 @@
1
+ import { macosBackend } from "./macos/backend.ts";
2
+ import { isBrowserApp, isChromeFamilyApp, openBrowserLocationWithAppleScript } from "./macos/browser.ts";
3
+ import { ensureMacosReady } from "./macos/permissions.ts";
4
+ import type { ComputerUsePlatformBackend, PlatformName } from "./types.ts";
5
+ import { linuxBackend } from "./linux/backend.ts";
6
+ import { windowsBackend } from "./windows/backend.ts";
7
+
8
+ const macosPlatformBackend: ComputerUsePlatformBackend = {
9
+ name: "macos",
10
+ ensureReady: ensureMacosReady,
11
+ listApps: macosBackend.listApps,
12
+ listRoots: macosBackend.listRoots,
13
+ getFrontmost: macosBackend.getFrontmost,
14
+ focusWindow: macosBackend.focusWindow,
15
+ observe: macosBackend.observe,
16
+ act: macosBackend.act,
17
+ actBatch: macosBackend.actBatch,
18
+ readText: macosBackend.readText,
19
+ waitFor: macosBackend.waitFor,
20
+ isBrowserApp,
21
+ isChromeFamilyApp,
22
+ openBrowserLocation: openBrowserLocationWithAppleScript,
23
+ };
24
+
25
+ class UnsupportedPlatformBackend implements ComputerUsePlatformBackend {
26
+ readonly name: PlatformName;
27
+ private readonly platform: NodeJS.Platform;
28
+
29
+ constructor(platform: NodeJS.Platform) {
30
+ this.platform = platform;
31
+ this.name = platform === "win32" ? "windows" : "linux";
32
+ }
33
+
34
+ private unsupported(): never {
35
+ throw new Error(`tangu-computer-use does not support platform '${this.platform}' yet.`);
36
+ }
37
+
38
+ async ensureReady(): Promise<never> { this.unsupported(); }
39
+ async listApps(): Promise<never> { this.unsupported(); }
40
+ async listRoots(): Promise<never> { this.unsupported(); }
41
+ async getFrontmost(): Promise<never> { this.unsupported(); }
42
+ async focusWindow(): Promise<never> { this.unsupported(); }
43
+ async observe(): Promise<never> { this.unsupported(); }
44
+ async act(): Promise<never> { this.unsupported(); }
45
+ async readText(): Promise<never> { this.unsupported(); }
46
+ async waitFor(): Promise<never> { this.unsupported(); }
47
+ isBrowserApp(): never { this.unsupported(); }
48
+ isChromeFamilyApp(): never { this.unsupported(); }
49
+ async openBrowserLocation(): Promise<boolean> { this.unsupported(); }
50
+ }
51
+
52
+ export function platformBackendForRuntime(platform: NodeJS.Platform = process.platform): ComputerUsePlatformBackend {
53
+ if (platform === "darwin") return macosPlatformBackend;
54
+ if (platform === "win32") return windowsBackend;
55
+ if (platform === "linux") return linuxBackend;
56
+ return new UnsupportedPlatformBackend(platform);
57
+ }
58
+
59
+ export const currentPlatformBackend = platformBackendForRuntime();
@@ -0,0 +1,186 @@
1
+ import { parseLookResponse, type LookResponse } from "../../outline.ts";
2
+ import { assertPlatformArchitecture } from "../architecture.ts";
3
+ import { toBoolean, toFiniteNumber, toOptionalString } from "../coerce.ts";
4
+ import type {
5
+ ComputerUsePlatformBackend,
6
+ FramePoints,
7
+ HelperActResult,
8
+ PlatformActRequest,
9
+ PlatformApp,
10
+ PlatformFocusWindowResult,
11
+ PlatformFrontmostResult,
12
+ PlatformObserveRequest,
13
+ PlatformReadTextRequest,
14
+ PlatformReadTextResponse,
15
+ PlatformReadyState,
16
+ PlatformRoot,
17
+ PlatformRootKind,
18
+ PlatformRootQuery,
19
+ PlatformTarget,
20
+ PlatformWaitForRequest,
21
+ PlatformWaitForResponse,
22
+ } from "../types.ts";
23
+ import { LINUX_HELPER_PROTOCOL_VERSION, linuxHelper } from "./helper.ts";
24
+
25
+ function normalizedProcessName(appName: string): string {
26
+ return appName.toLowerCase().split("/").pop()!.replace(/\.desktop$/i, "");
27
+ }
28
+
29
+ function classifyBrowser(appName: string): false | "chromium" | "firefox" {
30
+ switch (normalizedProcessName(appName)) {
31
+ case "chrome":
32
+ case "google-chrome":
33
+ case "google-chrome-stable":
34
+ case "chromium":
35
+ case "chromium-browser":
36
+ case "microsoft-edge":
37
+ case "microsoft-edge-stable":
38
+ case "brave":
39
+ case "brave-browser":
40
+ case "vivaldi":
41
+ case "vivaldi-stable":
42
+ case "opera":
43
+ return "chromium";
44
+ case "firefox":
45
+ case "firefox-esr":
46
+ return "firefox";
47
+ default:
48
+ return false;
49
+ }
50
+ }
51
+
52
+ function parseFramePoints(raw: unknown): FramePoints {
53
+ const frame = (raw as any)?.framePoints ?? (raw as any)?.bounds ?? {};
54
+ return {
55
+ x: toFiniteNumber(frame.x, 0),
56
+ y: toFiniteNumber(frame.y, 0),
57
+ w: Math.max(1, toFiniteNumber(frame.w ?? frame.width, 1)),
58
+ h: Math.max(1, toFiniteNumber(frame.h ?? frame.height, 1)),
59
+ };
60
+ }
61
+
62
+ function parseRootKind(raw: unknown): PlatformRootKind {
63
+ return raw === "menu" || raw === "sheet" || raw === "dialog" || raw === "popover" || raw === "window" ? raw : "window";
64
+ }
65
+
66
+ function parseRoots(result: unknown): PlatformRoot[] {
67
+ const roots = Array.isArray(result) ? result : (result as any)?.roots;
68
+ if (!Array.isArray(roots)) return [];
69
+ return roots.map((raw: any, index): PlatformRoot => ({
70
+ kind: parseRootKind(raw?.kind),
71
+ rootRef: toOptionalString(raw?.rootRef ?? raw?.windowRef ?? raw?.ref),
72
+ windowRef: toOptionalString(raw?.windowRef ?? raw?.rootRef ?? raw?.ref),
73
+ windowId: Number.isFinite(raw?.windowId) ? Math.trunc(raw.windowId) : undefined,
74
+ pid: Number.isFinite(raw?.pid) ? Math.trunc(raw.pid) : undefined,
75
+ appName: toOptionalString(raw?.appName ?? raw?.processName),
76
+ bundleId: toOptionalString(raw?.bundleId ?? raw?.desktopId),
77
+ title: toOptionalString(raw?.title) ?? "",
78
+ role: toOptionalString(raw?.role),
79
+ subrole: toOptionalString(raw?.subrole),
80
+ zOrder: Math.trunc(toFiniteNumber(raw?.zOrder, index)),
81
+ framePoints: parseFramePoints(raw),
82
+ scaleFactor: Math.max(1, toFiniteNumber(raw?.scaleFactor, 1)),
83
+ isOnscreen: raw?.isOnscreen === undefined ? true : toBoolean(raw.isOnscreen),
84
+ isFocused: toBoolean(raw?.isFocused),
85
+ isMinimized: toBoolean(raw?.isMinimized),
86
+ isMain: toBoolean(raw?.isMain ?? raw?.isFocused),
87
+ isModal: toBoolean(raw?.isModal),
88
+ metadata: raw?.metadata,
89
+ }));
90
+ }
91
+
92
+ function appsFromRoots(roots: PlatformRoot[]): PlatformApp[] {
93
+ const seen = new Set<number>();
94
+ return roots.flatMap((root) => {
95
+ if (!root.pid || seen.has(root.pid)) return [];
96
+ seen.add(root.pid);
97
+ return [{
98
+ appName: root.appName ?? "Unknown",
99
+ bundleId: root.bundleId,
100
+ pid: root.pid,
101
+ isFrontmost: root.isFocused,
102
+ }];
103
+ });
104
+ }
105
+
106
+ function helperAction(request: PlatformActRequest): Record<string, unknown> {
107
+ if (!("focus" in request.target)) return { ...request };
108
+ return { ...request, target: request.target.focus, params: { ...request.params, preserveFocus: true } };
109
+ }
110
+
111
+ async function ensureReady(_ctx: unknown, state: PlatformReadyState, signal?: AbortSignal): Promise<PlatformReadyState> {
112
+ await linuxHelper.ensureInstalled(signal);
113
+ const diagnostics = await linuxHelper.command<any>("diagnostics", {}, { signal, timeoutMs: 5_000 });
114
+ if (diagnostics?.protocolVersion !== LINUX_HELPER_PROTOCOL_VERSION) {
115
+ throw new Error(`Linux helper protocol mismatch: expected ${LINUX_HELPER_PROTOCOL_VERSION}, got ${diagnostics?.protocolVersion ?? "unknown"}. Restart Tangu to use the installed helper.`);
116
+ }
117
+ assertPlatformArchitecture("Linux", diagnostics);
118
+ if (diagnostics?.accessibility === false) {
119
+ throw new Error("Linux accessibility is unavailable. Ensure AT-SPI is enabled and a D-Bus desktop accessibility bus is running.");
120
+ }
121
+ return { ...state, lastPermissionCheckAt: Date.now(), helperDiagnostics: diagnostics };
122
+ }
123
+
124
+ export const linuxBackend: ComputerUsePlatformBackend = {
125
+ name: "linux",
126
+ shutdown(): void {
127
+ linuxHelper.dispose();
128
+ },
129
+ ensureReady,
130
+ async listApps(signal?: AbortSignal): Promise<PlatformApp[]> {
131
+ return appsFromRoots(parseRoots(await linuxHelper.command("listRoots", {}, { signal })));
132
+ },
133
+ async listRoots(query: PlatformRootQuery, signal?: AbortSignal): Promise<PlatformRoot[]> {
134
+ const roots = parseRoots(await linuxHelper.command("listRoots", Number.isFinite(query.pid) ? { pid: Math.trunc(query.pid!) } : {}, { signal }));
135
+ const title = query.title?.trim().toLowerCase();
136
+ return title ? roots.filter((root) => root.title.trim().toLowerCase().includes(title)) : roots;
137
+ },
138
+ async getFrontmost(signal?: AbortSignal): Promise<PlatformFrontmostResult> {
139
+ const roots = parseRoots(await linuxHelper.command("listRoots", {}, { signal }));
140
+ const focused = roots.find((root) => root.isFocused) ?? roots[0];
141
+ if (!focused?.pid) throw new Error("No frontmost window was available.");
142
+ return {
143
+ appName: focused.appName ?? "Unknown",
144
+ bundleId: focused.bundleId,
145
+ pid: focused.pid,
146
+ windowTitle: focused.title,
147
+ windowId: focused.windowId,
148
+ rootRef: focused.rootRef,
149
+ };
150
+ },
151
+ async focusWindow(target: PlatformTarget, signal?: AbortSignal): Promise<PlatformFocusWindowResult> {
152
+ return await linuxHelper.command("focusWindow", { ...target }, { signal });
153
+ },
154
+ async observe(request: PlatformObserveRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<LookResponse> {
155
+ return parseLookResponse(await linuxHelper.command("look", {
156
+ ...request.target,
157
+ baseLookId: request.baseLookId,
158
+ maxDimension: request.maxDimension,
159
+ readText: request.readText,
160
+ scopeRef: request.scopeRef,
161
+ includeImage: request.includeImage,
162
+ }, options));
163
+ },
164
+ async act(request: PlatformActRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult> {
165
+ return await linuxHelper.command("act", helperAction(request), options);
166
+ },
167
+ async actBatch(requests: PlatformActRequest[], options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult> {
168
+ return await linuxHelper.command("actBatch", { actions: requests.map(helperAction) }, options);
169
+ },
170
+ async readText(args: PlatformReadTextRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformReadTextResponse> {
171
+ return await linuxHelper.command("atspiReadText", { ...args }, options);
172
+ },
173
+ async waitFor(args: PlatformWaitForRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformWaitForResponse> {
174
+ return await linuxHelper.command("atspiWaitFor", { ...args }, options);
175
+ },
176
+ isBrowserApp(appName: string): boolean {
177
+ return classifyBrowser(appName) !== false;
178
+ },
179
+ isChromeFamilyApp(appName: string): boolean {
180
+ return classifyBrowser(appName) === "chromium";
181
+ },
182
+ async openBrowserLocation(target: { appName: string; bundleId?: string }, url: string, signal?: AbortSignal): Promise<boolean> {
183
+ await linuxHelper.command("openBrowserLocation", { ...target, url }, { signal, timeoutMs: 10_000 });
184
+ return true;
185
+ },
186
+ };