@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,651 @@
1
+ export interface OutlineRect {
2
+ x: number;
3
+ y: number;
4
+ w: number;
5
+ h: number;
6
+ }
7
+
8
+ export interface OutlineText {
9
+ string: string;
10
+ confidence: number;
11
+ rect?: OutlineRect;
12
+ }
13
+
14
+ export interface OutlineScrollExtent {
15
+ seen: number;
16
+ total: number;
17
+ }
18
+
19
+ export interface OutlineNode {
20
+ ref: string;
21
+ wireRef?: string;
22
+ role: string;
23
+ subrole: string;
24
+ identifier: string;
25
+ title: string;
26
+ description: string;
27
+ value: string;
28
+ actions: string[];
29
+ canPress: boolean;
30
+ canFocus: boolean;
31
+ canSetValue: boolean;
32
+ canScroll: boolean;
33
+ canIncrement: boolean;
34
+ canDecrement: boolean;
35
+ isTextInput: boolean;
36
+ rect?: OutlineRect;
37
+ focused: boolean;
38
+ offscreen: boolean;
39
+ pictureOnly: boolean;
40
+ truncated: boolean;
41
+ scrollExtent?: OutlineScrollExtent;
42
+ text: OutlineText[];
43
+ children: OutlineNode[];
44
+ parent?: OutlineNode;
45
+ }
46
+
47
+ export interface Outline {
48
+ lookId: string;
49
+ root: OutlineNode;
50
+ nodes: OutlineNode[];
51
+ refToWireRef: Map<string, string>;
52
+ wireRefToRef: Map<string, string>;
53
+ }
54
+
55
+ export interface LookImage {
56
+ jpegBase64: string;
57
+ mimeType?: "image/jpeg" | "image/png";
58
+ width: number;
59
+ height: number;
60
+ }
61
+
62
+ export interface LookWindow {
63
+ windowId: number;
64
+ rootRef?: string;
65
+ kind?: string;
66
+ framePoints: OutlineRect;
67
+ scaleFactor: number;
68
+ isModal: boolean;
69
+ metadata?: Record<string, unknown>;
70
+ role: string;
71
+ subrole: string;
72
+ }
73
+
74
+ export interface LookResponse {
75
+ lookId: string;
76
+ capturedAt: number;
77
+ window: LookWindow;
78
+ image?: LookImage;
79
+ outline: OutlineNode;
80
+ timings: Record<string, number>;
81
+ readText?: { requested?: "auto" | "always" | "never"; executed: boolean };
82
+ parsedOutline?: Outline;
83
+ }
84
+
85
+ export interface OutlineSearchMatch {
86
+ ref: string;
87
+ role: string;
88
+ label: string;
89
+ actions: string[];
90
+ path: string;
91
+ matchReason?: "exact" | "prefix" | "substring" | "fuzzy" | "filter";
92
+ score?: number;
93
+ node: OutlineNode;
94
+ }
95
+
96
+ export type SerializedOutlineSearchMatch = Omit<OutlineSearchMatch, "node"> & { node?: never };
97
+
98
+ export interface FoldResult {
99
+ text: string;
100
+ renderedRefs: string[];
101
+ nodeCount: number;
102
+ fullUnfoldLineCount: number;
103
+ truncated: boolean;
104
+ }
105
+
106
+ export type OutlineChange =
107
+ | { type: "added"; ref: string; parent?: string; node: SerializedOutlineNode }
108
+ | { type: "updated"; ref: string; path: string[]; fields: Partial<Omit<SerializedOutlineNode, "children">> }
109
+ | { type: "removed"; ref: string; parent?: string };
110
+
111
+ export interface OutlineDiff {
112
+ changes: OutlineChange[];
113
+ changedNodeCount: number;
114
+ fullNodeCount: number;
115
+ useFullView: boolean;
116
+ reason?: "root_replaced" | "change_budget_exceeded" | "identity_confidence_low";
117
+ }
118
+
119
+ export type SerializedOutlineNode = Omit<OutlineNode, "parent" | "children"> & { children: SerializedOutlineNode[] };
120
+
121
+ export interface SerializedOutline {
122
+ lookId: string;
123
+ root: SerializedOutlineNode;
124
+ }
125
+
126
+ const DEFAULT_BUDGET = { maxDepth: 2, maxNodes: 150 };
127
+
128
+ function isRecord(value: unknown): value is Record<string, unknown> {
129
+ return typeof value === "object" && value !== null;
130
+ }
131
+
132
+ function toString(value: unknown): string {
133
+ return typeof value === "string" ? value : "";
134
+ }
135
+
136
+ function toNumber(value: unknown, fallback = 0): number {
137
+ if (typeof value === "number" && Number.isFinite(value)) return value;
138
+ if (typeof value === "string") {
139
+ const parsed = Number(value);
140
+ if (Number.isFinite(parsed)) return parsed;
141
+ }
142
+ return fallback;
143
+ }
144
+
145
+ function toBoolean(value: unknown): boolean {
146
+ return value === true;
147
+ }
148
+
149
+ function parseRect(raw: unknown): OutlineRect {
150
+ const rect = isRecord(raw) ? raw : {};
151
+ return {
152
+ x: toNumber(rect.x),
153
+ y: toNumber(rect.y),
154
+ w: Math.max(0, toNumber(rect.w)),
155
+ h: Math.max(0, toNumber(rect.h)),
156
+ };
157
+ }
158
+
159
+ function parseText(raw: unknown): OutlineText[] {
160
+ if (!Array.isArray(raw)) return [];
161
+ return raw
162
+ .map((item): OutlineText | undefined => {
163
+ if (!isRecord(item)) return undefined;
164
+ const string = toString(item.string);
165
+ if (!string) return undefined;
166
+ return {
167
+ string,
168
+ confidence: Math.max(0, Math.min(1, toNumber(item.confidence))),
169
+ rect: parseRect(item.rect),
170
+ };
171
+ })
172
+ .filter((item): item is OutlineText => Boolean(item));
173
+ }
174
+
175
+ function parseNode(raw: unknown, parent?: OutlineNode): OutlineNode {
176
+ const record = isRecord(raw) ? raw : {};
177
+ const wireRef = toString(record.ref) || undefined;
178
+ const node: OutlineNode = {
179
+ ref: "",
180
+ wireRef,
181
+ role: toString(record.role),
182
+ subrole: toString(record.subrole),
183
+ identifier: toString(record.identifier),
184
+ title: toString(record.title),
185
+ description: toString(record.description),
186
+ value: toString(record.value),
187
+ actions: Array.isArray(record.actions) ? record.actions.filter((value): value is string => typeof value === "string") : [],
188
+ canPress: toBoolean(record.canPress),
189
+ canFocus: toBoolean(record.canFocus),
190
+ canSetValue: toBoolean(record.canSetValue),
191
+ canScroll: toBoolean(record.canScroll),
192
+ canIncrement: toBoolean(record.canIncrement),
193
+ canDecrement: toBoolean(record.canDecrement),
194
+ isTextInput: toBoolean(record.isTextInput),
195
+ rect: parseRect(record.rect),
196
+ focused: toBoolean(record.focused),
197
+ offscreen: toBoolean(record.offscreen),
198
+ pictureOnly: toBoolean(record.pictureOnly),
199
+ truncated: toBoolean(record.truncated),
200
+ scrollExtent: isRecord(record.scrollExtent)
201
+ ? { seen: Math.max(0, Math.trunc(toNumber(record.scrollExtent.seen))), total: Math.max(0, Math.trunc(toNumber(record.scrollExtent.total))) }
202
+ : undefined,
203
+ text: parseText(record.text),
204
+ children: [],
205
+ parent,
206
+ };
207
+ node.children = (Array.isArray(record.children) ? record.children : []).map((child) => parseNode(child, node));
208
+ return node;
209
+ }
210
+
211
+ export function parseLookResponse(raw: unknown): LookResponse {
212
+ const record = isRecord(raw) ? raw : {};
213
+ const image = isRecord(record.image) ? record.image : undefined;
214
+ const window = isRecord(record.window) ? record.window : {};
215
+ const metadata = isRecord(window.metadata) ? window.metadata : undefined;
216
+ const outline = buildOutline(toString(record.lookId), parseNode(record.outline));
217
+ const readText = isRecord(record.readText) ? record.readText : undefined;
218
+ const requestedReadText = readText?.requested === "auto" || readText?.requested === "always" || readText?.requested === "never" ? readText.requested : undefined;
219
+ const look: LookResponse = {
220
+ lookId: toString(record.lookId),
221
+ capturedAt: toNumber(record.capturedAt, Date.now() / 1000),
222
+ window: {
223
+ windowId: Math.trunc(toNumber(window.windowId)),
224
+ rootRef: toString(window.rootRef) || undefined,
225
+ kind: toString(window.kind) || undefined,
226
+ framePoints: parseRect(window.framePoints),
227
+ scaleFactor: Math.max(1, toNumber(window.scaleFactor, 1)),
228
+ isModal: toBoolean(window.isModal),
229
+ metadata,
230
+ role: toString(window.role),
231
+ subrole: toString(window.subrole),
232
+ },
233
+ image: image ? {
234
+ jpegBase64: toString(image.jpegBase64),
235
+ mimeType: image.mimeType === "image/png" ? "image/png" : "image/jpeg",
236
+ width: Math.max(1, Math.trunc(toNumber(image.width, 1))),
237
+ height: Math.max(1, Math.trunc(toNumber(image.height, 1))),
238
+ } : undefined,
239
+ outline: outline.root,
240
+ timings: isRecord(record.timings) ? Object.fromEntries(Object.entries(record.timings).map(([key, value]) => [key, toNumber(value)])) : {},
241
+ readText: readText ? { requested: requestedReadText, executed: toBoolean(readText.executed) } : undefined,
242
+ parsedOutline: outline,
243
+ };
244
+ if (!look.lookId) throw new Error("Helper returned a look without lookId.");
245
+ return look;
246
+ }
247
+
248
+ function buildOutline(lookId: string, root: OutlineNode): Outline {
249
+ const nodes: OutlineNode[] = [];
250
+ const refToWireRef = new Map<string, string>();
251
+ const wireRefToRef = new Map<string, string>();
252
+ const queue = [root];
253
+ let index = 0;
254
+ while (index < queue.length) {
255
+ const node = queue[index++];
256
+ node.ref = `@e${nodes.length + 1}`;
257
+ nodes.push(node);
258
+ if (node.wireRef) {
259
+ refToWireRef.set(node.ref, node.wireRef);
260
+ wireRefToRef.set(node.wireRef, node.ref);
261
+ }
262
+ queue.push(...node.children);
263
+ }
264
+ return { lookId, root, nodes, refToWireRef, wireRefToRef };
265
+ }
266
+
267
+ export function nodeByRef(outline: Outline, ref: string): OutlineNode | undefined {
268
+ return outline.nodes.find((node) => node.ref === ref || node.wireRef === ref);
269
+ }
270
+
271
+ export function outlineNodeLabel(node: OutlineNode): string {
272
+ return node.title || node.description || node.value || node.identifier || node.text.map((item) => item.string).join(" ").trim();
273
+ }
274
+
275
+ function displayName(node: OutlineNode): string {
276
+ const label = outlineNodeLabel(node);
277
+ return `${node.role || "AXUnknown"}${node.subrole ? `/${node.subrole}` : ""}${label ? ` ${JSON.stringify(label)}` : ""}`;
278
+ }
279
+
280
+ export function outlineNodePath(node: OutlineNode): string {
281
+ const parts: string[] = [];
282
+ let current: OutlineNode | undefined = node;
283
+ while (current) {
284
+ parts.unshift(displayName(current));
285
+ current = current.parent;
286
+ }
287
+ return parts.join(" ▸ ");
288
+ }
289
+
290
+ function countDescendants(node: OutlineNode): { total: number; roles: Map<string, number>; pictureOnly: number } {
291
+ const roles = new Map<string, number>();
292
+ let total = 0;
293
+ let pictureOnly = 0;
294
+ const visit = (current: OutlineNode) => {
295
+ for (const child of current.children) {
296
+ total += 1;
297
+ if (child.pictureOnly) pictureOnly += 1;
298
+ const role = child.pictureOnly ? "picture-only" : roleName(child.role);
299
+ roles.set(role, (roles.get(role) ?? 0) + 1);
300
+ visit(child);
301
+ }
302
+ };
303
+ visit(node);
304
+ return { total, roles, pictureOnly };
305
+ }
306
+
307
+ function roleName(role: string): string {
308
+ const stripped = role.replace(/^AX/, "").toLowerCase();
309
+ return stripped || "nodes";
310
+ }
311
+
312
+ function plural(count: number, singular: string): string {
313
+ if (singular === "picture-only") return "picture-only";
314
+ return `${singular}${count === 1 ? "" : "s"}`;
315
+ }
316
+
317
+ function foldedSummary(node: OutlineNode): string {
318
+ const counts = countDescendants(node);
319
+ const roleCounts = [...counts.roles.entries()]
320
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
321
+ .slice(0, 4)
322
+ .map(([role, count]) => `${count} ${plural(count, role)}`);
323
+ const countText = `${counts.total}: ${roleCounts.join(", ") || "0 children"}`;
324
+ const scroll = node.scrollExtent ? ` [scrollable ${node.scrollExtent.seen}/${node.scrollExtent.total}]` : "";
325
+ return ` ▸ (${countText})${scroll}`;
326
+ }
327
+
328
+ function annotationText(node: OutlineNode): string {
329
+ const annotations = [
330
+ node.offscreen ? "offscreen" : undefined,
331
+ node.pictureOnly ? "pictureOnly" : undefined,
332
+ node.truncated ? "truncated" : undefined,
333
+ node.scrollExtent ? `scrollable ${node.scrollExtent.seen}/${node.scrollExtent.total}` : undefined,
334
+ ].filter((item): item is string => Boolean(item));
335
+ return annotations.length ? ` [${annotations.join(", ")}]` : "";
336
+ }
337
+
338
+ function lineForNode(node: OutlineNode, depth: number, folded: boolean): string {
339
+ const actions = node.actions.length ? ` {${node.actions.join(",")}}` : "";
340
+ return `${" ".repeat(depth)}${node.ref} ${displayName(node)}${actions}${annotationText(node)}${folded ? foldedSummary(node) : ""}`;
341
+ }
342
+
343
+ function pathRefs(node: OutlineNode): string[] {
344
+ const refs: string[] = [];
345
+ let current: OutlineNode | undefined = node;
346
+ while (current) {
347
+ refs.unshift(current.ref);
348
+ current = current.parent;
349
+ }
350
+ return refs;
351
+ }
352
+
353
+ function defaultUnfoldRefs(outline: Outline): Set<string> {
354
+ const refs = new Set<string>([outline.root.ref]);
355
+ for (const node of outline.nodes) {
356
+ if (node.truncated || node.parent?.role === "AXSheet" || node.role === "AXSheet" || node.role === "AXDialog") {
357
+ for (const ref of pathRefs(node)) refs.add(ref);
358
+ }
359
+ if (node.focused) {
360
+ for (const ref of pathRefs(node)) refs.add(ref);
361
+ for (const child of node.children) refs.add(child.ref);
362
+ }
363
+ }
364
+ return refs;
365
+ }
366
+
367
+ export function foldToBudget(outline: Outline, budget: Partial<typeof DEFAULT_BUDGET> = {}, unfoldPaths: string[] = []): FoldResult {
368
+ const maxDepth = Math.max(0, Math.trunc(budget.maxDepth ?? DEFAULT_BUDGET.maxDepth));
369
+ const maxNodes = Math.max(1, Math.trunc(budget.maxNodes ?? DEFAULT_BUDGET.maxNodes));
370
+ const unfolded = defaultUnfoldRefs(outline);
371
+ for (const ref of unfoldPaths) {
372
+ const node = nodeByRef(outline, ref);
373
+ if (!node) continue;
374
+ for (const pathRef of pathRefs(node)) unfolded.add(pathRef);
375
+ for (const child of node.children) unfolded.add(child.ref);
376
+ }
377
+
378
+ const lines: string[] = [];
379
+ const renderedRefs: string[] = [];
380
+ let truncated = false;
381
+ const render = (node: OutlineNode, depth: number) => {
382
+ if (lines.length >= maxNodes) {
383
+ truncated = true;
384
+ return;
385
+ }
386
+ const shouldUnfold = depth < maxDepth || unfolded.has(node.ref);
387
+ const folded = node.children.length > 0 && !shouldUnfold;
388
+ lines.push(lineForNode(node, depth, folded));
389
+ renderedRefs.push(node.ref);
390
+ if (folded) return;
391
+ for (const child of node.children) render(child, depth + 1);
392
+ };
393
+ render(outline.root, 0);
394
+ if (truncated) {
395
+ const remaining = Math.max(0, outline.nodes.length - renderedRefs.length);
396
+ lines.push(`… render budget reached: ${remaining} more nodes not shown; use search_ui or expand_ui(@eN)`);
397
+ }
398
+ return {
399
+ text: lines.join("\n"),
400
+ renderedRefs,
401
+ nodeCount: outline.nodes.length,
402
+ fullUnfoldLineCount: outline.nodes.length,
403
+ truncated,
404
+ };
405
+ }
406
+
407
+ function actionMatches(node: OutlineNode, action: string): boolean {
408
+ const query = action.toLowerCase();
409
+ if (query === "press" && node.canPress) return true;
410
+ if (query === "focus" && node.canFocus) return true;
411
+ if ((query === "setvalue" || query === "set_text" || query === "settext") && node.canSetValue) return true;
412
+ if (query === "scroll" && node.canScroll) return true;
413
+ if (query === "increment" && node.canIncrement) return true;
414
+ if (query === "decrement" && node.canDecrement) return true;
415
+ return node.actions.some((candidate) => candidate.toLowerCase().includes(query));
416
+ }
417
+
418
+ function normalizedSearchRole(value: string): string {
419
+ return value.trim().toLowerCase().replace(/^ax/, "").replace(/[ _-]+/g, "");
420
+ }
421
+
422
+ function damerauLevenshtein(a: string, b: string): number {
423
+ const rows = a.length + 1;
424
+ const cols = b.length + 1;
425
+ const matrix = Array.from({ length: rows }, (_, i) => Array.from({ length: cols }, (_, j) => i === 0 ? j : j === 0 ? i : 0));
426
+ for (let i = 1; i < rows; i += 1) {
427
+ for (let j = 1; j < cols; j += 1) {
428
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
429
+ matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost);
430
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) matrix[i][j] = Math.min(matrix[i][j], matrix[i - 2][j - 2] + 1);
431
+ }
432
+ }
433
+ return matrix[a.length][b.length];
434
+ }
435
+
436
+ export function rankedTextMatch(values: string[], text: string): { reason: "exact" | "prefix" | "substring" | "fuzzy"; score: number } | undefined {
437
+ const query = text.toLowerCase().replace(/\s+/g, " ").trim().slice(0, 256);
438
+ if (!query) return undefined;
439
+ const candidates = values.map((value) => value.toLowerCase().replace(/\s+/g, " ").trim().slice(0, 512)).filter(Boolean);
440
+ if (candidates.some((value) => value === query)) return { reason: "exact", score: 1 };
441
+ if (candidates.some((value) => value.startsWith(query) || value.split(/\W+/).some((token) => token.startsWith(query)))) return { reason: "prefix", score: 0.95 };
442
+ if (candidates.some((value) => value.includes(query))) return { reason: "substring", score: 0.9 };
443
+ let score = 0;
444
+ for (const value of candidates) {
445
+ for (const candidate of [value, ...value.split(/\W+/)]) {
446
+ if (!candidate || candidate.length > 256) continue;
447
+ score = Math.max(score, 1 - damerauLevenshtein(query, candidate) / Math.max(query.length, candidate.length));
448
+ }
449
+ }
450
+ return score >= 0.72 ? { reason: "fuzzy", score } : undefined;
451
+ }
452
+
453
+ export function searchOutlineRanked(outline: Outline, text?: string, role?: string, capability?: string, limit = 12): { matches: OutlineSearchMatch[]; totalMatches: number } {
454
+ const query = text?.trim();
455
+ const roleQuery = role ? normalizedSearchRole(role) : undefined;
456
+ const actionQuery = capability?.trim();
457
+ const strong: Array<OutlineSearchMatch & { order: number }> = [];
458
+ const fuzzy: Array<OutlineSearchMatch & { order: number }> = [];
459
+ for (const [order, node] of outline.nodes.entries()) {
460
+ if (roleQuery && normalizedSearchRole(node.role) !== roleQuery) continue;
461
+ if (actionQuery && !actionMatches(node, actionQuery)) continue;
462
+ const label = outlineNodeLabel(node);
463
+ const match = query ? rankedTextMatch([label, node.identifier, node.title, node.description, node.value, ...node.text.map((item) => item.string)], query) : undefined;
464
+ if (query && !match) continue;
465
+ const result = { ref: node.ref, role: node.role, label, actions: node.actions, path: outlineNodePath(node), matchReason: match?.reason ?? "filter" as const, score: match?.score ?? 1, node, order };
466
+ (match?.reason === "fuzzy" ? fuzzy : strong).push(result);
467
+ }
468
+ const rank = { exact: 0, prefix: 1, substring: 2, filter: 3, fuzzy: 4 } as const;
469
+ const sorted = [...strong.sort((a, b) => rank[a.matchReason!] - rank[b.matchReason!] || b.score! - a.score! || a.order - b.order)];
470
+ const useFuzzy = sorted.length < limit;
471
+ if (useFuzzy) sorted.push(...fuzzy.sort((a, b) => b.score! - a.score! || a.order - b.order));
472
+ return { matches: sorted.slice(0, limit).map(({ order: _order, ...match }) => match), totalMatches: strong.length + (useFuzzy ? fuzzy.length : 0) };
473
+ }
474
+
475
+ export function searchOutline(outline: Outline, text?: string, role?: string, action?: string, limit = 50): OutlineSearchMatch[] {
476
+ const query = text?.trim().toLowerCase();
477
+ const roleQuery = role?.trim();
478
+ const actionQuery = action?.trim();
479
+ const matches: OutlineSearchMatch[] = [];
480
+ for (const node of outline.nodes) {
481
+ const label = outlineNodeLabel(node);
482
+ // outlineNodeLabel short-circuits (title || description || value), so
483
+ // list the fields individually too or a titled node's value/description
484
+ // can never match.
485
+ const haystack = [label, node.role, node.subrole, node.identifier, node.title, node.description, node.value, ...node.text.map((item) => item.string)].join(" ").toLowerCase();
486
+ if (query && !haystack.includes(query)) continue;
487
+ if (roleQuery && node.role !== roleQuery) continue;
488
+ if (actionQuery && !actionMatches(node, actionQuery)) continue;
489
+ matches.push({ ref: node.ref, role: node.role, label, actions: node.actions, path: outlineNodePath(node), node });
490
+ if (matches.length >= limit) break;
491
+ }
492
+ return matches;
493
+ }
494
+
495
+ export function serializeOutlineSearchMatch(match: OutlineSearchMatch): SerializedOutlineSearchMatch {
496
+ const { node: _node, ...serialized } = match;
497
+ return serialized;
498
+ }
499
+
500
+ export function countOutlineNodes(root: OutlineNode): number {
501
+ let count = 1;
502
+ for (const child of root.children) count += countOutlineNodes(child);
503
+ return count;
504
+ }
505
+
506
+ export function serializeOutline(outline: Outline): SerializedOutline {
507
+ return { lookId: outline.lookId, root: serializeOutlineNode(outline.root) };
508
+ }
509
+
510
+ export function serializeOutlineNode(node: OutlineNode): SerializedOutlineNode {
511
+ const { parent: _parent, children, ...rest } = node;
512
+ return { ...rest, children: children.map(serializeOutlineNode) };
513
+ }
514
+
515
+ export function serializeOutlineNodeShallow(node: OutlineNode): SerializedOutlineNode {
516
+ const { parent: _parent, children: _children, ...rest } = node;
517
+ return { ...rest, children: [] };
518
+ }
519
+
520
+ export function restoreOutline(serialized: SerializedOutline): Outline {
521
+ const nodes: OutlineNode[] = [];
522
+ const refToWireRef = new Map<string, string>();
523
+ const wireRefToRef = new Map<string, string>();
524
+ const restoreNode = (raw: SerializedOutlineNode, parent?: OutlineNode): OutlineNode => {
525
+ const node: OutlineNode = { ...raw, children: [], parent };
526
+ nodes.push(node);
527
+ if (node.wireRef) {
528
+ refToWireRef.set(node.ref, node.wireRef);
529
+ wireRefToRef.set(node.wireRef, node.ref);
530
+ }
531
+ node.children = raw.children.map((child) => restoreNode(child, node));
532
+ return node;
533
+ };
534
+ const root = restoreNode(serialized.root);
535
+ return { lookId: serialized.lookId, root, nodes, refToWireRef, wireRefToRef };
536
+ }
537
+
538
+ function outlineRefNumber(ref: string): number {
539
+ const match = /^@e(\d+)$/.exec(ref);
540
+ return match ? Number(match[1]) : 0;
541
+ }
542
+
543
+ function clearScopedRects(node: OutlineNode): void {
544
+ // Scoped look geometry is in the scoped image's pixel space, not the full look.
545
+ // Until helper-side act carries stable look geometry, grafted nodes are AX-only
546
+ // for ref actions; coordinate fallback must re-observe the full window.
547
+ node.rect = undefined;
548
+ for (const text of node.text) text.rect = undefined;
549
+ for (const child of node.children) clearScopedRects(child);
550
+ }
551
+
552
+ function rebuildIndexes(outline: Outline): void {
553
+ outline.nodes = [];
554
+ outline.refToWireRef = new Map();
555
+ outline.wireRefToRef = new Map();
556
+ const queue = [outline.root];
557
+ let index = 0;
558
+ while (index < queue.length) {
559
+ const node = queue[index++];
560
+ outline.nodes.push(node);
561
+ if (node.wireRef) {
562
+ outline.refToWireRef.set(node.ref, node.wireRef);
563
+ outline.wireRefToRef.set(node.wireRef, node.ref);
564
+ }
565
+ queue.push(...node.children);
566
+ }
567
+ }
568
+
569
+ function copyNodeFields(target: OutlineNode, source: OutlineNode, preserveWireRef = false): void {
570
+ if (!preserveWireRef) target.wireRef = source.wireRef;
571
+ target.role = source.role;
572
+ target.subrole = source.subrole;
573
+ target.identifier = source.identifier;
574
+ target.title = source.title;
575
+ target.description = source.description;
576
+ target.value = source.value;
577
+ target.actions = [...source.actions];
578
+ target.canPress = source.canPress;
579
+ target.canFocus = source.canFocus;
580
+ target.canSetValue = source.canSetValue;
581
+ target.canScroll = source.canScroll;
582
+ target.canIncrement = source.canIncrement;
583
+ target.canDecrement = source.canDecrement;
584
+ target.isTextInput = source.isTextInput;
585
+ target.rect = source.rect;
586
+ target.focused = source.focused;
587
+ target.offscreen = source.offscreen;
588
+ target.pictureOnly = source.pictureOnly;
589
+ target.truncated = false;
590
+ target.scrollExtent = source.scrollExtent ? { ...source.scrollExtent } : undefined;
591
+ target.text = source.text.map((text) => ({ ...text, rect: text.rect ? { ...text.rect } : undefined }));
592
+ }
593
+
594
+ function preserveUnreused(node: OutlineNode, parent: OutlineNode, used: Set<OutlineNode>): OutlineNode | undefined {
595
+ if (used.has(node)) return undefined;
596
+ node.parent = parent;
597
+ node.children = node.children
598
+ .map((child) => preserveUnreused(child, node, used))
599
+ .filter((child): child is OutlineNode => Boolean(child));
600
+ return node;
601
+ }
602
+
603
+ function cloneForGraft(source: OutlineNode, parent: OutlineNode, nextRef: () => string, reusableByWireRef: Map<string, OutlineNode>, used: Set<OutlineNode>): OutlineNode {
604
+ const existing = source.wireRef ? reusableByWireRef.get(source.wireRef) : undefined;
605
+ if (existing) used.add(existing);
606
+ const oldChildren = existing?.children ?? [];
607
+ const node: OutlineNode = existing ?? {
608
+ ...source,
609
+ ref: nextRef(),
610
+ children: [],
611
+ parent,
612
+ text: [],
613
+ actions: [],
614
+ scrollExtent: undefined,
615
+ };
616
+ copyNodeFields(node, source, Boolean(existing));
617
+ node.parent = parent;
618
+ const graftedChildren = source.children.map((child) => cloneForGraft(child, node, nextRef, reusableByWireRef, used));
619
+ const preservedChildren = oldChildren
620
+ .map((child) => preserveUnreused(child, node, used))
621
+ .filter((child): child is OutlineNode => Boolean(child));
622
+ node.children = [...graftedChildren, ...preservedChildren];
623
+ return node;
624
+ }
625
+
626
+ export function graftScopedOutline(outline: Outline, targetRef: string, scoped: Outline): OutlineNode {
627
+ const target = nodeByRef(outline, targetRef);
628
+ if (!target) throw new Error(`Cannot graft scoped outline: target ${targetRef} is not in the current outline.`);
629
+ const targetRect = target.rect ? { ...target.rect } : undefined;
630
+ const reusableByWireRef = new Map<string, OutlineNode>();
631
+ const collect = (node: OutlineNode) => {
632
+ if (node.wireRef) reusableByWireRef.set(node.wireRef, node);
633
+ for (const child of node.children) collect(child);
634
+ };
635
+ collect(target);
636
+ let nextNumber = Math.max(...outline.nodes.map((node) => outlineRefNumber(node.ref)), 0) + 1;
637
+ const nextRef = () => `@e${nextNumber++}`;
638
+ const oldChildren = target.children;
639
+ const used = new Set<OutlineNode>([target]);
640
+ copyNodeFields(target, scoped.root, true);
641
+ target.ref = targetRef;
642
+ target.rect = targetRect;
643
+ const graftedChildren = scoped.root.children.map((child) => cloneForGraft(child, target, nextRef, reusableByWireRef, used));
644
+ const preservedChildren = oldChildren
645
+ .map((child) => preserveUnreused(child, target, used))
646
+ .filter((child): child is OutlineNode => Boolean(child));
647
+ target.children = [...graftedChildren, ...preservedChildren];
648
+ for (const child of target.children) clearScopedRects(child);
649
+ rebuildIndexes(outline);
650
+ return target;
651
+ }