@bun-win32/uia 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +130 -0
- package/README.md +79 -0
- package/agent.ts +83 -0
- package/automation.ts +51 -0
- package/cache.ts +67 -0
- package/com.ts +62 -0
- package/condition.ts +132 -0
- package/constants.ts +233 -0
- package/element.ts +512 -0
- package/index.ts +40 -0
- package/input.ts +149 -0
- package/msaa.ts +99 -0
- package/package.json +86 -0
- package/patterns.ts +234 -0
- package/png.ts +75 -0
- package/reads.ts +66 -0
- package/tree.ts +95 -0
- package/window.ts +107 -0
package/msaa.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// MSAA (oleacc IAccessible) fallback for legacy / owner-draw windows that expose no useful UIA tree.
|
|
2
|
+
// IAccessible is IDispatch-derived (IUnknown 0-2, IDispatch 3-6, then its members): get_accChildCount
|
|
3
|
+
// 8, get_accName 10, get_accRole 13. The VARIANT child-id is passed by pointer (the 16-byte aggregate
|
|
4
|
+
// goes by hidden reference). AccessibleChildren returns VARIANTs: VT_DISPATCH → QI to IAccessible;
|
|
5
|
+
// VT_I4 → a simple child-id leaf of the same parent (never a pointer).
|
|
6
|
+
|
|
7
|
+
import { FFIType } from 'bun:ffi';
|
|
8
|
+
|
|
9
|
+
import Oleacc, { IID_IAccessible, OBJID } from '@bun-win32/oleacc';
|
|
10
|
+
|
|
11
|
+
import { comRelease, guid, vcall } from './com';
|
|
12
|
+
import { S_OK, VT_DISPATCH, VT_I4 } from './constants';
|
|
13
|
+
import { decodeBstr } from './reads';
|
|
14
|
+
|
|
15
|
+
const IACC_QUERYINTERFACE = 0;
|
|
16
|
+
const IACC_GET_ACCCHILDCOUNT = 8;
|
|
17
|
+
const IACC_GET_ACCNAME = 10;
|
|
18
|
+
const IACC_GET_ACCROLE = 13;
|
|
19
|
+
const VARIANT_SIZE = 16;
|
|
20
|
+
const CHILDID_SELF = 0;
|
|
21
|
+
|
|
22
|
+
export interface MsaaNode {
|
|
23
|
+
name: string;
|
|
24
|
+
role: number;
|
|
25
|
+
children: MsaaNode[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function childVariant(childId: number): Buffer {
|
|
29
|
+
const variant = Buffer.alloc(VARIANT_SIZE);
|
|
30
|
+
variant.writeUInt16LE(VT_I4, 0);
|
|
31
|
+
variant.writeInt32LE(childId, 8);
|
|
32
|
+
return variant;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function accName(accessible: bigint, childId: number): string {
|
|
36
|
+
const out = Buffer.alloc(8);
|
|
37
|
+
if (vcall(accessible, IACC_GET_ACCNAME, [FFIType.ptr, FFIType.ptr], [childVariant(childId).ptr!, out.ptr!]) !== S_OK) return '';
|
|
38
|
+
return decodeBstr(out.readBigUInt64LE(0));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function accRole(accessible: bigint, childId: number): number {
|
|
42
|
+
const roleVariant = Buffer.alloc(VARIANT_SIZE);
|
|
43
|
+
if (vcall(accessible, IACC_GET_ACCROLE, [FFIType.ptr, FFIType.ptr], [childVariant(childId).ptr!, roleVariant.ptr!]) !== S_OK) return -1;
|
|
44
|
+
return roleVariant.readUInt16LE(0) === VT_I4 ? roleVariant.readInt32LE(8) : -1;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function accChildCount(accessible: bigint): number {
|
|
48
|
+
const out = Buffer.alloc(4);
|
|
49
|
+
if (vcall(accessible, IACC_GET_ACCCHILDCOUNT, [FFIType.ptr], [out.ptr!]) !== S_OK) return 0;
|
|
50
|
+
return out.readInt32LE(0);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Acquire the root IAccessible for a window via MSAA (OBJID_WINDOW). Returns 0n on failure. */
|
|
54
|
+
export function accessibleFromWindow(hWnd: bigint): bigint {
|
|
55
|
+
const out = Buffer.alloc(8);
|
|
56
|
+
if (Oleacc.AccessibleObjectFromWindow(hWnd, OBJID.OBJID_WINDOW >>> 0, guid(`{${IID_IAccessible}}`).ptr!, out.ptr!) !== S_OK) return 0n;
|
|
57
|
+
return out.readBigUInt64LE(0);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function walk(accessible: bigint, childId: number, maxDepth: number, depth: number): MsaaNode {
|
|
61
|
+
const node: MsaaNode = { name: accName(accessible, childId), role: accRole(accessible, childId), children: [] };
|
|
62
|
+
if (childId !== CHILDID_SELF || depth >= maxDepth) return node;
|
|
63
|
+
const count = accChildCount(accessible);
|
|
64
|
+
if (count <= 0) return node;
|
|
65
|
+
const children = Buffer.alloc(VARIANT_SIZE * count);
|
|
66
|
+
const obtained = Buffer.alloc(4);
|
|
67
|
+
if (Oleacc.AccessibleChildren(accessible, 0, count, children.ptr!, obtained.ptr!) !== S_OK) return node;
|
|
68
|
+
const got = obtained.readInt32LE(0);
|
|
69
|
+
for (let index = 0; index < got; index += 1) {
|
|
70
|
+
const base = index * VARIANT_SIZE;
|
|
71
|
+
const variantType = children.readUInt16LE(base);
|
|
72
|
+
if (variantType === VT_DISPATCH) {
|
|
73
|
+
const dispatch = children.readBigUInt64LE(base + 8);
|
|
74
|
+
if (dispatch === 0n) continue;
|
|
75
|
+
const childOut = Buffer.alloc(8);
|
|
76
|
+
const queried = vcall(dispatch, IACC_QUERYINTERFACE, [FFIType.ptr, FFIType.ptr], [guid(`{${IID_IAccessible}}`).ptr!, childOut.ptr!]);
|
|
77
|
+
const childAccessible = childOut.readBigUInt64LE(0);
|
|
78
|
+
if (queried === S_OK && childAccessible !== 0n) {
|
|
79
|
+
node.children.push(walk(childAccessible, CHILDID_SELF, maxDepth, depth + 1));
|
|
80
|
+
comRelease(childAccessible);
|
|
81
|
+
}
|
|
82
|
+
comRelease(dispatch);
|
|
83
|
+
} else if (variantType === VT_I4) {
|
|
84
|
+
node.children.push(walk(accessible, children.readInt32LE(base + 8), maxDepth, depth + 1));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return node;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Walk a window's MSAA (IAccessible) tree — the legacy/owner-draw fallback. Null when MSAA is absent. */
|
|
91
|
+
export function msaaTree(hWnd: bigint, maxDepth = 8): MsaaNode | null {
|
|
92
|
+
const root = accessibleFromWindow(hWnd);
|
|
93
|
+
if (root === 0n) return null;
|
|
94
|
+
try {
|
|
95
|
+
return walk(root, CHILDID_SELF, maxDepth, 0);
|
|
96
|
+
} finally {
|
|
97
|
+
comRelease(root);
|
|
98
|
+
}
|
|
99
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"author": "Stev Peifer <stev.p@outlook.com>",
|
|
3
|
+
"bugs": {
|
|
4
|
+
"url": "https://github.com/ObscuritySRL/bun-win32/issues"
|
|
5
|
+
},
|
|
6
|
+
"dependencies": {
|
|
7
|
+
"@bun-win32/combase": "1.1.0",
|
|
8
|
+
"@bun-win32/core": "1.1.4",
|
|
9
|
+
"@bun-win32/gdi32": "1.0.14",
|
|
10
|
+
"@bun-win32/oleacc": "1.0.1",
|
|
11
|
+
"@bun-win32/oleaut32": "1.0.1",
|
|
12
|
+
"@bun-win32/user32": "3.0.22"
|
|
13
|
+
},
|
|
14
|
+
"description": "Playwright for Windows desktop apps — query the UI Automation accessibility tree, invoke controls by name, type, and assert, from Bun. Zero native dependencies, no node-gyp, no server.",
|
|
15
|
+
"devDependencies": {
|
|
16
|
+
"@types/bun": "latest"
|
|
17
|
+
},
|
|
18
|
+
"exports": {
|
|
19
|
+
".": "./index.ts"
|
|
20
|
+
},
|
|
21
|
+
"license": "MIT",
|
|
22
|
+
"module": "index.ts",
|
|
23
|
+
"name": "@bun-win32/uia",
|
|
24
|
+
"peerDependencies": {
|
|
25
|
+
"typescript": "^5"
|
|
26
|
+
},
|
|
27
|
+
"private": false,
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
30
|
+
},
|
|
31
|
+
"homepage": "https://github.com/ObscuritySRL/bun-win32#readme",
|
|
32
|
+
"repository": {
|
|
33
|
+
"type": "git",
|
|
34
|
+
"url": "git://github.com/ObscuritySRL/bun-win32.git",
|
|
35
|
+
"directory": "packages/uia"
|
|
36
|
+
},
|
|
37
|
+
"type": "module",
|
|
38
|
+
"version": "1.0.0",
|
|
39
|
+
"main": "./index.ts",
|
|
40
|
+
"keywords": [
|
|
41
|
+
"accessibility",
|
|
42
|
+
"automation",
|
|
43
|
+
"bun",
|
|
44
|
+
"desktop",
|
|
45
|
+
"e2e",
|
|
46
|
+
"ffi",
|
|
47
|
+
"playwright",
|
|
48
|
+
"testing",
|
|
49
|
+
"typescript",
|
|
50
|
+
"uiautomation",
|
|
51
|
+
"win32",
|
|
52
|
+
"windows"
|
|
53
|
+
],
|
|
54
|
+
"files": [
|
|
55
|
+
"AI.md",
|
|
56
|
+
"README.md",
|
|
57
|
+
"agent.ts",
|
|
58
|
+
"automation.ts",
|
|
59
|
+
"cache.ts",
|
|
60
|
+
"com.ts",
|
|
61
|
+
"condition.ts",
|
|
62
|
+
"constants.ts",
|
|
63
|
+
"element.ts",
|
|
64
|
+
"index.ts",
|
|
65
|
+
"input.ts",
|
|
66
|
+
"msaa.ts",
|
|
67
|
+
"patterns.ts",
|
|
68
|
+
"png.ts",
|
|
69
|
+
"reads.ts",
|
|
70
|
+
"tree.ts",
|
|
71
|
+
"window.ts"
|
|
72
|
+
],
|
|
73
|
+
"sideEffects": false,
|
|
74
|
+
"engines": {
|
|
75
|
+
"bun": ">=1.1.0"
|
|
76
|
+
},
|
|
77
|
+
"scripts": {
|
|
78
|
+
"example:agent-grounding": "bun ./example/agent-grounding.ts",
|
|
79
|
+
"example:automate-calculator": "bun ./example/automate-calculator.ts",
|
|
80
|
+
"example:benchmark": "bun ./example/benchmark.ts",
|
|
81
|
+
"example:hello-uia": "bun ./example/hello-uia.ts",
|
|
82
|
+
"example:inspector": "bun ./example/inspector.ts",
|
|
83
|
+
"example:selftest": "bun ./example/uia.selftest.ts",
|
|
84
|
+
"example:uia-report": "bun ./example/uia-report.ts"
|
|
85
|
+
}
|
|
86
|
+
}
|
package/patterns.ts
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// Control-pattern actions. Each acquires the pattern via GetCurrentPattern (slot 16), invokes the
|
|
2
|
+
// method, and releases the pattern interface. State reads return the raw enum number; compare against
|
|
3
|
+
// the exported enums. Patterns the element does not support throw (actions) or return null/-1 (reads),
|
|
4
|
+
// pointing the caller at the SendInput fallbacks (.click()/.type()) where one exists.
|
|
5
|
+
|
|
6
|
+
import { FFIType, type Pointer } from 'bun:ffi';
|
|
7
|
+
|
|
8
|
+
import Oleaut32 from '@bun-win32/oleaut32';
|
|
9
|
+
|
|
10
|
+
import { comRelease, hresult, vcall } from './com';
|
|
11
|
+
import { PatternId, S_OK, SLOT } from './constants';
|
|
12
|
+
import { decodeBstr, getBstr, getDouble, getLong } from './reads';
|
|
13
|
+
|
|
14
|
+
export enum ToggleState {
|
|
15
|
+
Off = 0,
|
|
16
|
+
On = 1,
|
|
17
|
+
Indeterminate = 2,
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export enum ExpandCollapseState {
|
|
21
|
+
Collapsed = 0,
|
|
22
|
+
Expanded = 1,
|
|
23
|
+
PartiallyExpanded = 2,
|
|
24
|
+
LeafNode = 3,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export enum WindowVisualState {
|
|
28
|
+
Normal = 0,
|
|
29
|
+
Maximized = 1,
|
|
30
|
+
Minimized = 2,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Acquire a control pattern interface. Returns 0n when the element does not support it. */
|
|
34
|
+
function getPattern(ptr: bigint, patternId: number): bigint {
|
|
35
|
+
const out = Buffer.alloc(8);
|
|
36
|
+
if (vcall(ptr, SLOT.GetCurrentPattern, [FFIType.i32, FFIType.ptr], [patternId, out.ptr!]) !== S_OK) return 0n;
|
|
37
|
+
return out.readBigUInt64LE(0);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function invokeNoArg(pattern: bigint, slot: number, label: string): void {
|
|
41
|
+
const hr = vcall(pattern, slot, [], []);
|
|
42
|
+
if (hr !== S_OK) throw new Error(`${label} failed: ${hresult(hr)}`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Press the element via InvokePattern. */
|
|
46
|
+
export function invoke(ptr: bigint): void {
|
|
47
|
+
const pattern = getPattern(ptr, PatternId.Invoke);
|
|
48
|
+
if (pattern === 0n) throw new Error('element does not support InvokePattern — use .click() for the coordinate fallback');
|
|
49
|
+
try {
|
|
50
|
+
invokeNoArg(pattern, SLOT.Invoke, 'Invoke');
|
|
51
|
+
} finally {
|
|
52
|
+
comRelease(pattern);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Set a ValuePattern control's value (e.g. a text box) in one call — no keystrokes, no VARIANT. */
|
|
57
|
+
export function setValue(ptr: bigint, text: string): void {
|
|
58
|
+
const pattern = getPattern(ptr, PatternId.Value);
|
|
59
|
+
if (pattern === 0n) throw new Error('element does not support ValuePattern — use .type() to send keystrokes');
|
|
60
|
+
const bstr = Oleaut32.SysAllocString(Buffer.from(`${text}\0`, 'utf16le').ptr!);
|
|
61
|
+
try {
|
|
62
|
+
const hr = vcall(pattern, SLOT.SetValue, [FFIType.ptr], [bstr]);
|
|
63
|
+
if (hr !== S_OK) throw new Error(`ValuePattern.SetValue failed: ${hresult(hr)}`);
|
|
64
|
+
} finally {
|
|
65
|
+
Oleaut32.SysFreeString(bstr);
|
|
66
|
+
comRelease(pattern);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Read a ValuePattern control's value, or '' if unsupported. */
|
|
71
|
+
export function getValue(ptr: bigint): string {
|
|
72
|
+
const pattern = getPattern(ptr, PatternId.Value);
|
|
73
|
+
if (pattern === 0n) return '';
|
|
74
|
+
try {
|
|
75
|
+
return getBstr(pattern, SLOT.get_CurrentValue);
|
|
76
|
+
} finally {
|
|
77
|
+
comRelease(pattern);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Toggle a TogglePattern control (checkbox). */
|
|
82
|
+
export function toggle(ptr: bigint): void {
|
|
83
|
+
const pattern = getPattern(ptr, PatternId.Toggle);
|
|
84
|
+
if (pattern === 0n) throw new Error('element does not support TogglePattern');
|
|
85
|
+
try {
|
|
86
|
+
invokeNoArg(pattern, SLOT.Toggle, 'Toggle');
|
|
87
|
+
} finally {
|
|
88
|
+
comRelease(pattern);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Read a TogglePattern's state (ToggleState), or -1 if unsupported. */
|
|
93
|
+
export function toggleState(ptr: bigint): number {
|
|
94
|
+
const pattern = getPattern(ptr, PatternId.Toggle);
|
|
95
|
+
if (pattern === 0n) return -1;
|
|
96
|
+
try {
|
|
97
|
+
return getLong(pattern, SLOT.get_CurrentToggleState);
|
|
98
|
+
} finally {
|
|
99
|
+
comRelease(pattern);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Expand an ExpandCollapsePattern control (combo box, tree item). */
|
|
104
|
+
export function expand(ptr: bigint): void {
|
|
105
|
+
const pattern = getPattern(ptr, PatternId.ExpandCollapse);
|
|
106
|
+
if (pattern === 0n) throw new Error('element does not support ExpandCollapsePattern');
|
|
107
|
+
try {
|
|
108
|
+
invokeNoArg(pattern, SLOT.Expand, 'Expand');
|
|
109
|
+
} finally {
|
|
110
|
+
comRelease(pattern);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Collapse an ExpandCollapsePattern control. */
|
|
115
|
+
export function collapse(ptr: bigint): void {
|
|
116
|
+
const pattern = getPattern(ptr, PatternId.ExpandCollapse);
|
|
117
|
+
if (pattern === 0n) throw new Error('element does not support ExpandCollapsePattern');
|
|
118
|
+
try {
|
|
119
|
+
invokeNoArg(pattern, SLOT.Collapse, 'Collapse');
|
|
120
|
+
} finally {
|
|
121
|
+
comRelease(pattern);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Read an ExpandCollapsePattern's state (ExpandCollapseState), or -1 if unsupported. */
|
|
126
|
+
export function expandCollapseState(ptr: bigint): number {
|
|
127
|
+
const pattern = getPattern(ptr, PatternId.ExpandCollapse);
|
|
128
|
+
if (pattern === 0n) return -1;
|
|
129
|
+
try {
|
|
130
|
+
return getLong(pattern, SLOT.get_CurrentExpandCollapseState);
|
|
131
|
+
} finally {
|
|
132
|
+
comRelease(pattern);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Select a SelectionItemPattern control (list item, radio button), replacing the selection. */
|
|
137
|
+
export function select(ptr: bigint): void {
|
|
138
|
+
const pattern = getPattern(ptr, PatternId.SelectionItem);
|
|
139
|
+
if (pattern === 0n) throw new Error('element does not support SelectionItemPattern');
|
|
140
|
+
try {
|
|
141
|
+
invokeNoArg(pattern, SLOT.Select, 'Select');
|
|
142
|
+
} finally {
|
|
143
|
+
comRelease(pattern);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Whether a SelectionItemPattern control is selected (false if unsupported). */
|
|
148
|
+
export function isSelected(ptr: bigint): boolean {
|
|
149
|
+
const pattern = getPattern(ptr, PatternId.SelectionItem);
|
|
150
|
+
if (pattern === 0n) return false;
|
|
151
|
+
try {
|
|
152
|
+
return getLong(pattern, SLOT.get_CurrentIsSelected) !== 0;
|
|
153
|
+
} finally {
|
|
154
|
+
comRelease(pattern);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Scroll a ScrollItemPattern control into view within its scrollable container. */
|
|
159
|
+
export function scrollIntoView(ptr: bigint): void {
|
|
160
|
+
const pattern = getPattern(ptr, PatternId.ScrollItem);
|
|
161
|
+
if (pattern === 0n) throw new Error('element does not support ScrollItemPattern');
|
|
162
|
+
try {
|
|
163
|
+
invokeNoArg(pattern, SLOT.ScrollIntoView, 'ScrollIntoView');
|
|
164
|
+
} finally {
|
|
165
|
+
comRelease(pattern);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Set a RangeValuePattern control's value (slider). Throws if unsupported. */
|
|
170
|
+
export function setRangeValue(ptr: bigint, value: number): void {
|
|
171
|
+
const pattern = getPattern(ptr, PatternId.RangeValue);
|
|
172
|
+
if (pattern === 0n) throw new Error('element does not support RangeValuePattern');
|
|
173
|
+
try {
|
|
174
|
+
const hr = vcall(pattern, SLOT.SetValue, [FFIType.f64], [value]);
|
|
175
|
+
if (hr !== S_OK) throw new Error(`RangeValuePattern.SetValue failed: ${hresult(hr)}`);
|
|
176
|
+
} finally {
|
|
177
|
+
comRelease(pattern);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Read a RangeValuePattern control's value (slider), or NaN if unsupported. */
|
|
182
|
+
export function rangeValue(ptr: bigint): number {
|
|
183
|
+
const pattern = getPattern(ptr, PatternId.RangeValue);
|
|
184
|
+
if (pattern === 0n) return Number.NaN;
|
|
185
|
+
try {
|
|
186
|
+
return getDouble(pattern, SLOT.get_CurrentValue);
|
|
187
|
+
} finally {
|
|
188
|
+
comRelease(pattern);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Read the full text of a TextPattern document (the document range), or '' if unsupported. */
|
|
193
|
+
export function readText(ptr: bigint): string {
|
|
194
|
+
const pattern = getPattern(ptr, PatternId.Text);
|
|
195
|
+
if (pattern === 0n) return '';
|
|
196
|
+
try {
|
|
197
|
+
const rangeOut = Buffer.alloc(8);
|
|
198
|
+
if (vcall(pattern, SLOT.get_DocumentRange, [FFIType.ptr], [rangeOut.ptr!]) !== S_OK) return '';
|
|
199
|
+
const range = rangeOut.readBigUInt64LE(0);
|
|
200
|
+
if (range === 0n) return '';
|
|
201
|
+
try {
|
|
202
|
+
const textOut = Buffer.alloc(8);
|
|
203
|
+
if (vcall(range, SLOT.GetText, [FFIType.i32, FFIType.ptr], [-1, textOut.ptr!]) !== S_OK) return '';
|
|
204
|
+
return decodeBstr(textOut.readBigUInt64LE(0));
|
|
205
|
+
} finally {
|
|
206
|
+
comRelease(range);
|
|
207
|
+
}
|
|
208
|
+
} finally {
|
|
209
|
+
comRelease(pattern);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Close a window via WindowPattern. */
|
|
214
|
+
export function windowClose(ptr: bigint): void {
|
|
215
|
+
const pattern = getPattern(ptr, PatternId.Window);
|
|
216
|
+
if (pattern === 0n) throw new Error('element does not support WindowPattern');
|
|
217
|
+
try {
|
|
218
|
+
invokeNoArg(pattern, SLOT.Close, 'Close');
|
|
219
|
+
} finally {
|
|
220
|
+
comRelease(pattern);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Set a window's visual state (normal/maximized/minimized) via WindowPattern. */
|
|
225
|
+
export function setWindowVisualState(ptr: bigint, state: WindowVisualState): void {
|
|
226
|
+
const pattern = getPattern(ptr, PatternId.Window);
|
|
227
|
+
if (pattern === 0n) throw new Error('element does not support WindowPattern');
|
|
228
|
+
try {
|
|
229
|
+
const hr = vcall(pattern, SLOT.SetWindowVisualState, [FFIType.i32], [state]);
|
|
230
|
+
if (hr !== S_OK) throw new Error(`SetWindowVisualState failed: ${hresult(hr)}`);
|
|
231
|
+
} finally {
|
|
232
|
+
comRelease(pattern);
|
|
233
|
+
}
|
|
234
|
+
}
|
package/png.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Minimal pure-TS PNG encoder (adapted from @bun-win32/terminal's png.ts) for PrintWindow screenshots.
|
|
2
|
+
// `Bun.deflateSync` returns a raw DEFLATE stream, so the IDAT payload is hand-wrapped in a zlib
|
|
3
|
+
// container (0x78 0x01 + data + Adler-32 of the unfiltered scanlines).
|
|
4
|
+
|
|
5
|
+
const crcTable = (() => {
|
|
6
|
+
const table = new Uint32Array(256);
|
|
7
|
+
for (let index = 0; index < 256; index++) {
|
|
8
|
+
let value = index;
|
|
9
|
+
for (let bit = 0; bit < 8; bit++) value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
|
10
|
+
table[index] = value >>> 0;
|
|
11
|
+
}
|
|
12
|
+
return table;
|
|
13
|
+
})();
|
|
14
|
+
|
|
15
|
+
const crc32 = (bytes: Uint8Array): number => {
|
|
16
|
+
let value = 0xffffffff;
|
|
17
|
+
for (let index = 0; index < bytes.length; index++) value = crcTable[(value ^ bytes[index]!) & 0xff]! ^ (value >>> 8);
|
|
18
|
+
return (value ^ 0xffffffff) >>> 0;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const adler32 = (bytes: Uint8Array): number => {
|
|
22
|
+
let low = 1;
|
|
23
|
+
let high = 0;
|
|
24
|
+
for (let index = 0; index < bytes.length; index++) {
|
|
25
|
+
low = (low + bytes[index]!) % 65521;
|
|
26
|
+
high = (high + low) % 65521;
|
|
27
|
+
}
|
|
28
|
+
return ((high << 16) | low) >>> 0;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const uint32BigEndian = (value: number): Uint8Array => Uint8Array.of((value >>> 24) & 0xff, (value >>> 16) & 0xff, (value >>> 8) & 0xff, value & 0xff);
|
|
32
|
+
|
|
33
|
+
const pngChunk = (type: string, data: Uint8Array): Uint8Array => {
|
|
34
|
+
const typeBytes = Uint8Array.from(type, (character) => character.charCodeAt(0));
|
|
35
|
+
const body = new Uint8Array(typeBytes.length + data.length);
|
|
36
|
+
body.set(typeBytes, 0);
|
|
37
|
+
body.set(data, typeBytes.length);
|
|
38
|
+
const chunk = new Uint8Array(4 + body.length + 4);
|
|
39
|
+
chunk.set(uint32BigEndian(data.length), 0);
|
|
40
|
+
chunk.set(body, 4);
|
|
41
|
+
chunk.set(uint32BigEndian(crc32(body)), 4 + body.length);
|
|
42
|
+
return chunk;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** Encode a tightly packed width×height RGB8 buffer to a PNG byte array. */
|
|
46
|
+
export const encodePNG = (rgbPixels: Uint8Array, width: number, height: number): Uint8Array => {
|
|
47
|
+
const scanlineLength = 1 + width * 3;
|
|
48
|
+
const filtered = new Uint8Array(height * scanlineLength);
|
|
49
|
+
for (let row = 0; row < height; row++) {
|
|
50
|
+
filtered[row * scanlineLength] = 0;
|
|
51
|
+
filtered.set(rgbPixels.subarray(row * width * 3, (row + 1) * width * 3), row * scanlineLength + 1);
|
|
52
|
+
}
|
|
53
|
+
const deflated = Bun.deflateSync(filtered);
|
|
54
|
+
const zlib = new Uint8Array(2 + deflated.length + 4);
|
|
55
|
+
zlib[0] = 0x78;
|
|
56
|
+
zlib[1] = 0x01;
|
|
57
|
+
zlib.set(deflated, 2);
|
|
58
|
+
zlib.set(uint32BigEndian(adler32(filtered)), 2 + deflated.length);
|
|
59
|
+
const headerData = new Uint8Array(13);
|
|
60
|
+
headerData.set(uint32BigEndian(width), 0);
|
|
61
|
+
headerData.set(uint32BigEndian(height), 4);
|
|
62
|
+
headerData[8] = 8;
|
|
63
|
+
headerData[9] = 2;
|
|
64
|
+
const signature = Uint8Array.of(137, 80, 78, 71, 13, 10, 26, 10);
|
|
65
|
+
const chunks = [signature, pngChunk('IHDR', headerData), pngChunk('IDAT', zlib), pngChunk('IEND', new Uint8Array(0))];
|
|
66
|
+
let totalLength = 0;
|
|
67
|
+
for (const chunk of chunks) totalLength += chunk.length;
|
|
68
|
+
const png = new Uint8Array(totalLength);
|
|
69
|
+
let offset = 0;
|
|
70
|
+
for (const chunk of chunks) {
|
|
71
|
+
png.set(chunk, offset);
|
|
72
|
+
offset += chunk.length;
|
|
73
|
+
}
|
|
74
|
+
return png;
|
|
75
|
+
};
|
package/reads.ts
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// Typed property readers built on the COM vtable invoker. Out-parameters use hoisted, reused scratch
|
|
2
|
+
// buffers (no per-read allocation); each value is read out immediately so the buffers never alias a
|
|
3
|
+
// live value, and `.ptr` is read inline at each call (small Buffers relocate — never cache it). BSTR
|
|
4
|
+
// names are bulk-copied in one operation BEFORE SysFreeString (never per-character, never after free).
|
|
5
|
+
|
|
6
|
+
import { FFIType, type Pointer, toArrayBuffer } from 'bun:ffi';
|
|
7
|
+
|
|
8
|
+
import Oleaut32 from '@bun-win32/oleaut32';
|
|
9
|
+
|
|
10
|
+
import { vcall } from './com';
|
|
11
|
+
import { S_OK } from './constants';
|
|
12
|
+
|
|
13
|
+
export interface Rect {
|
|
14
|
+
x: number;
|
|
15
|
+
y: number;
|
|
16
|
+
width: number;
|
|
17
|
+
height: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const scratch8 = Buffer.alloc(8);
|
|
21
|
+
const scratch4 = Buffer.alloc(4);
|
|
22
|
+
const scratch16 = Buffer.alloc(16);
|
|
23
|
+
|
|
24
|
+
/** Bulk-copy a BSTR's UTF-16 region into a string in one operation, then free it. No-op on null. */
|
|
25
|
+
export function decodeBstr(bstr: bigint): string {
|
|
26
|
+
if (bstr === 0n) return '';
|
|
27
|
+
const pointer = Number(bstr) as Pointer;
|
|
28
|
+
const length = Oleaut32.SysStringLen(pointer); // characters, not bytes
|
|
29
|
+
const text = length === 0 ? '' : Buffer.from(toArrayBuffer(pointer, 0, length * 2)).toString('utf16le');
|
|
30
|
+
Oleaut32.SysFreeString(pointer);
|
|
31
|
+
return text;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Read a `[propget] BSTR*` accessor, bulk-copying the UTF-16 region before freeing the BSTR. */
|
|
35
|
+
export function getBstr(ptr: bigint, slot: number): string {
|
|
36
|
+
if (vcall(ptr, slot, [FFIType.ptr], [scratch8.ptr!]) !== S_OK) return '';
|
|
37
|
+
return decodeBstr(scratch8.readBigUInt64LE(0));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Read a `[propget] LONG*` (or BOOL*) accessor. */
|
|
41
|
+
export function getLong(ptr: bigint, slot: number): number {
|
|
42
|
+
if (vcall(ptr, slot, [FFIType.ptr], [scratch4.ptr!]) !== S_OK) return 0;
|
|
43
|
+
return scratch4.readInt32LE(0);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Read a `[propget] double*` accessor (e.g. RangeValuePattern values). */
|
|
47
|
+
export function getDouble(ptr: bigint, slot: number): number {
|
|
48
|
+
if (vcall(ptr, slot, [FFIType.ptr], [scratch8.ptr!]) !== S_OK) return Number.NaN;
|
|
49
|
+
return scratch8.readDoubleLE(0);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Read a `[propget] UIA_HWND*` / handle accessor. */
|
|
53
|
+
export function getHandle(ptr: bigint, slot: number): bigint {
|
|
54
|
+
if (vcall(ptr, slot, [FFIType.ptr], [scratch8.ptr!]) !== S_OK) return 0n;
|
|
55
|
+
return scratch8.readBigUInt64LE(0);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Read a `[propget] RECT*` accessor (4× LONG) into an {x,y,width,height} rectangle. */
|
|
59
|
+
export function getRect(ptr: bigint, slot: number): Rect {
|
|
60
|
+
if (vcall(ptr, slot, [FFIType.ptr], [scratch16.ptr!]) !== S_OK) return { x: 0, y: 0, width: 0, height: 0 };
|
|
61
|
+
const left = scratch16.readInt32LE(0);
|
|
62
|
+
const top = scratch16.readInt32LE(4);
|
|
63
|
+
const right = scratch16.readInt32LE(8);
|
|
64
|
+
const bottom = scratch16.readInt32LE(12);
|
|
65
|
+
return { x: left, y: top, width: right - left, height: bottom - top };
|
|
66
|
+
}
|
package/tree.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// Agent grounding: serialize a window's accessibility subtree to compact JSON for an LLM — ground-truth
|
|
2
|
+
// element identity + bounds, no pixel-counting. One BuildUpdatedCache round-trip prefetches the whole
|
|
3
|
+
// subtree; the recursion then reads cached children/properties in-process. The agent profile prunes to
|
|
4
|
+
// interactive + named controls (fewer tokens) the way Microsoft UFO2 grounds desktop agents.
|
|
5
|
+
|
|
6
|
+
import { AutomationElementMode, createCacheRequest } from './cache';
|
|
7
|
+
import { ControlType, TreeScope } from './constants';
|
|
8
|
+
import type { Element } from './element';
|
|
9
|
+
import type { Rect } from './reads';
|
|
10
|
+
|
|
11
|
+
export interface UiaNode {
|
|
12
|
+
role: string;
|
|
13
|
+
name: string;
|
|
14
|
+
automationId?: string;
|
|
15
|
+
className?: string;
|
|
16
|
+
bounds?: Rect;
|
|
17
|
+
enabled?: boolean;
|
|
18
|
+
children: UiaNode[];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface SerializeOptions {
|
|
22
|
+
/** Maximum tree depth (default 40). */
|
|
23
|
+
maxDepth?: number;
|
|
24
|
+
/** Prune to interactive/named controls — the compact profile to hand an LLM agent. */
|
|
25
|
+
agentProfile?: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const INTERACTIVE = new Set<number>([
|
|
29
|
+
ControlType.Button,
|
|
30
|
+
ControlType.CheckBox,
|
|
31
|
+
ControlType.ComboBox,
|
|
32
|
+
ControlType.Document,
|
|
33
|
+
ControlType.Edit,
|
|
34
|
+
ControlType.Hyperlink,
|
|
35
|
+
ControlType.ListItem,
|
|
36
|
+
ControlType.MenuItem,
|
|
37
|
+
ControlType.RadioButton,
|
|
38
|
+
ControlType.Slider,
|
|
39
|
+
ControlType.Spinner,
|
|
40
|
+
ControlType.SplitButton,
|
|
41
|
+
ControlType.TabItem,
|
|
42
|
+
ControlType.TreeItem,
|
|
43
|
+
]);
|
|
44
|
+
|
|
45
|
+
function walk(element: Element, options: SerializeOptions, maxDepth: number, depth: number): UiaNode | null {
|
|
46
|
+
const controlType = element.cachedControlType;
|
|
47
|
+
const node: UiaNode = {
|
|
48
|
+
role: ControlType[controlType] ?? `Type(${controlType})`,
|
|
49
|
+
name: element.cachedName,
|
|
50
|
+
children: [],
|
|
51
|
+
};
|
|
52
|
+
const automationId = element.cachedAutomationId;
|
|
53
|
+
if (automationId.length > 0) node.automationId = automationId;
|
|
54
|
+
const className = element.cachedClassName;
|
|
55
|
+
if (className.length > 0) node.className = className;
|
|
56
|
+
const bounds = element.cachedBoundingRectangle;
|
|
57
|
+
if (bounds.width !== 0 || bounds.height !== 0) node.bounds = bounds;
|
|
58
|
+
node.enabled = element.cachedIsEnabled;
|
|
59
|
+
|
|
60
|
+
if (depth < maxDepth) {
|
|
61
|
+
for (const child of element.cachedChildren) {
|
|
62
|
+
const childNode = walk(child, options, maxDepth, depth + 1);
|
|
63
|
+
child.release();
|
|
64
|
+
if (childNode !== null) node.children.push(childNode);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (options.agentProfile && node.children.length === 0 && node.name.trim().length === 0 && !INTERACTIVE.has(controlType)) return null;
|
|
69
|
+
return node;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Serialize an element's subtree to a JSON-able tree. Build once via a single cached round-trip. */
|
|
73
|
+
export function serialize(element: Element, options: SerializeOptions = {}): UiaNode {
|
|
74
|
+
const maxDepth = options.maxDepth ?? 40;
|
|
75
|
+
const request = createCacheRequest(undefined, TreeScope.TreeScope_Subtree, AutomationElementMode.None);
|
|
76
|
+
const cached = element.buildUpdatedCache(request);
|
|
77
|
+
try {
|
|
78
|
+
return walk(cached, options, maxDepth, 0) ?? { role: 'Pane', name: '', children: [] };
|
|
79
|
+
} finally {
|
|
80
|
+
request.release();
|
|
81
|
+
if (cached.ptr !== element.ptr) cached.release();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Count the nodes in a serialized tree (for benchmarks / agent-grounding stats). */
|
|
86
|
+
export function countNodes(node: UiaNode): number {
|
|
87
|
+
let total = 1;
|
|
88
|
+
for (const child of node.children) total += countNodes(child);
|
|
89
|
+
return total;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Rough token estimate of the JSON form (~4 chars/token) — what an agent pays per grounding step. */
|
|
93
|
+
export function estimateTokens(node: UiaNode): number {
|
|
94
|
+
return Math.ceil(JSON.stringify(node).length / 4);
|
|
95
|
+
}
|