@bun-win32/uia 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/element.ts ADDED
@@ -0,0 +1,512 @@
1
+ // Element: a live IUIAutomationElement pointer with typed property reads, tree search, and the
2
+ // proven control-pattern actions. Property readers live in reads.ts; pattern actions in patterns.ts.
3
+
4
+ import { FFIType } from 'bun:ffi';
5
+
6
+ import User32 from '@bun-win32/user32';
7
+
8
+ import { automation } from './automation';
9
+ import type { CacheRequest } from './cache';
10
+ import { comRelease, hresult, vcall } from './com';
11
+ import { compileCondition, type ElementProperties, formatNoMatch, matches, type Selector } from './condition';
12
+ import { ControlType, S_OK, SLOT, TreeScope } from './constants';
13
+ import { clickAt, type as inputType } from './input';
14
+ import { screenshot as windowScreenshot, windowForProcess } from './window';
15
+ import {
16
+ collapse,
17
+ expand,
18
+ expandCollapseState,
19
+ getValue,
20
+ invoke,
21
+ isSelected,
22
+ rangeValue,
23
+ readText,
24
+ scrollIntoView,
25
+ select,
26
+ setRangeValue,
27
+ setValue,
28
+ setWindowVisualState,
29
+ toggle,
30
+ toggleState,
31
+ windowClose,
32
+ type WindowVisualState,
33
+ } from './patterns';
34
+ import { getBstr, getHandle, getLong, getRect, type Rect } from './reads';
35
+
36
+ // Reused scratch for out-parameters in the tree-search path. Each value is read out immediately.
37
+ const scratch8 = Buffer.alloc(8);
38
+ const scratch4 = Buffer.alloc(4);
39
+
40
+ /** Read the four properties the client-side matcher needs, in one pass (live). */
41
+ function readProperties(ptr: bigint): ElementProperties {
42
+ return {
43
+ automationId: getBstr(ptr, SLOT.get_CurrentAutomationId),
44
+ className: getBstr(ptr, SLOT.get_CurrentClassName),
45
+ controlType: getLong(ptr, SLOT.get_CurrentControlType),
46
+ name: getBstr(ptr, SLOT.get_CurrentName),
47
+ };
48
+ }
49
+
50
+ /** Read the matcher's four properties from the prefetched cache (zero further round-trips). */
51
+ function readCachedProperties(ptr: bigint): ElementProperties {
52
+ return {
53
+ automationId: getBstr(ptr, SLOT.get_CachedAutomationId),
54
+ className: getBstr(ptr, SLOT.get_CachedClassName),
55
+ controlType: getLong(ptr, SLOT.get_CachedControlType),
56
+ name: getBstr(ptr, SLOT.get_CachedName),
57
+ };
58
+ }
59
+
60
+ function findFirstPointer(scopeElement: bigint, scope: number, condition: bigint): bigint {
61
+ if (vcall(scopeElement, SLOT.FindFirst, [FFIType.i32, FFIType.u64, FFIType.ptr], [scope, condition, scratch8.ptr!]) !== S_OK) return 0n;
62
+ return scratch8.readBigUInt64LE(0);
63
+ }
64
+
65
+ function findAllPointers(scopeElement: bigint, scope: number, condition: bigint): bigint[] {
66
+ if (vcall(scopeElement, SLOT.FindAll, [FFIType.i32, FFIType.u64, FFIType.ptr], [scope, condition, scratch8.ptr!]) !== S_OK) return [];
67
+ const pArray = scratch8.readBigUInt64LE(0);
68
+ if (pArray === 0n) return [];
69
+ try {
70
+ if (vcall(pArray, SLOT.get_Length, [FFIType.ptr], [scratch4.ptr!]) !== S_OK) return [];
71
+ const length = scratch4.readInt32LE(0);
72
+ const pointers: bigint[] = new Array(length);
73
+ let count = 0;
74
+ for (let index = 0; index < length; index += 1) {
75
+ if (vcall(pArray, SLOT.GetElement, [FFIType.i32, FFIType.ptr], [index, scratch8.ptr!]) !== S_OK) continue;
76
+ const pointer = scratch8.readBigUInt64LE(0);
77
+ if (pointer !== 0n) {
78
+ pointers[count] = pointer;
79
+ count += 1;
80
+ }
81
+ }
82
+ pointers.length = count;
83
+ return pointers;
84
+ } finally {
85
+ comRelease(pArray);
86
+ }
87
+ }
88
+
89
+ export class Element {
90
+ readonly ptr: bigint;
91
+
92
+ constructor(ptr: bigint) {
93
+ this.ptr = ptr;
94
+ }
95
+
96
+ get automationId(): string {
97
+ return getBstr(this.ptr, SLOT.get_CurrentAutomationId);
98
+ }
99
+
100
+ get boundingRectangle(): Rect {
101
+ return getRect(this.ptr, SLOT.get_CurrentBoundingRectangle);
102
+ }
103
+
104
+ get className(): string {
105
+ return getBstr(this.ptr, SLOT.get_CurrentClassName);
106
+ }
107
+
108
+ get controlType(): number {
109
+ return getLong(this.ptr, SLOT.get_CurrentControlType);
110
+ }
111
+
112
+ get controlTypeName(): string {
113
+ const id = this.controlType;
114
+ return ControlType[id] ?? `Type(${id})`;
115
+ }
116
+
117
+ get isEnabled(): boolean {
118
+ return getLong(this.ptr, SLOT.get_CurrentIsEnabled) !== 0;
119
+ }
120
+
121
+ get name(): string {
122
+ return getBstr(this.ptr, SLOT.get_CurrentName);
123
+ }
124
+
125
+ get nativeWindowHandle(): bigint {
126
+ return getHandle(this.ptr, SLOT.get_CurrentNativeWindowHandle);
127
+ }
128
+
129
+ /** Immediate children (control view) as Elements. The caller owns and should release them. */
130
+ get children(): Element[] {
131
+ return this.findAll({}, TreeScope.TreeScope_Children);
132
+ }
133
+
134
+ /** The control-view parent, or null at a root. The caller owns the returned Element. */
135
+ get parent(): Element | null {
136
+ if (vcall(automation(), SLOT.get_ControlViewWalker, [FFIType.ptr], [scratch8.ptr!]) !== S_OK) return null;
137
+ const walker = scratch8.readBigUInt64LE(0);
138
+ if (walker === 0n) return null;
139
+ try {
140
+ if (vcall(walker, SLOT.GetParentElement, [FFIType.u64, FFIType.ptr], [this.ptr, scratch8.ptr!]) !== S_OK) return null;
141
+ const pointer = scratch8.readBigUInt64LE(0);
142
+ return pointer === 0n ? null : new Element(pointer);
143
+ } finally {
144
+ comRelease(walker);
145
+ }
146
+ }
147
+
148
+ /** The first descendant (by default) matching the selector, or null. Releases the non-matches. */
149
+ find(selector: Selector, scope: number = TreeScope.TreeScope_Descendants): Element | null {
150
+ const pAutomation = automation();
151
+ const { condition, needsClientFilter } = compileCondition(pAutomation, selector);
152
+ try {
153
+ if (!needsClientFilter) {
154
+ const pointer = findFirstPointer(this.ptr, scope, condition);
155
+ return pointer === 0n ? null : new Element(pointer);
156
+ }
157
+ const pointers = findAllPointers(this.ptr, scope, condition);
158
+ for (let index = 0; index < pointers.length; index += 1) {
159
+ const pointer = pointers[index]!;
160
+ if (matches(readProperties(pointer), selector)) {
161
+ for (let rest = index + 1; rest < pointers.length; rest += 1) comRelease(pointers[rest]!);
162
+ return new Element(pointer);
163
+ }
164
+ comRelease(pointer);
165
+ }
166
+ return null;
167
+ } finally {
168
+ comRelease(condition);
169
+ }
170
+ }
171
+
172
+ /**
173
+ * Poll `find` until the selector matches or `timeout` (ms) elapses — the killer feature for flaky
174
+ * native UIs. Paced by an async sleep (never busy-spins). On timeout, throws an error quoting the
175
+ * selector, this element's name, and the nearest candidates.
176
+ */
177
+ async waitFor(selector: Selector, options: { timeout?: number; interval?: number } = {}): Promise<Element> {
178
+ const timeout = options.timeout ?? 5000;
179
+ const interval = options.interval ?? 100;
180
+ const start = Bun.nanoseconds();
181
+ for (;;) {
182
+ const found = this.find(selector);
183
+ if (found !== null) return found;
184
+ if ((Bun.nanoseconds() - start) / 1e6 >= timeout) throw new Error(this.describeNoMatch(selector));
185
+ await Bun.sleep(interval);
186
+ }
187
+ }
188
+
189
+ /** Build the actionable no-match message by scanning the candidate set under this element. */
190
+ describeNoMatch(selector: Selector): string {
191
+ const candidates = this.findAll(selector.controlType !== undefined ? { controlType: selector.controlType } : {});
192
+ const names = candidates.map((candidate) => candidate.name);
193
+ for (const candidate of candidates) candidate.release();
194
+ return formatNoMatch(selector, this.name, names);
195
+ }
196
+
197
+ /** Every descendant (by default) matching the selector. The caller owns and should release them. */
198
+ findAll(selector: Selector, scope: number = TreeScope.TreeScope_Descendants): Element[] {
199
+ const pAutomation = automation();
200
+ const { condition, needsClientFilter } = compileCondition(pAutomation, selector);
201
+ try {
202
+ const pointers = findAllPointers(this.ptr, scope, condition);
203
+ if (!needsClientFilter) return pointers.map((pointer) => new Element(pointer));
204
+ const result: Element[] = [];
205
+ for (const pointer of pointers) {
206
+ if (matches(readProperties(pointer), selector)) result.push(new Element(pointer));
207
+ else comRelease(pointer);
208
+ }
209
+ return result;
210
+ } finally {
211
+ comRelease(condition);
212
+ }
213
+ }
214
+
215
+ /**
216
+ * Every descendant matching the selector, prefetched through one cached round-trip. The returned
217
+ * Elements expose their cached* properties with zero further round-trips. The caller owns them.
218
+ */
219
+ findAllCached(selector: Selector, request: CacheRequest, scope: number = TreeScope.TreeScope_Descendants): Element[] {
220
+ const pAutomation = automation();
221
+ const { condition, needsClientFilter } = compileCondition(pAutomation, selector);
222
+ try {
223
+ if (vcall(this.ptr, SLOT.FindAllBuildCache, [FFIType.i32, FFIType.u64, FFIType.u64, FFIType.ptr], [scope, condition, request.ptr, scratch8.ptr!]) !== S_OK) return [];
224
+ const pArray = scratch8.readBigUInt64LE(0);
225
+ if (pArray === 0n) return [];
226
+ try {
227
+ if (vcall(pArray, SLOT.get_Length, [FFIType.ptr], [scratch4.ptr!]) !== S_OK) return [];
228
+ const length = scratch4.readInt32LE(0);
229
+ const result: Element[] = [];
230
+ for (let index = 0; index < length; index += 1) {
231
+ if (vcall(pArray, SLOT.GetElement, [FFIType.i32, FFIType.ptr], [index, scratch8.ptr!]) !== S_OK) continue;
232
+ const pointer = scratch8.readBigUInt64LE(0);
233
+ if (pointer === 0n) continue;
234
+ if (!needsClientFilter || matches(readCachedProperties(pointer), selector)) result.push(new Element(pointer));
235
+ else comRelease(pointer);
236
+ }
237
+ return result;
238
+ } finally {
239
+ comRelease(pArray);
240
+ }
241
+ } finally {
242
+ comRelease(condition);
243
+ }
244
+ }
245
+
246
+ /** Refresh this element's cache (properties + structure) per the request. Returns the cached element. */
247
+ buildUpdatedCache(request: CacheRequest): Element {
248
+ if (vcall(this.ptr, SLOT.BuildUpdatedCache, [FFIType.u64, FFIType.ptr], [request.ptr, scratch8.ptr!]) !== S_OK) return this;
249
+ const pointer = scratch8.readBigUInt64LE(0);
250
+ return pointer === 0n ? this : new Element(pointer);
251
+ }
252
+
253
+ // --- cached property reads (valid only on elements returned by findAllCached / buildUpdatedCache) ---
254
+
255
+ /** Cached immediate children (in-proc; valid after a Subtree-scoped buildUpdatedCache). */
256
+ get cachedChildren(): Element[] {
257
+ if (vcall(this.ptr, SLOT.GetCachedChildren, [FFIType.ptr], [scratch8.ptr!]) !== S_OK) return [];
258
+ const pArray = scratch8.readBigUInt64LE(0);
259
+ if (pArray === 0n) return [];
260
+ try {
261
+ if (vcall(pArray, SLOT.get_Length, [FFIType.ptr], [scratch4.ptr!]) !== S_OK) return [];
262
+ const length = scratch4.readInt32LE(0);
263
+ const children: Element[] = new Array(length);
264
+ let count = 0;
265
+ for (let index = 0; index < length; index += 1) {
266
+ if (vcall(pArray, SLOT.GetElement, [FFIType.i32, FFIType.ptr], [index, scratch8.ptr!]) !== S_OK) continue;
267
+ const pointer = scratch8.readBigUInt64LE(0);
268
+ if (pointer !== 0n) {
269
+ children[count] = new Element(pointer);
270
+ count += 1;
271
+ }
272
+ }
273
+ children.length = count;
274
+ return children;
275
+ } finally {
276
+ comRelease(pArray);
277
+ }
278
+ }
279
+
280
+ get cachedAutomationId(): string {
281
+ return getBstr(this.ptr, SLOT.get_CachedAutomationId);
282
+ }
283
+
284
+ get cachedBoundingRectangle(): Rect {
285
+ return getRect(this.ptr, SLOT.get_CachedBoundingRectangle);
286
+ }
287
+
288
+ get cachedClassName(): string {
289
+ return getBstr(this.ptr, SLOT.get_CachedClassName);
290
+ }
291
+
292
+ get cachedControlType(): number {
293
+ return getLong(this.ptr, SLOT.get_CachedControlType);
294
+ }
295
+
296
+ get cachedIsEnabled(): boolean {
297
+ return getLong(this.ptr, SLOT.get_CachedIsEnabled) !== 0;
298
+ }
299
+
300
+ get cachedName(): string {
301
+ return getBstr(this.ptr, SLOT.get_CachedName);
302
+ }
303
+
304
+ /** Release the underlying COM pointer. */
305
+ release(): void {
306
+ comRelease(this.ptr);
307
+ }
308
+
309
+ // --- control-pattern actions (each proven against a real control in Phase 5) ---
310
+
311
+ /** Press via InvokePattern. Throws if unsupported (try `.click()`). */
312
+ invoke(): void {
313
+ invoke(this.ptr);
314
+ }
315
+
316
+ /** Read a ValuePattern value (e.g. a text box), or '' if unsupported. */
317
+ get value(): string {
318
+ return getValue(this.ptr);
319
+ }
320
+
321
+ /** Set a ValuePattern value in one call — no keystrokes. Throws if unsupported (try `.type()`). */
322
+ setValue(text: string): void {
323
+ setValue(this.ptr, text);
324
+ }
325
+
326
+ /** Read the TextPattern document text, or '' if unsupported. */
327
+ text(): string {
328
+ return readText(this.ptr);
329
+ }
330
+
331
+ /** Toggle a checkbox via TogglePattern. Throws if unsupported. */
332
+ toggle(): void {
333
+ toggle(this.ptr);
334
+ }
335
+
336
+ /** TogglePattern state (0 Off, 1 On, 2 Indeterminate), or -1 if unsupported. */
337
+ get toggleState(): number {
338
+ return toggleState(this.ptr);
339
+ }
340
+
341
+ /** Expand via ExpandCollapsePattern. Throws if unsupported. */
342
+ expand(): void {
343
+ expand(this.ptr);
344
+ }
345
+
346
+ /** Collapse via ExpandCollapsePattern. Throws if unsupported. */
347
+ collapse(): void {
348
+ collapse(this.ptr);
349
+ }
350
+
351
+ /** ExpandCollapsePattern state (0 Collapsed, 1 Expanded, 2 Partial, 3 Leaf), or -1 if unsupported. */
352
+ get expandCollapseState(): number {
353
+ return expandCollapseState(this.ptr);
354
+ }
355
+
356
+ /** Select via SelectionItemPattern, replacing the selection. Throws if unsupported. */
357
+ select(): void {
358
+ select(this.ptr);
359
+ }
360
+
361
+ /** Whether selected (SelectionItemPattern); false if unsupported. */
362
+ get isSelected(): boolean {
363
+ return isSelected(this.ptr);
364
+ }
365
+
366
+ /** Scroll into view via ScrollItemPattern. Throws if unsupported. */
367
+ scrollIntoView(): void {
368
+ scrollIntoView(this.ptr);
369
+ }
370
+
371
+ /** RangeValuePattern value (slider), or NaN if unsupported. */
372
+ get rangeValue(): number {
373
+ return rangeValue(this.ptr);
374
+ }
375
+
376
+ /** Set a RangeValuePattern value (slider). Throws if unsupported. */
377
+ setRangeValue(value: number): void {
378
+ setRangeValue(this.ptr, value);
379
+ }
380
+
381
+ /** Close a window via WindowPattern. Throws if unsupported. */
382
+ close(): void {
383
+ windowClose(this.ptr);
384
+ }
385
+
386
+ /** Set a window's visual state (WindowVisualState) via WindowPattern. Throws if unsupported. */
387
+ setVisualState(state: WindowVisualState): void {
388
+ setWindowVisualState(this.ptr, state);
389
+ }
390
+
391
+ // --- synthetic input fallbacks (SendInput) for controls without a usable pattern ---
392
+
393
+ /** Give the element keyboard focus (UIA SetFocus). Returns this for chaining. */
394
+ focus(): this {
395
+ vcall(this.ptr, SLOT.SetFocus, [], []);
396
+ return this;
397
+ }
398
+
399
+ /** Focus the element, then type Unicode text into it via SendInput. Returns this for chaining. */
400
+ type(text: string): this {
401
+ this.focus();
402
+ inputType(text);
403
+ return this;
404
+ }
405
+
406
+ /** Click the element's bounding-rectangle center via SendInput (the no-InvokePattern fallback). */
407
+ click(): this {
408
+ const hWnd = this.nativeWindowHandle;
409
+ if (hWnd !== 0n) User32.SetForegroundWindow(hWnd);
410
+ const rect = this.boundingRectangle;
411
+ clickAt(rect.x + Math.floor(rect.width / 2), rect.y + Math.floor(rect.height / 2));
412
+ return this;
413
+ }
414
+ }
415
+
416
+ /** Attach an Element to a window handle (ElementFromHandle, slot 6 — NativeWindowHandle round-trips). */
417
+ export function fromHandle(hWnd: bigint): Element {
418
+ const hr = vcall(automation(), SLOT.ElementFromHandle, [FFIType.u64, FFIType.ptr], [hWnd, scratch8.ptr!]);
419
+ const pointer = scratch8.readBigUInt64LE(0);
420
+ if (hr !== S_OK || pointer === 0n) throw new Error(`ElementFromHandle(0x${hWnd.toString(16)}) failed: ${hresult(hr)}`);
421
+ return new Element(pointer);
422
+ }
423
+
424
+ /** The element with keyboard focus. */
425
+ export function focused(): Element {
426
+ const hr = vcall(automation(), SLOT.GetFocusedElement, [FFIType.ptr], [scratch8.ptr!]);
427
+ const pointer = scratch8.readBigUInt64LE(0);
428
+ if (hr !== S_OK || pointer === 0n) throw new Error(`GetFocusedElement failed: ${hresult(hr)}`);
429
+ return new Element(pointer);
430
+ }
431
+
432
+ /** The element at a screen point (POINT packed by value: x in the low dword, y in the high dword). */
433
+ export function fromPoint(x: number, y: number): Element {
434
+ const point = (BigInt(y >>> 0) << 32n) | BigInt(x >>> 0);
435
+ const hr = vcall(automation(), SLOT.ElementFromPoint, [FFIType.u64, FFIType.ptr], [point, scratch8.ptr!]);
436
+ const pointer = scratch8.readBigUInt64LE(0);
437
+ if (hr !== S_OK || pointer === 0n) throw new Error(`ElementFromPoint(${x},${y}) failed: ${hresult(hr)}`);
438
+ return new Element(pointer);
439
+ }
440
+
441
+ /** The desktop root element. Never FindAll(Descendants) from here — scope to a window. */
442
+ export function root(): Element {
443
+ const hr = vcall(automation(), SLOT.GetRootElement, [FFIType.ptr], [scratch8.ptr!]);
444
+ const pointer = scratch8.readBigUInt64LE(0);
445
+ if (hr !== S_OK || pointer === 0n) throw new Error(`GetRootElement failed: ${hresult(hr)}`);
446
+ return new Element(pointer);
447
+ }
448
+
449
+ /** A top-level window — an Element scoped to a window handle, with `using`-friendly disposal. */
450
+ export class Window extends Element {
451
+ readonly hWnd: bigint;
452
+
453
+ constructor(ptr: bigint, hWnd: bigint) {
454
+ super(ptr);
455
+ this.hWnd = hWnd;
456
+ }
457
+
458
+ /** Bring the window to the foreground (best-effort; blocked on a locked session). */
459
+ activate(): this {
460
+ User32.SetForegroundWindow(this.hWnd);
461
+ return this;
462
+ }
463
+
464
+ /** Capture the window via PrintWindow as PNG bytes (blank on a locked session). */
465
+ screenshot(): Uint8Array {
466
+ return windowScreenshot(this.hWnd);
467
+ }
468
+
469
+ /** Release the window element. Enables `using app = uia.attach(...)`. */
470
+ dispose(): void {
471
+ this.release();
472
+ }
473
+
474
+ [Symbol.dispose](): void {
475
+ this.dispose();
476
+ }
477
+ }
478
+
479
+ function resolveWindow(target: string | bigint | { className?: string; process?: number; title?: string }): bigint {
480
+ if (typeof target === 'bigint') return target;
481
+ if (typeof target !== 'string' && target.process !== undefined) return windowForProcess(target.process);
482
+ const title = typeof target === 'string' ? target : target.title;
483
+ const className = typeof target === 'string' ? undefined : target.className;
484
+ const classBuffer = className === undefined ? null : Buffer.from(`${className}\0`, 'utf16le').ptr!;
485
+ const titleBuffer = title === undefined ? null : Buffer.from(`${title}\0`, 'utf16le').ptr!;
486
+ return User32.FindWindowW(classBuffer, titleBuffer);
487
+ }
488
+
489
+ /** Attach to a top-level window by title, handle, `{ title, className }`, or `{ process }`. Throws if absent. */
490
+ export function attach(target: string | bigint | { className?: string; process?: number; title?: string }): Window {
491
+ const hWnd = resolveWindow(target);
492
+ if (hWnd === 0n) throw new Error(`attach: no window found for ${JSON.stringify(target, (_key, value) => (typeof value === 'bigint' ? `0x${value.toString(16)}` : value))}`);
493
+ const hr = vcall(automation(), SLOT.ElementFromHandle, [FFIType.u64, FFIType.ptr], [hWnd, scratch8.ptr!]);
494
+ const pointer = scratch8.readBigUInt64LE(0);
495
+ if (hr !== S_OK || pointer === 0n) throw new Error(`attach: ElementFromHandle failed: ${hresult(hr)}`);
496
+ return new Window(pointer, hWnd);
497
+ }
498
+
499
+ /** Spawn a process and wait for its window to appear, then attach — no hand-rolled FindWindow loop. */
500
+ export async function launch(command: string | readonly string[], target: { className?: string; title?: string }, timeout = 8000): Promise<Window> {
501
+ Bun.spawn(typeof command === 'string' ? command.split(' ') : [...command], { stdout: 'ignore', stderr: 'ignore' });
502
+ const start = Bun.nanoseconds();
503
+ for (;;) {
504
+ const hWnd = resolveWindow(target);
505
+ if (hWnd !== 0n) {
506
+ Bun.sleepSync(400); // let the content realize
507
+ return attach(hWnd);
508
+ }
509
+ if ((Bun.nanoseconds() - start) / 1e6 >= timeout) throw new Error(`launch: window ${JSON.stringify(target)} did not appear within ${timeout}ms`);
510
+ await Bun.sleep(150);
511
+ }
512
+ }
package/index.ts ADDED
@@ -0,0 +1,40 @@
1
+ import { execute } from './agent';
2
+ import { initialize, uninitialize } from './automation';
3
+ import { attach, focused, fromPoint, launch, root } from './element';
4
+ import { clickAt, sendKeys, type } from './input';
5
+ import { msaaTree } from './msaa';
6
+ import { serialize } from './tree';
7
+ import { listWindows } from './window';
8
+
9
+ /** The Playwright-for-desktop facade: attach to a window, then find/waitFor/act/serialize. */
10
+ export const uia = {
11
+ attach,
12
+ click: clickAt,
13
+ execute,
14
+ focused,
15
+ fromPoint,
16
+ initialize,
17
+ launch,
18
+ msaaTree,
19
+ root,
20
+ sendKeys,
21
+ tree: serialize,
22
+ type,
23
+ uninitialize,
24
+ windows: listWindows,
25
+ };
26
+
27
+ export { type AgentAction, type AgentActionResult, AGENT_TOOLS, execute, groundingTree } from './agent';
28
+ export { automation, initialize, uninitialize } from './automation';
29
+ export { AutomationElementMode, CacheRequest, createCacheRequest, DEFAULT_CACHE_PROPERTIES } from './cache';
30
+ export { comRelease, guid, hresult, vcall } from './com';
31
+ export { type ElementProperties, formatNoMatch, matches, selectorToString, type Selector } from './condition';
32
+ export { ControlType, PatternId, PropertyConditionFlags, PropertyId, SLOT, TreeScope } from './constants';
33
+ export { attach, Element, focused, fromHandle, fromPoint, launch, root, Window } from './element';
34
+ export { clickAt, INPUT_SIZE, packKeyboardInput, packMouseInput, sendKeys, type, virtualKeyCode } from './input';
35
+ export { accessibleFromWindow, type MsaaNode, msaaTree } from './msaa';
36
+ export { ExpandCollapseState, ToggleState, WindowVisualState } from './patterns';
37
+ export { encodePNG } from './png';
38
+ export { decodeBstr, getBstr, getHandle, getLong, getRect, type Rect } from './reads';
39
+ export { countNodes, estimateTokens, serialize, type SerializeOptions, type UiaNode } from './tree';
40
+ export { findWindow, listWindows, screenshot, type WindowInfo, windowForProcess } from './window';
package/input.ts ADDED
@@ -0,0 +1,149 @@
1
+ // Synthetic input via SendInput, for the cases UIA patterns can't cover: typing into a control with
2
+ // no ValuePattern, and clicking a control with no InvokePattern. Greenfield — the x64 INPUT struct is
3
+ // 40 bytes (DWORD type @0, 4 pad, union @8: MOUSEINPUT 32B / KEYBDINPUT 24B with ULONG_PTR fields
4
+ // 8-byte aligned). cbSize MUST be 40 or SendInput silently injects nothing.
5
+
6
+ import User32 from '@bun-win32/user32';
7
+
8
+ export const INPUT_MOUSE = 0x0000_0000;
9
+ export const INPUT_KEYBOARD = 0x0000_0001;
10
+
11
+ export const KEYEVENTF_EXTENDEDKEY = 0x0000_0001;
12
+ export const KEYEVENTF_KEYUP = 0x0000_0002;
13
+ export const KEYEVENTF_UNICODE = 0x0000_0004;
14
+ export const KEYEVENTF_SCANCODE = 0x0000_0008;
15
+
16
+ export const MOUSEEVENTF_MOVE = 0x0000_0001;
17
+ export const MOUSEEVENTF_LEFTDOWN = 0x0000_0002;
18
+ export const MOUSEEVENTF_LEFTUP = 0x0000_0004;
19
+ export const MOUSEEVENTF_RIGHTDOWN = 0x0000_0008;
20
+ export const MOUSEEVENTF_RIGHTUP = 0x0000_0010;
21
+ export const MOUSEEVENTF_ABSOLUTE = 0x0000_8000;
22
+ export const MOUSEEVENTF_VIRTUALDESK = 0x0000_4000;
23
+
24
+ /** sizeof(INPUT) on x64 — pass this as cbSize, or SendInput injects nothing. */
25
+ export const INPUT_SIZE = 40;
26
+
27
+ const NAMED_KEYS: Record<string, number> = {
28
+ alt: 0x12,
29
+ backspace: 0x08,
30
+ control: 0x11,
31
+ ctrl: 0x11,
32
+ del: 0x2e,
33
+ delete: 0x2e,
34
+ down: 0x28,
35
+ end: 0x23,
36
+ enter: 0x0d,
37
+ esc: 0x1b,
38
+ escape: 0x1b,
39
+ f1: 0x70,
40
+ f10: 0x79,
41
+ f11: 0x7a,
42
+ f12: 0x7b,
43
+ f2: 0x71,
44
+ f3: 0x72,
45
+ f4: 0x73,
46
+ f5: 0x74,
47
+ f6: 0x75,
48
+ f7: 0x76,
49
+ f8: 0x77,
50
+ f9: 0x78,
51
+ home: 0x24,
52
+ insert: 0x2d,
53
+ left: 0x25,
54
+ menu: 0x12,
55
+ meta: 0x5b,
56
+ pagedown: 0x22,
57
+ pageup: 0x21,
58
+ return: 0x0d,
59
+ right: 0x27,
60
+ shift: 0x10,
61
+ space: 0x20,
62
+ tab: 0x09,
63
+ up: 0x26,
64
+ win: 0x5b,
65
+ };
66
+
67
+ // Keys that require KEYEVENTF_EXTENDEDKEY to behave correctly across apps.
68
+ const EXTENDED_KEYS = new Set([0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x2d, 0x2e]);
69
+
70
+ /** Map a key name ('Enter', 'Control', 'A', '5', 'F4') to its virtual-key code. Throws if unknown. */
71
+ export function virtualKeyCode(name: string): number {
72
+ const lower = name.toLowerCase();
73
+ if (lower in NAMED_KEYS) return NAMED_KEYS[lower]!;
74
+ if (name.length === 1) {
75
+ const code = name.toUpperCase().charCodeAt(0);
76
+ if ((code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39)) return code;
77
+ }
78
+ throw new Error(`unknown key: ${JSON.stringify(name)}`);
79
+ }
80
+
81
+ /** Pack a KEYBDINPUT-bearing INPUT at `offset` (the buffer must be zero-filled first). */
82
+ export function packKeyboardInput(buffer: Buffer, offset: number, virtualKey: number, scanCode: number, flags: number): void {
83
+ buffer.writeUInt32LE(INPUT_KEYBOARD, offset); // type @0
84
+ buffer.writeUInt16LE(virtualKey, offset + 8); // KEYBDINPUT.wVk @8
85
+ buffer.writeUInt16LE(scanCode, offset + 10); // wScan @10
86
+ buffer.writeUInt32LE(flags, offset + 12); // dwFlags @12
87
+ buffer.writeUInt32LE(0, offset + 16); // time @16
88
+ buffer.writeBigUInt64LE(0n, offset + 24); // dwExtraInfo @24 (ULONG_PTR, 8-byte aligned)
89
+ }
90
+
91
+ /** Pack a MOUSEINPUT-bearing INPUT at `offset` (the buffer must be zero-filled first). */
92
+ export function packMouseInput(buffer: Buffer, offset: number, dx: number, dy: number, mouseData: number, flags: number): void {
93
+ buffer.writeUInt32LE(INPUT_MOUSE, offset); // type @0
94
+ buffer.writeInt32LE(dx, offset + 8); // MOUSEINPUT.dx @8
95
+ buffer.writeInt32LE(dy, offset + 12); // dy @12
96
+ buffer.writeUInt32LE(mouseData, offset + 16); // mouseData @16
97
+ buffer.writeUInt32LE(flags, offset + 20); // dwFlags @20
98
+ buffer.writeUInt32LE(0, offset + 24); // time @24
99
+ buffer.writeBigUInt64LE(0n, offset + 32); // dwExtraInfo @32
100
+ }
101
+
102
+ /** Type Unicode text into the focused control, code unit by code unit (locale/layout-independent). */
103
+ export function type(text: string): void {
104
+ if (text.length === 0) return;
105
+ const count = text.length * 2; // down + up per UTF-16 code unit
106
+ const buffer = Buffer.alloc(count * INPUT_SIZE);
107
+ let offset = 0;
108
+ for (let index = 0; index < text.length; index += 1) {
109
+ const code = text.charCodeAt(index);
110
+ packKeyboardInput(buffer, offset, 0, code, KEYEVENTF_UNICODE);
111
+ offset += INPUT_SIZE;
112
+ packKeyboardInput(buffer, offset, 0, code, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP);
113
+ offset += INPUT_SIZE;
114
+ }
115
+ User32.SendInput(count, buffer.ptr!, INPUT_SIZE);
116
+ }
117
+
118
+ /** Send a key chord like 'Enter', 'Control+S', 'Control+Shift+Tab' to the focused control. */
119
+ export function sendKeys(combo: string): void {
120
+ const parts = combo.split('+').map((part) => part.trim());
121
+ const key = parts[parts.length - 1]!;
122
+ const modifiers = parts.slice(0, -1).map(virtualKeyCode);
123
+ const keyCode = virtualKeyCode(key);
124
+ const sequence: Array<{ virtualKey: number; flags: number }> = [];
125
+ for (const modifier of modifiers) sequence.push({ virtualKey: modifier, flags: 0 });
126
+ sequence.push({ virtualKey: keyCode, flags: EXTENDED_KEYS.has(keyCode) ? KEYEVENTF_EXTENDEDKEY : 0 });
127
+ sequence.push({ virtualKey: keyCode, flags: KEYEVENTF_KEYUP | (EXTENDED_KEYS.has(keyCode) ? KEYEVENTF_EXTENDEDKEY : 0) });
128
+ for (let index = modifiers.length - 1; index >= 0; index -= 1) sequence.push({ virtualKey: modifiers[index]!, flags: KEYEVENTF_KEYUP });
129
+ const buffer = Buffer.alloc(sequence.length * INPUT_SIZE);
130
+ let offset = 0;
131
+ for (const event of sequence) {
132
+ packKeyboardInput(buffer, offset, event.virtualKey, 0, event.flags);
133
+ offset += INPUT_SIZE;
134
+ }
135
+ User32.SendInput(sequence.length, buffer.ptr!, INPUT_SIZE);
136
+ }
137
+
138
+ /**
139
+ * Move the cursor to a screen point (physical pixels — the process is DPI-aware) and left-click.
140
+ * Uses SetCursorPos for exact placement, then SendInput button events. Requires an unlocked,
141
+ * interactive desktop (synthetic input is blocked on a locked session).
142
+ */
143
+ export function clickAt(x: number, y: number): void {
144
+ User32.SetCursorPos(x, y);
145
+ const buffer = Buffer.alloc(2 * INPUT_SIZE);
146
+ packMouseInput(buffer, 0, 0, 0, 0, MOUSEEVENTF_LEFTDOWN);
147
+ packMouseInput(buffer, INPUT_SIZE, 0, 0, 0, MOUSEEVENTF_LEFTUP);
148
+ User32.SendInput(2, buffer.ptr!, INPUT_SIZE);
149
+ }