@epsilon-asi/actors 0.0.21 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/dist/browser/RuntimeConfig.d.ts +26 -0
  2. package/dist/browser/RuntimeConfig.d.ts.map +1 -1
  3. package/dist/browser/RuntimeConfig.js +29 -1
  4. package/dist/browser/RuntimeConfig.js.map +1 -1
  5. package/dist/core/ActorContext.d.ts +2 -0
  6. package/dist/core/ActorContext.d.ts.map +1 -1
  7. package/dist/core/ActorRunner.d.ts +3 -0
  8. package/dist/core/ActorRunner.d.ts.map +1 -1
  9. package/dist/core/ActorRunner.js +11 -1
  10. package/dist/core/ActorRunner.js.map +1 -1
  11. package/dist/index.d.ts +2 -0
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +2 -0
  14. package/dist/index.js.map +1 -1
  15. package/dist/native/CompositeNativeWindowDriver.d.ts +11 -0
  16. package/dist/native/CompositeNativeWindowDriver.d.ts.map +1 -0
  17. package/dist/native/CompositeNativeWindowDriver.js +31 -0
  18. package/dist/native/CompositeNativeWindowDriver.js.map +1 -0
  19. package/dist/native/NativeActionRegistry.d.ts +14 -0
  20. package/dist/native/NativeActionRegistry.d.ts.map +1 -0
  21. package/dist/native/NativeActionRegistry.js +101 -0
  22. package/dist/native/NativeActionRegistry.js.map +1 -0
  23. package/dist/native/NativeAutomation.d.ts +3 -0
  24. package/dist/native/NativeAutomation.d.ts.map +1 -0
  25. package/dist/native/NativeAutomation.js +12 -0
  26. package/dist/native/NativeAutomation.js.map +1 -0
  27. package/dist/native/NativeCoordinateMapper.d.ts +23 -0
  28. package/dist/native/NativeCoordinateMapper.d.ts.map +1 -0
  29. package/dist/native/NativeCoordinateMapper.js +201 -0
  30. package/dist/native/NativeCoordinateMapper.js.map +1 -0
  31. package/dist/native/NativeFileDialogService.d.ts +26 -0
  32. package/dist/native/NativeFileDialogService.d.ts.map +1 -0
  33. package/dist/native/NativeFileDialogService.js +121 -0
  34. package/dist/native/NativeFileDialogService.js.map +1 -0
  35. package/dist/native/NativeImageFinder.d.ts +12 -0
  36. package/dist/native/NativeImageFinder.d.ts.map +1 -0
  37. package/dist/native/NativeImageFinder.js +29 -0
  38. package/dist/native/NativeImageFinder.js.map +1 -0
  39. package/dist/native/NativeKeyboard.d.ts +10 -0
  40. package/dist/native/NativeKeyboard.d.ts.map +1 -0
  41. package/dist/native/NativeKeyboard.js +16 -0
  42. package/dist/native/NativeKeyboard.js.map +1 -0
  43. package/dist/native/NativeMouse.d.ts +38 -0
  44. package/dist/native/NativeMouse.d.ts.map +1 -0
  45. package/dist/native/NativeMouse.js +82 -0
  46. package/dist/native/NativeMouse.js.map +1 -0
  47. package/dist/native/NativeWindowService.d.ts +31 -0
  48. package/dist/native/NativeWindowService.d.ts.map +1 -0
  49. package/dist/native/NativeWindowService.js +183 -0
  50. package/dist/native/NativeWindowService.js.map +1 -0
  51. package/dist/native/UnsupportedNativeAutomation.d.ts +4 -0
  52. package/dist/native/UnsupportedNativeAutomation.d.ts.map +1 -0
  53. package/dist/native/UnsupportedNativeAutomation.js +77 -0
  54. package/dist/native/UnsupportedNativeAutomation.js.map +1 -0
  55. package/dist/native/WindowMatcher.d.ts +4 -0
  56. package/dist/native/WindowMatcher.d.ts.map +1 -0
  57. package/dist/native/WindowMatcher.js +39 -0
  58. package/dist/native/WindowMatcher.js.map +1 -0
  59. package/dist/native/drivers.d.ts +37 -0
  60. package/dist/native/drivers.d.ts.map +1 -0
  61. package/dist/native/drivers.js +2 -0
  62. package/dist/native/drivers.js.map +1 -0
  63. package/dist/native/errors.d.ts +23 -0
  64. package/dist/native/errors.d.ts.map +1 -0
  65. package/dist/native/errors.js +45 -0
  66. package/dist/native/errors.js.map +1 -0
  67. package/dist/native/index.d.ts +13 -0
  68. package/dist/native/index.d.ts.map +1 -0
  69. package/dist/native/index.js +13 -0
  70. package/dist/native/index.js.map +1 -0
  71. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts +11 -0
  72. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts.map +1 -0
  73. package/dist/native/macos/MacOSAccessibilityWindowDriver.js +180 -0
  74. package/dist/native/macos/MacOSAccessibilityWindowDriver.js.map +1 -0
  75. package/dist/native/macos/MacOSAppleScriptClient.d.ts +24 -0
  76. package/dist/native/macos/MacOSAppleScriptClient.d.ts.map +1 -0
  77. package/dist/native/macos/MacOSAppleScriptClient.js +163 -0
  78. package/dist/native/macos/MacOSAppleScriptClient.js.map +1 -0
  79. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts +10 -0
  80. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts.map +1 -0
  81. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js +12 -0
  82. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js.map +1 -0
  83. package/dist/native/macos/MacOSNativeAutomation.d.ts +3 -0
  84. package/dist/native/macos/MacOSNativeAutomation.d.ts.map +1 -0
  85. package/dist/native/macos/MacOSNativeAutomation.js +88 -0
  86. package/dist/native/macos/MacOSNativeAutomation.js.map +1 -0
  87. package/dist/native/nut/NutNativeImageFinder.d.ts +17 -0
  88. package/dist/native/nut/NutNativeImageFinder.d.ts.map +1 -0
  89. package/dist/native/nut/NutNativeImageFinder.js +84 -0
  90. package/dist/native/nut/NutNativeImageFinder.js.map +1 -0
  91. package/dist/native/nut/NutNativeKeyboardDriver.d.ts +8 -0
  92. package/dist/native/nut/NutNativeKeyboardDriver.d.ts.map +1 -0
  93. package/dist/native/nut/NutNativeKeyboardDriver.js +39 -0
  94. package/dist/native/nut/NutNativeKeyboardDriver.js.map +1 -0
  95. package/dist/native/nut/NutNativeMouseDriver.d.ts +8 -0
  96. package/dist/native/nut/NutNativeMouseDriver.d.ts.map +1 -0
  97. package/dist/native/nut/NutNativeMouseDriver.js +24 -0
  98. package/dist/native/nut/NutNativeMouseDriver.js.map +1 -0
  99. package/dist/native/nut/NutNativeScreenDriver.d.ts +6 -0
  100. package/dist/native/nut/NutNativeScreenDriver.d.ts.map +1 -0
  101. package/dist/native/nut/NutNativeScreenDriver.js +12 -0
  102. package/dist/native/nut/NutNativeScreenDriver.js.map +1 -0
  103. package/dist/native/nut/NutNativeWindowDriver.d.ts +6 -0
  104. package/dist/native/nut/NutNativeWindowDriver.d.ts.map +1 -0
  105. package/dist/native/nut/NutNativeWindowDriver.js +53 -0
  106. package/dist/native/nut/NutNativeWindowDriver.js.map +1 -0
  107. package/dist/native/nut/loadNut.d.ts +58 -0
  108. package/dist/native/nut/loadNut.d.ts.map +1 -0
  109. package/dist/native/nut/loadNut.js +25 -0
  110. package/dist/native/nut/loadNut.js.map +1 -0
  111. package/dist/native/types.d.ts +194 -0
  112. package/dist/native/types.d.ts.map +1 -0
  113. package/dist/native/types.js +2 -0
  114. package/dist/native/types.js.map +1 -0
  115. package/dist/native/utils/appleScriptEscape.d.ts +7 -0
  116. package/dist/native/utils/appleScriptEscape.d.ts.map +1 -0
  117. package/dist/native/utils/appleScriptEscape.js +11 -0
  118. package/dist/native/utils/appleScriptEscape.js.map +1 -0
  119. package/dist/native/utils/geometry.d.ts +12 -0
  120. package/dist/native/utils/geometry.d.ts.map +1 -0
  121. package/dist/native/utils/geometry.js +77 -0
  122. package/dist/native/utils/geometry.js.map +1 -0
  123. package/dist/native/utils/redactNative.d.ts +2 -0
  124. package/dist/native/utils/redactNative.d.ts.map +1 -0
  125. package/dist/native/utils/redactNative.js +7 -0
  126. package/dist/native/utils/redactNative.js.map +1 -0
  127. package/dist/native/utils/waitFor.d.ts +7 -0
  128. package/dist/native/utils/waitFor.d.ts.map +1 -0
  129. package/dist/native/utils/waitFor.js +17 -0
  130. package/dist/native/utils/waitFor.js.map +1 -0
  131. package/dist/sites/upwork-com/upwork-com.actor.d.ts +4 -1
  132. package/dist/sites/upwork-com/upwork-com.actor.d.ts.map +1 -1
  133. package/dist/sites/upwork-com/upwork-com.actor.js +30 -12
  134. package/dist/sites/upwork-com/upwork-com.actor.js.map +1 -1
  135. package/dist/sites/upwork-com/upwork-com.types.d.ts +3 -1
  136. package/dist/sites/upwork-com/upwork-com.types.d.ts.map +1 -1
  137. package/dist/sites/upwork-com/upwork-com.types.js.map +1 -1
  138. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts +70 -0
  139. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts.map +1 -0
  140. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js +334 -0
  141. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js.map +1 -0
  142. package/dist/sites/upwork-com/util/scrapeJobListing.d.ts +1 -0
  143. package/dist/sites/upwork-com/util/scrapeJobListing.d.ts.map +1 -1
  144. package/dist/sites/upwork-com/util/scrapeJobListing.js +4 -0
  145. package/dist/sites/upwork-com/util/scrapeJobListing.js.map +1 -1
  146. package/package.json +5 -1
  147. package/src/browser/RuntimeConfig.ts +57 -1
  148. package/src/core/ActorContext.ts +2 -0
  149. package/src/core/ActorRunner.ts +13 -1
  150. package/src/index.ts +2 -0
  151. package/src/native/CompositeNativeWindowDriver.ts +30 -0
  152. package/src/native/NativeActionRegistry.ts +114 -0
  153. package/src/native/NativeAutomation.ts +15 -0
  154. package/src/native/NativeCoordinateMapper.ts +258 -0
  155. package/src/native/NativeFileDialogService.ts +138 -0
  156. package/src/native/NativeImageFinder.ts +33 -0
  157. package/src/native/NativeKeyboard.ts +18 -0
  158. package/src/native/NativeMouse.ts +116 -0
  159. package/src/native/NativeWindowService.ts +229 -0
  160. package/src/native/UnsupportedNativeAutomation.ts +92 -0
  161. package/src/native/WindowMatcher.ts +31 -0
  162. package/src/native/drivers.ts +38 -0
  163. package/src/native/errors.ts +51 -0
  164. package/src/native/index.ts +12 -0
  165. package/src/native/macos/MacOSAccessibilityWindowDriver.ts +183 -0
  166. package/src/native/macos/MacOSAppleScriptClient.ts +182 -0
  167. package/src/native/macos/MacOSFileDialogAccessibilityStrategy.ts +11 -0
  168. package/src/native/macos/MacOSNativeAutomation.ts +86 -0
  169. package/src/native/nut/NutNativeImageFinder.ts +98 -0
  170. package/src/native/nut/NutNativeKeyboardDriver.ts +38 -0
  171. package/src/native/nut/NutNativeMouseDriver.ts +27 -0
  172. package/src/native/nut/NutNativeScreenDriver.ts +14 -0
  173. package/src/native/nut/NutNativeWindowDriver.ts +61 -0
  174. package/src/native/nut/loadNut.ts +86 -0
  175. package/src/native/types.ts +224 -0
  176. package/src/native/utils/appleScriptEscape.ts +11 -0
  177. package/src/native/utils/geometry.ts +88 -0
  178. package/src/native/utils/redactNative.ts +6 -0
  179. package/src/native/utils/waitFor.ts +25 -0
  180. package/src/sites/upwork-com/upwork-com.actor.ts +46 -15
  181. package/src/sites/upwork-com/upwork-com.types.ts +4 -1
  182. package/src/sites/upwork-com/util/parseJobApplicationDetails.ts +622 -0
  183. package/src/sites/upwork-com/util/scrapeJobListing.ts +4 -3
  184. package/tests/fixtures/makeContext.ts +7 -2
  185. package/tests/fixtures/native/FakeNativeAutomation.ts +138 -0
  186. package/tests/unit/browser/RuntimeConfig.native.test.ts +63 -0
  187. package/tests/unit/core/ActorRunner.native.test.ts +69 -0
  188. package/tests/unit/native/MacOSAppleScriptClient.test.ts +35 -0
  189. package/tests/unit/native/NativeActionRegistry.test.ts +34 -0
  190. package/tests/unit/native/NativeCoordinateMapper.test.ts +92 -0
  191. package/tests/unit/native/NativeFileDialogService.test.ts +91 -0
  192. package/tests/unit/native/NativeMouse.test.ts +91 -0
  193. package/tests/unit/native/NativeWindowService.test.ts +87 -0
  194. package/tests/unit/native/WindowMatcher.test.ts +32 -0
  195. package/tests/unit/native/appleScriptEscape.test.ts +9 -0
  196. package/tests/unit/sites/myvistage-com.login.test.ts +1 -1
  197. package/tests/unit/sites/myvistage-com.postComment.test.ts +0 -1
  198. package/tests/unit/sites/upwork-com.login.test.ts +1 -1
@@ -0,0 +1,229 @@
1
+ import type { Logger } from '../logging/Logger.js';
2
+ import type { NativeScreenDriver, NativeWindowDriver, NativeWindowHandle } from './drivers.js';
3
+ import { NativeAutomationError, NativeWindowNotFoundError } from './errors.js';
4
+ import { matchesWindow, matcherToString } from './WindowMatcher.js';
5
+ import type {
6
+ EnsureWindowInViewOptions,
7
+ NativeImageFinder,
8
+ NativeMouse,
9
+ NativeRegion,
10
+ NativeWindowInfo,
11
+ WaitOptions,
12
+ WindowButtonOptions,
13
+ WindowMatcher
14
+ } from './types.js';
15
+ import { centerOf, visibleEnough } from './utils/geometry.js';
16
+ import { waitFor } from './utils/waitFor.js';
17
+
18
+ export interface NativeWindowServiceDependencies {
19
+ driver: NativeWindowDriver;
20
+ screen: NativeScreenDriver;
21
+ mouse?: NativeMouse;
22
+ images?: NativeImageFinder;
23
+ logger?: Logger;
24
+ defaultTimeoutMs?: number;
25
+ }
26
+
27
+ export class DefaultNativeWindowService {
28
+ private readonly defaultTimeoutMs: number;
29
+
30
+ constructor(private readonly deps: NativeWindowServiceDependencies) {
31
+ this.defaultTimeoutMs = deps.defaultTimeoutMs ?? 15_000;
32
+ }
33
+
34
+ async list(): Promise<NativeWindowInfo[]> {
35
+ const handles = await this.deps.driver.list();
36
+ return Promise.all(handles.map(handle => handle.info()));
37
+ }
38
+
39
+ async find(matcher: WindowMatcher, options: WaitOptions = {}): Promise<NativeWindowInfo | null> {
40
+ if (options.timeoutMs !== undefined && options.timeoutMs > 0) {
41
+ try {
42
+ return await this.waitFor(matcher, options);
43
+ } catch (error) {
44
+ if (error instanceof NativeWindowNotFoundError) return null;
45
+ throw error;
46
+ }
47
+ }
48
+
49
+ const handle = await this.findHandle(matcher);
50
+ return handle === null ? null : handle.info();
51
+ }
52
+
53
+ async waitFor(matcher: WindowMatcher, options: WaitOptions = {}): Promise<NativeWindowInfo> {
54
+ try {
55
+ const handle = await this.waitForHandle(matcher, options);
56
+ return handle.info();
57
+ } catch (error) {
58
+ if (error instanceof NativeWindowNotFoundError) throw error;
59
+ throw new NativeWindowNotFoundError({ matcher: matcherToString(matcher), cause: error });
60
+ }
61
+ }
62
+
63
+ async isOpen(matcher: WindowMatcher): Promise<boolean> {
64
+ return (await this.findHandle(matcher)) !== null;
65
+ }
66
+
67
+ async focus(matcher: WindowMatcher): Promise<NativeWindowInfo> {
68
+ const handle = await this.waitForHandle(matcher);
69
+ const before = await handle.info();
70
+ if (before.isMinimized === true) {
71
+ await handle.restore();
72
+ }
73
+ await handle.focus();
74
+ return this.refreshHandleInfo(handle);
75
+ }
76
+
77
+ async ensureInView(matcher: WindowMatcher, options: EnsureWindowInViewOptions = {}): Promise<NativeWindowInfo> {
78
+ const restore = options.restore ?? true;
79
+ const focus = options.focus ?? true;
80
+ const moveIntoPrimaryDisplay = options.moveIntoPrimaryDisplay ?? true;
81
+ const minimumVisibleWidth = options.minimumVisibleWidth ?? 100;
82
+ const minimumVisibleHeight = options.minimumVisibleHeight ?? 80;
83
+
84
+ const handle = await this.waitForHandle(matcher, options);
85
+ let info = await handle.info();
86
+
87
+ if (restore && info.isMinimized === true) {
88
+ await handle.restore();
89
+ info = await handle.info();
90
+ }
91
+
92
+ if (focus) {
93
+ await handle.focus();
94
+ info = await handle.info();
95
+ }
96
+
97
+ if (moveIntoPrimaryDisplay) {
98
+ const primaryDisplay: NativeRegion = {
99
+ x: 0,
100
+ y: 0,
101
+ width: await this.deps.screen.width(),
102
+ height: await this.deps.screen.height()
103
+ };
104
+
105
+ if (!visibleEnough(info.region, primaryDisplay, minimumVisibleWidth, minimumVisibleHeight)) {
106
+ const target = {
107
+ x: Math.max(0, Math.min(info.region.x, Math.max(0, primaryDisplay.width - info.region.width))),
108
+ y: Math.max(0, Math.min(info.region.y, Math.max(0, primaryDisplay.height - info.region.height)))
109
+ };
110
+ await handle.move(target);
111
+ info = await handle.info();
112
+ }
113
+ }
114
+
115
+ return info;
116
+ }
117
+
118
+ async setFullscreen(matcher: WindowMatcher, fullscreen: boolean, options: WindowButtonOptions = {}): Promise<void> {
119
+ const handle = await this.waitForHandle(matcher, options);
120
+ const info = await handle.info();
121
+
122
+ if (info.isFullscreen === fullscreen) {
123
+ return;
124
+ }
125
+
126
+ if (handle.setFullscreen !== undefined) {
127
+ await handle.setFullscreen(fullscreen);
128
+ return;
129
+ }
130
+
131
+ if (handle.toggleFullscreen !== undefined) {
132
+ await handle.toggleFullscreen();
133
+ return;
134
+ }
135
+
136
+ if ((options.strategy ?? 'auto') !== 'accessibility' && this.deps.images !== undefined && this.deps.mouse !== undefined && options.fullscreenButtonTemplate !== undefined) {
137
+ await this.clickWindowButtonTemplate(info.region, options.fullscreenButtonTemplate, options);
138
+ return;
139
+ }
140
+
141
+ throw new NativeAutomationError('Window fullscreen control is unavailable for the matched window.', {
142
+ matcher: matcherToString(matcher),
143
+ strategy: options.strategy ?? 'auto'
144
+ });
145
+ }
146
+
147
+ async toggleFullscreen(matcher: WindowMatcher, options: WindowButtonOptions = {}): Promise<void> {
148
+ const handle = await this.waitForHandle(matcher, options);
149
+ const info = await handle.info();
150
+
151
+ if (handle.toggleFullscreen !== undefined) {
152
+ await handle.toggleFullscreen();
153
+ return;
154
+ }
155
+
156
+ if (handle.setFullscreen !== undefined && info.isFullscreen !== undefined) {
157
+ await handle.setFullscreen(!info.isFullscreen);
158
+ return;
159
+ }
160
+
161
+ if ((options.strategy ?? 'auto') !== 'accessibility' && this.deps.images !== undefined && this.deps.mouse !== undefined && options.fullscreenButtonTemplate !== undefined) {
162
+ await this.clickWindowButtonTemplate(info.region, options.fullscreenButtonTemplate, options);
163
+ return;
164
+ }
165
+
166
+ throw new NativeAutomationError('Window fullscreen toggle is unavailable for the matched window.', {
167
+ matcher: matcherToString(matcher),
168
+ strategy: options.strategy ?? 'auto'
169
+ });
170
+ }
171
+
172
+ async minimize(matcher: WindowMatcher): Promise<void> {
173
+ const handle = await this.waitForHandle(matcher);
174
+ await handle.minimize();
175
+ }
176
+
177
+ async restore(matcher: WindowMatcher): Promise<void> {
178
+ const handle = await this.waitForHandle(matcher);
179
+ await handle.restore();
180
+ }
181
+
182
+ private async findHandle(matcher: WindowMatcher): Promise<NativeWindowHandle | null> {
183
+ const handles = await this.deps.driver.list();
184
+ for (const handle of handles) {
185
+ const info = await handle.info();
186
+ if (matchesWindow(info, matcher)) {
187
+ return handle;
188
+ }
189
+ }
190
+ return null;
191
+ }
192
+
193
+ private async waitForHandle(matcher: WindowMatcher, options: WaitOptions = {}): Promise<NativeWindowHandle> {
194
+ try {
195
+ return await waitFor(
196
+ () => this.findHandle(matcher),
197
+ {
198
+ timeoutMs: options.timeoutMs ?? this.defaultTimeoutMs,
199
+ intervalMs: options.intervalMs ?? 100,
200
+ description: `native window (${matcherToString(matcher)})`
201
+ }
202
+ );
203
+ } catch (error) {
204
+ throw new NativeWindowNotFoundError({ matcher: matcherToString(matcher), cause: error });
205
+ }
206
+ }
207
+
208
+ private async refreshHandleInfo(handle: NativeWindowHandle): Promise<NativeWindowInfo> {
209
+ return handle.info();
210
+ }
211
+
212
+ private async clickWindowButtonTemplate(region: NativeRegion, template: string, options: WindowButtonOptions): Promise<void> {
213
+ if (this.deps.images === undefined || this.deps.mouse === undefined) {
214
+ throw new NativeAutomationError('Visual window button strategy requires image finder and native mouse dependencies.');
215
+ }
216
+ const padding = options.searchRegionPaddingPx ?? 96;
217
+ const searchRegion: NativeRegion = {
218
+ x: region.x,
219
+ y: region.y,
220
+ width: Math.min(region.width, padding),
221
+ height: Math.min(region.height, padding)
222
+ };
223
+ const imageOptions: { searchRegion: NativeRegion; timeoutMs?: number; intervalMs?: number } = { searchRegion };
224
+ if (options.timeoutMs !== undefined) imageOptions.timeoutMs = options.timeoutMs;
225
+ if (options.intervalMs !== undefined) imageOptions.intervalMs = options.intervalMs;
226
+ const match = await this.deps.images.waitForTemplate(template, imageOptions);
227
+ await this.deps.mouse.click(centerOf(match.region));
228
+ }
229
+ }
@@ -0,0 +1,92 @@
1
+ import type {
2
+ BrowserPoint,
3
+ CalibrationOptions,
4
+ CoordinateMapOptions,
5
+ CoordinateTransform,
6
+ NativeAutomation,
7
+ NativeFileDialogService,
8
+ NativeImageFinder,
9
+ NativeImageMatch,
10
+ NativeImageSearchOptions,
11
+ NativeKeyboard,
12
+ NativeMouse,
13
+ NativeMouseClickOptions,
14
+ NativeMouseMoveOptions,
15
+ NativeRegion,
16
+ NativeTemplateRef,
17
+ NativeWindowInfo,
18
+ NativeWindowService,
19
+ ScreenPoint,
20
+ SelectFileOptions,
21
+ WaitOptions,
22
+ WindowButtonOptions,
23
+ WindowMatcher
24
+ } from './types.js';
25
+ import { DefaultNativeActionRegistry, registerDefaultNativeActions } from './NativeActionRegistry.js';
26
+ import { UnsupportedPlatformError } from './errors.js';
27
+ import type { Logger } from '../logging/Logger.js';
28
+ import type { PageLike } from '../browser/PuppeteerLike.js';
29
+
30
+ class UnsupportedNativeWindowService implements NativeWindowService {
31
+ constructor(private readonly platform: NodeJS.Platform) {}
32
+ async list(): Promise<NativeWindowInfo[]> { throw new UnsupportedPlatformError(this.platform); }
33
+ async find(_matcher: WindowMatcher, _options?: WaitOptions): Promise<NativeWindowInfo | null> { throw new UnsupportedPlatformError(this.platform); }
34
+ async waitFor(_matcher: WindowMatcher, _options?: WaitOptions): Promise<NativeWindowInfo> { throw new UnsupportedPlatformError(this.platform); }
35
+ async isOpen(_matcher: WindowMatcher): Promise<boolean> { throw new UnsupportedPlatformError(this.platform); }
36
+ async focus(_matcher: WindowMatcher): Promise<NativeWindowInfo> { throw new UnsupportedPlatformError(this.platform); }
37
+ async ensureInView(_matcher: WindowMatcher): Promise<NativeWindowInfo> { throw new UnsupportedPlatformError(this.platform); }
38
+ async setFullscreen(_matcher: WindowMatcher, _fullscreen: boolean, _options?: WindowButtonOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
39
+ async toggleFullscreen(_matcher: WindowMatcher, _options?: WindowButtonOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
40
+ async minimize(_matcher: WindowMatcher): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
41
+ async restore(_matcher: WindowMatcher): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
42
+ }
43
+
44
+ class UnsupportedNativeFileDialogService implements NativeFileDialogService {
45
+ constructor(private readonly platform: NodeJS.Platform) {}
46
+ async selectFile(_filePath: string, _options?: SelectFileOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
47
+ }
48
+
49
+ class UnsupportedNativeMouse implements NativeMouse {
50
+ constructor(private readonly platform: NodeJS.Platform) {}
51
+ async moveTo(_point: ScreenPoint, _options?: NativeMouseMoveOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
52
+ async click(_point: ScreenPoint, _options?: NativeMouseClickOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
53
+ async clickRegion(_region: NativeRegion, _options?: NativeMouseClickOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
54
+ async clickBrowserPoint(_page: PageLike, _point: BrowserPoint, _options?: NativeMouseClickOptions): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
55
+ }
56
+
57
+ class UnsupportedNativeKeyboard implements NativeKeyboard {
58
+ constructor(private readonly platform: NodeJS.Platform) {}
59
+ async type(_text: string): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
60
+ async pressKey(_key: string): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
61
+ async pressShortcut(_keys: readonly string[]): Promise<void> { throw new UnsupportedPlatformError(this.platform); }
62
+ }
63
+
64
+ class UnsupportedNativeCoordinateMapper {
65
+ constructor(private readonly platform: NodeJS.Platform) {}
66
+ async calibrate(_page: PageLike, _options?: CalibrationOptions): Promise<CoordinateTransform> { throw new UnsupportedPlatformError(this.platform); }
67
+ async browserToScreen(_point: BrowserPoint, _options?: CoordinateMapOptions): Promise<ScreenPoint> { throw new UnsupportedPlatformError(this.platform); }
68
+ async screenToBrowser(_point: ScreenPoint, _options?: CoordinateMapOptions): Promise<BrowserPoint> { throw new UnsupportedPlatformError(this.platform); }
69
+ invalidate(): void {}
70
+ }
71
+
72
+ class UnsupportedNativeImageFinder implements NativeImageFinder {
73
+ constructor(private readonly platform: NodeJS.Platform) {}
74
+ async findTemplate(_template: string | NativeTemplateRef, _options?: NativeImageSearchOptions): Promise<NativeImageMatch | null> { throw new UnsupportedPlatformError(this.platform); }
75
+ async waitForTemplate(_template: string | NativeTemplateRef, _options?: NativeImageSearchOptions): Promise<NativeImageMatch> { throw new UnsupportedPlatformError(this.platform); }
76
+ }
77
+
78
+ export function createUnsupportedNativeAutomation(platform: NodeJS.Platform, logger: Logger): NativeAutomation {
79
+ const automation = {
80
+ windows: new UnsupportedNativeWindowService(platform),
81
+ fileDialogs: new UnsupportedNativeFileDialogService(platform),
82
+ mouse: new UnsupportedNativeMouse(platform),
83
+ keyboard: new UnsupportedNativeKeyboard(platform),
84
+ coordinates: new UnsupportedNativeCoordinateMapper(platform),
85
+ images: new UnsupportedNativeImageFinder(platform),
86
+ actions: undefined as unknown as DefaultNativeActionRegistry
87
+ } satisfies Omit<NativeAutomation, 'actions'> & { actions: DefaultNativeActionRegistry };
88
+
89
+ const registry = new DefaultNativeActionRegistry(() => ({ automation, logger }));
90
+ automation.actions = registerDefaultNativeActions(registry, automation, logger);
91
+ return automation;
92
+ }
@@ -0,0 +1,31 @@
1
+ import type { NativeWindowInfo, WindowMatcher } from './types.js';
2
+
3
+ function matchesRegex(value: string, regex: RegExp | string): boolean {
4
+ const expression = typeof regex === 'string' ? new RegExp(regex) : regex;
5
+ return expression.test(value);
6
+ }
7
+
8
+ function matchesOptional(actual: string | undefined, expected: string | undefined): boolean {
9
+ return expected === undefined || actual === expected;
10
+ }
11
+
12
+ export function matchesWindow(info: NativeWindowInfo, matcher: WindowMatcher): boolean {
13
+ if (!matchesOptional(info.appName, matcher.appName)) return false;
14
+ if (!matchesOptional(info.bundleId, matcher.bundleId)) return false;
15
+ if (!matchesOptional(info.role, matcher.role)) return false;
16
+ if (matcher.title !== undefined && info.title !== matcher.title) return false;
17
+ if (matcher.titleIncludes !== undefined && !info.title.includes(matcher.titleIncludes)) return false;
18
+ if (matcher.titleRegex !== undefined && !matchesRegex(info.title, matcher.titleRegex)) return false;
19
+ return true;
20
+ }
21
+
22
+ export function matcherToString(matcher: WindowMatcher): string {
23
+ const parts: string[] = [];
24
+ if (matcher.appName !== undefined) parts.push(`app=${matcher.appName}`);
25
+ if (matcher.bundleId !== undefined) parts.push(`bundle=${matcher.bundleId}`);
26
+ if (matcher.title !== undefined) parts.push(`title=${matcher.title}`);
27
+ if (matcher.titleIncludes !== undefined) parts.push(`titleIncludes=${matcher.titleIncludes}`);
28
+ if (matcher.titleRegex !== undefined) parts.push(`titleRegex=${String(matcher.titleRegex)}`);
29
+ if (matcher.role !== undefined) parts.push(`role=${matcher.role}`);
30
+ return parts.length === 0 ? '<any window>' : parts.join(', ');
31
+ }
@@ -0,0 +1,38 @@
1
+ import type { NativeMouseButton, NativePoint, NativeRegion, NativeSize, NativeWindowInfo } from './types.js';
2
+
3
+ export interface NativeMouseDriver {
4
+ getPosition(): Promise<NativePoint>;
5
+ move(point: NativePoint): Promise<void>;
6
+ click(button: NativeMouseButton): Promise<void>;
7
+ }
8
+
9
+ export interface NativeKeyboardDriver {
10
+ type(text: string, options?: { delayMs?: number }): Promise<void>;
11
+ pressKey(key: string): Promise<void>;
12
+ pressShortcut(keys: readonly string[]): Promise<void>;
13
+ }
14
+
15
+ export interface NativeScreenDriver {
16
+ width(): Promise<number>;
17
+ height(): Promise<number>;
18
+ }
19
+
20
+ export interface NativeWindowHandle {
21
+ info(): Promise<NativeWindowInfo>;
22
+ focus(): Promise<void>;
23
+ minimize(): Promise<void>;
24
+ restore(): Promise<void>;
25
+ move(point: NativePoint): Promise<void>;
26
+ resize(size: NativeSize): Promise<void>;
27
+ setFullscreen?(fullscreen: boolean): Promise<void>;
28
+ toggleFullscreen?(): Promise<void>;
29
+ }
30
+
31
+ export interface NativeWindowDriver {
32
+ list(): Promise<NativeWindowHandle[]>;
33
+ active?(): Promise<NativeWindowHandle | null>;
34
+ }
35
+
36
+ export interface AppleScriptExecutor {
37
+ execute(script: string, options?: { timeoutMs?: number }): Promise<string>;
38
+ }
@@ -0,0 +1,51 @@
1
+ export class NativeAutomationError extends Error {
2
+ constructor(message: string, readonly meta: Record<string, unknown> = {}) {
3
+ super(message);
4
+ this.name = 'NativeAutomationError';
5
+ }
6
+ }
7
+
8
+ export class UnsupportedPlatformError extends NativeAutomationError {
9
+ constructor(platform: NodeJS.Platform) {
10
+ super(`Native automation is not supported on platform: ${platform}.`, { platform });
11
+ this.name = 'UnsupportedPlatformError';
12
+ }
13
+ }
14
+
15
+ export class NativeDependencyError extends NativeAutomationError {
16
+ constructor(packageName: string, cause?: unknown) {
17
+ super(
18
+ `Native automation dependency ${packageName} is not available. Install the optional native automation peer dependencies before using this feature.`,
19
+ cause === undefined ? { packageName } : { packageName, cause }
20
+ );
21
+ this.name = 'NativeDependencyError';
22
+ }
23
+ }
24
+
25
+ export class NativeWindowNotFoundError extends NativeAutomationError {
26
+ constructor(meta: Record<string, unknown>) {
27
+ super('Native window was not found.', meta);
28
+ this.name = 'NativeWindowNotFoundError';
29
+ }
30
+ }
31
+
32
+ export class NativeImageNotFoundError extends NativeAutomationError {
33
+ constructor(meta: Record<string, unknown>) {
34
+ super('Native image/template was not found on screen.', meta);
35
+ this.name = 'NativeImageNotFoundError';
36
+ }
37
+ }
38
+
39
+ export class NativeFileDialogError extends NativeAutomationError {
40
+ constructor(message: string, meta: Record<string, unknown> = {}) {
41
+ super(message, meta);
42
+ this.name = 'NativeFileDialogError';
43
+ }
44
+ }
45
+
46
+ export class CoordinateCalibrationError extends NativeAutomationError {
47
+ constructor(message: string, meta: Record<string, unknown> = {}) {
48
+ super(message, meta);
49
+ this.name = 'CoordinateCalibrationError';
50
+ }
51
+ }
@@ -0,0 +1,12 @@
1
+ export * from './types.js';
2
+ export * from './errors.js';
3
+ export * from './drivers.js';
4
+ export * from './NativeActionRegistry.js';
5
+ export * from './NativeAutomation.js';
6
+ export * from './NativeCoordinateMapper.js';
7
+ export * from './NativeFileDialogService.js';
8
+ export * from './NativeImageFinder.js';
9
+ export * from './NativeKeyboard.js';
10
+ export * from './NativeMouse.js';
11
+ export * from './NativeWindowService.js';
12
+ export * from './WindowMatcher.js';
@@ -0,0 +1,183 @@
1
+ import type { Logger } from '../../logging/Logger.js';
2
+ import type { NativeWindowDriver, NativeWindowHandle } from '../drivers.js';
3
+ import type { NativePoint, NativeRegion, NativeSize, NativeWindowInfo } from '../types.js';
4
+ import { appleScriptString } from '../utils/appleScriptEscape.js';
5
+ import { MacOSAppleScriptClient } from './MacOSAppleScriptClient.js';
6
+
7
+ const FIELD_SEPARATOR = '\u001f';
8
+ const ROW_SEPARATOR = '\u001e';
9
+
10
+ function parseBoolean(value: string): boolean | undefined {
11
+ if (value === 'true') return true;
12
+ if (value === 'false') return false;
13
+ return undefined;
14
+ }
15
+
16
+ function parseNumber(value: string): number {
17
+ const parsed = Number(value);
18
+ return Number.isFinite(parsed) ? parsed : 0;
19
+ }
20
+
21
+ class MacOSAccessibilityWindowHandle implements NativeWindowHandle {
22
+ constructor(private readonly client: MacOSAppleScriptClient, private currentInfo: NativeWindowInfo) {}
23
+
24
+ async info(): Promise<NativeWindowInfo> {
25
+ return this.currentInfo;
26
+ }
27
+
28
+ async focus(): Promise<void> {
29
+ await this.client.focusWindow(this.matcher(), this.currentInfo.appName ?? 'Google Chrome');
30
+ this.currentInfo = { ...this.currentInfo, isFocused: true };
31
+ }
32
+
33
+ async minimize(): Promise<void> {
34
+ await this.client.minimize(this.matcher(), this.currentInfo.appName ?? 'Google Chrome');
35
+ this.currentInfo = { ...this.currentInfo, isMinimized: true };
36
+ }
37
+
38
+ async restore(): Promise<void> {
39
+ await this.executeOnWindow(`set value of attribute "AXMinimized" of candidateWindow to false`);
40
+ this.currentInfo = { ...this.currentInfo, isMinimized: false };
41
+ }
42
+
43
+ async move(point: NativePoint): Promise<void> {
44
+ await this.executeOnWindow(`set position of candidateWindow to {${Math.round(point.x)}, ${Math.round(point.y)}}`);
45
+ this.currentInfo = {
46
+ ...this.currentInfo,
47
+ region: { ...this.currentInfo.region, x: Math.round(point.x), y: Math.round(point.y) }
48
+ };
49
+ }
50
+
51
+ async resize(size: NativeSize): Promise<void> {
52
+ await this.executeOnWindow(`set size of candidateWindow to {${Math.round(size.width)}, ${Math.round(size.height)}}`);
53
+ this.currentInfo = {
54
+ ...this.currentInfo,
55
+ region: { ...this.currentInfo.region, width: Math.round(size.width), height: Math.round(size.height) }
56
+ };
57
+ }
58
+
59
+ async setFullscreen(fullscreen: boolean): Promise<void> {
60
+ await this.client.setFullscreen(this.matcher(), fullscreen, this.currentInfo.appName ?? 'Google Chrome');
61
+ this.currentInfo = { ...this.currentInfo, isFullscreen: fullscreen };
62
+ }
63
+
64
+ async toggleFullscreen(): Promise<void> {
65
+ await this.client.toggleFullscreen(this.matcher(), this.currentInfo.appName ?? 'Google Chrome');
66
+ if (this.currentInfo.isFullscreen === undefined) {
67
+ this.currentInfo = { ...this.currentInfo };
68
+ } else {
69
+ this.currentInfo = { ...this.currentInfo, isFullscreen: !this.currentInfo.isFullscreen };
70
+ }
71
+ }
72
+
73
+ private matcher() {
74
+ const matcher: { appName?: string; title?: string } = {};
75
+ if (this.currentInfo.appName !== undefined) matcher.appName = this.currentInfo.appName;
76
+ matcher.title = this.currentInfo.title;
77
+ return matcher;
78
+ }
79
+
80
+ private async executeOnWindow(action: string): Promise<void> {
81
+ const appName = this.currentInfo.appName ?? 'Google Chrome';
82
+ const title = this.currentInfo.title;
83
+ const script = `
84
+ tell application "System Events"
85
+ tell process ${appleScriptString(appName)}
86
+ repeat with candidateWindow in windows
87
+ if name of candidateWindow is ${appleScriptString(title)} then
88
+ ${action}
89
+ return
90
+ end if
91
+ end repeat
92
+ end tell
93
+ end tell
94
+ `;
95
+ await this.client.execute(script);
96
+ }
97
+ }
98
+
99
+ export class MacOSAccessibilityWindowDriver implements NativeWindowDriver {
100
+ constructor(private readonly client: MacOSAppleScriptClient, private readonly logger?: Logger) {}
101
+
102
+ async list(): Promise<NativeWindowHandle[]> {
103
+ const script = `
104
+ set fieldSeparator to ASCII character 31
105
+ set rowSeparator to ASCII character 30
106
+ set rows to {}
107
+ tell application "System Events"
108
+ repeat with appProcess in application processes
109
+ set appName to name of appProcess
110
+ repeat with candidateWindow in windows of appProcess
111
+ set windowTitle to ""
112
+ set px to 0
113
+ set py to 0
114
+ set windowWidth to 0
115
+ set windowHeight to 0
116
+ set minimizedValue to ""
117
+ set fullscreenValue to ""
118
+ try
119
+ set windowTitle to name of candidateWindow
120
+ end try
121
+ try
122
+ set windowPosition to position of candidateWindow
123
+ set px to item 1 of windowPosition
124
+ set py to item 2 of windowPosition
125
+ end try
126
+ try
127
+ set windowSize to size of candidateWindow
128
+ set windowWidth to item 1 of windowSize
129
+ set windowHeight to item 2 of windowSize
130
+ end try
131
+ try
132
+ set minimizedValue to (value of attribute "AXMinimized" of candidateWindow) as text
133
+ end try
134
+ try
135
+ set fullscreenValue to (value of attribute "AXFullScreen" of candidateWindow) as text
136
+ end try
137
+ copy (appName & fieldSeparator & windowTitle & fieldSeparator & px & fieldSeparator & py & fieldSeparator & windowWidth & fieldSeparator & windowHeight & fieldSeparator & minimizedValue & fieldSeparator & fullscreenValue) to end of rows
138
+ end repeat
139
+ end repeat
140
+ end tell
141
+ set AppleScript's text item delimiters to rowSeparator
142
+ return rows as text
143
+ `;
144
+
145
+ try {
146
+ const output = await this.client.execute(script);
147
+ if (output.length === 0) return [];
148
+ return output.split(ROW_SEPARATOR)
149
+ .map(row => this.parseRow(row))
150
+ .filter((info): info is NativeWindowInfo => info !== null)
151
+ .map(info => new MacOSAccessibilityWindowHandle(this.client, info));
152
+ } catch (error) {
153
+ this.logger?.debug('Failed to list windows through macOS Accessibility.', { error });
154
+ throw error;
155
+ }
156
+ }
157
+
158
+ private parseRow(row: string): NativeWindowInfo | null {
159
+ const fields = row.split(FIELD_SEPARATOR);
160
+ if (fields.length < 8) return null;
161
+ const [appName, title, xRaw, yRaw, widthRaw, heightRaw, minimizedRaw, fullscreenRaw] = fields;
162
+ if (appName === undefined || title === undefined || xRaw === undefined || yRaw === undefined || widthRaw === undefined || heightRaw === undefined) {
163
+ return null;
164
+ }
165
+
166
+ const region: NativeRegion = {
167
+ x: parseNumber(xRaw),
168
+ y: parseNumber(yRaw),
169
+ width: parseNumber(widthRaw),
170
+ height: parseNumber(heightRaw)
171
+ };
172
+ const info: NativeWindowInfo = {
173
+ appName,
174
+ title,
175
+ region
176
+ };
177
+ const isMinimized = parseBoolean(minimizedRaw ?? '');
178
+ const isFullscreen = parseBoolean(fullscreenRaw ?? '');
179
+ if (isMinimized !== undefined) info.isMinimized = isMinimized;
180
+ if (isFullscreen !== undefined) info.isFullscreen = isFullscreen;
181
+ return info;
182
+ }
183
+ }