@epsilon-asi/actors 0.0.21 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/dist/browser/RuntimeConfig.d.ts +26 -0
  2. package/dist/browser/RuntimeConfig.d.ts.map +1 -1
  3. package/dist/browser/RuntimeConfig.js +29 -1
  4. package/dist/browser/RuntimeConfig.js.map +1 -1
  5. package/dist/core/ActorContext.d.ts +2 -0
  6. package/dist/core/ActorContext.d.ts.map +1 -1
  7. package/dist/core/ActorRunner.d.ts +3 -0
  8. package/dist/core/ActorRunner.d.ts.map +1 -1
  9. package/dist/core/ActorRunner.js +11 -1
  10. package/dist/core/ActorRunner.js.map +1 -1
  11. package/dist/index.d.ts +2 -0
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +2 -0
  14. package/dist/index.js.map +1 -1
  15. package/dist/native/CompositeNativeWindowDriver.d.ts +11 -0
  16. package/dist/native/CompositeNativeWindowDriver.d.ts.map +1 -0
  17. package/dist/native/CompositeNativeWindowDriver.js +31 -0
  18. package/dist/native/CompositeNativeWindowDriver.js.map +1 -0
  19. package/dist/native/NativeActionRegistry.d.ts +14 -0
  20. package/dist/native/NativeActionRegistry.d.ts.map +1 -0
  21. package/dist/native/NativeActionRegistry.js +101 -0
  22. package/dist/native/NativeActionRegistry.js.map +1 -0
  23. package/dist/native/NativeAutomation.d.ts +3 -0
  24. package/dist/native/NativeAutomation.d.ts.map +1 -0
  25. package/dist/native/NativeAutomation.js +12 -0
  26. package/dist/native/NativeAutomation.js.map +1 -0
  27. package/dist/native/NativeCoordinateMapper.d.ts +23 -0
  28. package/dist/native/NativeCoordinateMapper.d.ts.map +1 -0
  29. package/dist/native/NativeCoordinateMapper.js +201 -0
  30. package/dist/native/NativeCoordinateMapper.js.map +1 -0
  31. package/dist/native/NativeFileDialogService.d.ts +26 -0
  32. package/dist/native/NativeFileDialogService.d.ts.map +1 -0
  33. package/dist/native/NativeFileDialogService.js +121 -0
  34. package/dist/native/NativeFileDialogService.js.map +1 -0
  35. package/dist/native/NativeImageFinder.d.ts +12 -0
  36. package/dist/native/NativeImageFinder.d.ts.map +1 -0
  37. package/dist/native/NativeImageFinder.js +29 -0
  38. package/dist/native/NativeImageFinder.js.map +1 -0
  39. package/dist/native/NativeKeyboard.d.ts +10 -0
  40. package/dist/native/NativeKeyboard.d.ts.map +1 -0
  41. package/dist/native/NativeKeyboard.js +16 -0
  42. package/dist/native/NativeKeyboard.js.map +1 -0
  43. package/dist/native/NativeMouse.d.ts +38 -0
  44. package/dist/native/NativeMouse.d.ts.map +1 -0
  45. package/dist/native/NativeMouse.js +82 -0
  46. package/dist/native/NativeMouse.js.map +1 -0
  47. package/dist/native/NativeWindowService.d.ts +31 -0
  48. package/dist/native/NativeWindowService.d.ts.map +1 -0
  49. package/dist/native/NativeWindowService.js +183 -0
  50. package/dist/native/NativeWindowService.js.map +1 -0
  51. package/dist/native/UnsupportedNativeAutomation.d.ts +4 -0
  52. package/dist/native/UnsupportedNativeAutomation.d.ts.map +1 -0
  53. package/dist/native/UnsupportedNativeAutomation.js +77 -0
  54. package/dist/native/UnsupportedNativeAutomation.js.map +1 -0
  55. package/dist/native/WindowMatcher.d.ts +4 -0
  56. package/dist/native/WindowMatcher.d.ts.map +1 -0
  57. package/dist/native/WindowMatcher.js +39 -0
  58. package/dist/native/WindowMatcher.js.map +1 -0
  59. package/dist/native/drivers.d.ts +37 -0
  60. package/dist/native/drivers.d.ts.map +1 -0
  61. package/dist/native/drivers.js +2 -0
  62. package/dist/native/drivers.js.map +1 -0
  63. package/dist/native/errors.d.ts +23 -0
  64. package/dist/native/errors.d.ts.map +1 -0
  65. package/dist/native/errors.js +45 -0
  66. package/dist/native/errors.js.map +1 -0
  67. package/dist/native/index.d.ts +13 -0
  68. package/dist/native/index.d.ts.map +1 -0
  69. package/dist/native/index.js +13 -0
  70. package/dist/native/index.js.map +1 -0
  71. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts +11 -0
  72. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts.map +1 -0
  73. package/dist/native/macos/MacOSAccessibilityWindowDriver.js +180 -0
  74. package/dist/native/macos/MacOSAccessibilityWindowDriver.js.map +1 -0
  75. package/dist/native/macos/MacOSAppleScriptClient.d.ts +24 -0
  76. package/dist/native/macos/MacOSAppleScriptClient.d.ts.map +1 -0
  77. package/dist/native/macos/MacOSAppleScriptClient.js +163 -0
  78. package/dist/native/macos/MacOSAppleScriptClient.js.map +1 -0
  79. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts +10 -0
  80. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts.map +1 -0
  81. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js +12 -0
  82. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js.map +1 -0
  83. package/dist/native/macos/MacOSNativeAutomation.d.ts +3 -0
  84. package/dist/native/macos/MacOSNativeAutomation.d.ts.map +1 -0
  85. package/dist/native/macos/MacOSNativeAutomation.js +88 -0
  86. package/dist/native/macos/MacOSNativeAutomation.js.map +1 -0
  87. package/dist/native/nut/NutNativeImageFinder.d.ts +17 -0
  88. package/dist/native/nut/NutNativeImageFinder.d.ts.map +1 -0
  89. package/dist/native/nut/NutNativeImageFinder.js +84 -0
  90. package/dist/native/nut/NutNativeImageFinder.js.map +1 -0
  91. package/dist/native/nut/NutNativeKeyboardDriver.d.ts +8 -0
  92. package/dist/native/nut/NutNativeKeyboardDriver.d.ts.map +1 -0
  93. package/dist/native/nut/NutNativeKeyboardDriver.js +39 -0
  94. package/dist/native/nut/NutNativeKeyboardDriver.js.map +1 -0
  95. package/dist/native/nut/NutNativeMouseDriver.d.ts +8 -0
  96. package/dist/native/nut/NutNativeMouseDriver.d.ts.map +1 -0
  97. package/dist/native/nut/NutNativeMouseDriver.js +24 -0
  98. package/dist/native/nut/NutNativeMouseDriver.js.map +1 -0
  99. package/dist/native/nut/NutNativeScreenDriver.d.ts +6 -0
  100. package/dist/native/nut/NutNativeScreenDriver.d.ts.map +1 -0
  101. package/dist/native/nut/NutNativeScreenDriver.js +12 -0
  102. package/dist/native/nut/NutNativeScreenDriver.js.map +1 -0
  103. package/dist/native/nut/NutNativeWindowDriver.d.ts +6 -0
  104. package/dist/native/nut/NutNativeWindowDriver.d.ts.map +1 -0
  105. package/dist/native/nut/NutNativeWindowDriver.js +53 -0
  106. package/dist/native/nut/NutNativeWindowDriver.js.map +1 -0
  107. package/dist/native/nut/loadNut.d.ts +58 -0
  108. package/dist/native/nut/loadNut.d.ts.map +1 -0
  109. package/dist/native/nut/loadNut.js +25 -0
  110. package/dist/native/nut/loadNut.js.map +1 -0
  111. package/dist/native/types.d.ts +194 -0
  112. package/dist/native/types.d.ts.map +1 -0
  113. package/dist/native/types.js +2 -0
  114. package/dist/native/types.js.map +1 -0
  115. package/dist/native/utils/appleScriptEscape.d.ts +7 -0
  116. package/dist/native/utils/appleScriptEscape.d.ts.map +1 -0
  117. package/dist/native/utils/appleScriptEscape.js +11 -0
  118. package/dist/native/utils/appleScriptEscape.js.map +1 -0
  119. package/dist/native/utils/geometry.d.ts +12 -0
  120. package/dist/native/utils/geometry.d.ts.map +1 -0
  121. package/dist/native/utils/geometry.js +77 -0
  122. package/dist/native/utils/geometry.js.map +1 -0
  123. package/dist/native/utils/redactNative.d.ts +2 -0
  124. package/dist/native/utils/redactNative.d.ts.map +1 -0
  125. package/dist/native/utils/redactNative.js +7 -0
  126. package/dist/native/utils/redactNative.js.map +1 -0
  127. package/dist/native/utils/waitFor.d.ts +7 -0
  128. package/dist/native/utils/waitFor.d.ts.map +1 -0
  129. package/dist/native/utils/waitFor.js +17 -0
  130. package/dist/native/utils/waitFor.js.map +1 -0
  131. package/dist/sites/upwork-com/upwork-com.actor.d.ts +4 -1
  132. package/dist/sites/upwork-com/upwork-com.actor.d.ts.map +1 -1
  133. package/dist/sites/upwork-com/upwork-com.actor.js +30 -12
  134. package/dist/sites/upwork-com/upwork-com.actor.js.map +1 -1
  135. package/dist/sites/upwork-com/upwork-com.types.d.ts +3 -1
  136. package/dist/sites/upwork-com/upwork-com.types.d.ts.map +1 -1
  137. package/dist/sites/upwork-com/upwork-com.types.js.map +1 -1
  138. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts +70 -0
  139. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts.map +1 -0
  140. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js +334 -0
  141. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js.map +1 -0
  142. package/dist/sites/upwork-com/util/scrapeJobListing.d.ts +1 -0
  143. package/dist/sites/upwork-com/util/scrapeJobListing.d.ts.map +1 -1
  144. package/dist/sites/upwork-com/util/scrapeJobListing.js +4 -0
  145. package/dist/sites/upwork-com/util/scrapeJobListing.js.map +1 -1
  146. package/package.json +5 -1
  147. package/src/browser/RuntimeConfig.ts +57 -1
  148. package/src/core/ActorContext.ts +2 -0
  149. package/src/core/ActorRunner.ts +13 -1
  150. package/src/index.ts +2 -0
  151. package/src/native/CompositeNativeWindowDriver.ts +30 -0
  152. package/src/native/NativeActionRegistry.ts +114 -0
  153. package/src/native/NativeAutomation.ts +15 -0
  154. package/src/native/NativeCoordinateMapper.ts +258 -0
  155. package/src/native/NativeFileDialogService.ts +138 -0
  156. package/src/native/NativeImageFinder.ts +33 -0
  157. package/src/native/NativeKeyboard.ts +18 -0
  158. package/src/native/NativeMouse.ts +116 -0
  159. package/src/native/NativeWindowService.ts +229 -0
  160. package/src/native/UnsupportedNativeAutomation.ts +92 -0
  161. package/src/native/WindowMatcher.ts +31 -0
  162. package/src/native/drivers.ts +38 -0
  163. package/src/native/errors.ts +51 -0
  164. package/src/native/index.ts +12 -0
  165. package/src/native/macos/MacOSAccessibilityWindowDriver.ts +183 -0
  166. package/src/native/macos/MacOSAppleScriptClient.ts +182 -0
  167. package/src/native/macos/MacOSFileDialogAccessibilityStrategy.ts +11 -0
  168. package/src/native/macos/MacOSNativeAutomation.ts +86 -0
  169. package/src/native/nut/NutNativeImageFinder.ts +98 -0
  170. package/src/native/nut/NutNativeKeyboardDriver.ts +38 -0
  171. package/src/native/nut/NutNativeMouseDriver.ts +27 -0
  172. package/src/native/nut/NutNativeScreenDriver.ts +14 -0
  173. package/src/native/nut/NutNativeWindowDriver.ts +61 -0
  174. package/src/native/nut/loadNut.ts +86 -0
  175. package/src/native/types.ts +224 -0
  176. package/src/native/utils/appleScriptEscape.ts +11 -0
  177. package/src/native/utils/geometry.ts +88 -0
  178. package/src/native/utils/redactNative.ts +6 -0
  179. package/src/native/utils/waitFor.ts +25 -0
  180. package/src/sites/upwork-com/upwork-com.actor.ts +46 -15
  181. package/src/sites/upwork-com/upwork-com.types.ts +4 -1
  182. package/src/sites/upwork-com/util/parseJobApplicationDetails.ts +622 -0
  183. package/src/sites/upwork-com/util/scrapeJobListing.ts +4 -3
  184. package/tests/fixtures/makeContext.ts +7 -2
  185. package/tests/fixtures/native/FakeNativeAutomation.ts +138 -0
  186. package/tests/unit/browser/RuntimeConfig.native.test.ts +63 -0
  187. package/tests/unit/core/ActorRunner.native.test.ts +69 -0
  188. package/tests/unit/native/MacOSAppleScriptClient.test.ts +35 -0
  189. package/tests/unit/native/NativeActionRegistry.test.ts +34 -0
  190. package/tests/unit/native/NativeCoordinateMapper.test.ts +92 -0
  191. package/tests/unit/native/NativeFileDialogService.test.ts +91 -0
  192. package/tests/unit/native/NativeMouse.test.ts +91 -0
  193. package/tests/unit/native/NativeWindowService.test.ts +87 -0
  194. package/tests/unit/native/WindowMatcher.test.ts +32 -0
  195. package/tests/unit/native/appleScriptEscape.test.ts +9 -0
  196. package/tests/unit/sites/myvistage-com.login.test.ts +1 -1
  197. package/tests/unit/sites/myvistage-com.postComment.test.ts +0 -1
  198. package/tests/unit/sites/upwork-com.login.test.ts +1 -1
@@ -0,0 +1,182 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ import type { AppleScriptExecutor } from '../drivers.js';
4
+ import type { WindowMatcher } from '../types.js';
5
+ import { appleScriptString } from '../utils/appleScriptEscape.js';
6
+
7
+ const execFileAsync = promisify(execFile);
8
+
9
+ export class OsaScriptExecutor implements AppleScriptExecutor {
10
+ async execute(script: string, options: { timeoutMs?: number } = {}): Promise<string> {
11
+ const { stdout } = await execFileAsync('osascript', ['-l', 'AppleScript', '-e', script], {
12
+ timeout: options.timeoutMs ?? 15_000,
13
+ maxBuffer: 1024 * 1024
14
+ });
15
+ return stdout.trim();
16
+ }
17
+ }
18
+
19
+ export interface MacOSAppleScriptClientOptions {
20
+ executor?: AppleScriptExecutor;
21
+ timeoutMs?: number;
22
+ }
23
+
24
+ function matcherCondition(matcher: WindowMatcher): string {
25
+ const conditions: string[] = [];
26
+ if (matcher.title !== undefined) {
27
+ conditions.push(`name of candidateWindow is ${appleScriptString(matcher.title)}`);
28
+ }
29
+ if (matcher.titleIncludes !== undefined) {
30
+ conditions.push(`name of candidateWindow contains ${appleScriptString(matcher.titleIncludes)}`);
31
+ }
32
+ if (matcher.role !== undefined) {
33
+ conditions.push(`role of candidateWindow is ${appleScriptString(matcher.role)}`);
34
+ }
35
+ return conditions.length === 0 ? 'true' : conditions.join(' and ');
36
+ }
37
+
38
+ function processTarget(matcher: WindowMatcher, fallbackAppName: string): string {
39
+ return appleScriptString(matcher.appName ?? fallbackAppName);
40
+ }
41
+
42
+ export class MacOSAppleScriptClient {
43
+ private readonly executor: AppleScriptExecutor;
44
+ private readonly timeoutMs: number;
45
+
46
+ constructor(options: MacOSAppleScriptClientOptions = {}) {
47
+ this.executor = options.executor ?? new OsaScriptExecutor();
48
+ this.timeoutMs = options.timeoutMs ?? 15_000;
49
+ }
50
+
51
+ async execute(script: string, timeoutMs?: number): Promise<string> {
52
+ return this.executor.execute(script, { timeoutMs: timeoutMs ?? this.timeoutMs });
53
+ }
54
+
55
+ async isWindowOpen(matcher: WindowMatcher, fallbackAppName = 'Google Chrome'): Promise<boolean> {
56
+ const script = `
57
+ tell application "System Events"
58
+ if not (exists process ${processTarget(matcher, fallbackAppName)}) then return "false"
59
+ tell process ${processTarget(matcher, fallbackAppName)}
60
+ repeat with candidateWindow in windows
61
+ if ${matcherCondition(matcher)} then return "true"
62
+ end repeat
63
+ end tell
64
+ end tell
65
+ return "false"
66
+ `;
67
+ return (await this.execute(script)) === 'true';
68
+ }
69
+
70
+ async focusWindow(matcher: WindowMatcher, fallbackAppName = 'Google Chrome'): Promise<void> {
71
+ const script = `
72
+ tell application ${processTarget(matcher, fallbackAppName)} to activate
73
+ tell application "System Events"
74
+ tell process ${processTarget(matcher, fallbackAppName)}
75
+ repeat with candidateWindow in windows
76
+ if ${matcherCondition(matcher)} then
77
+ perform action "AXRaise" of candidateWindow
78
+ return
79
+ end if
80
+ end repeat
81
+ end tell
82
+ end tell
83
+ `;
84
+ await this.execute(script);
85
+ }
86
+
87
+ async setFullscreen(matcher: WindowMatcher, fullscreen: boolean, fallbackAppName = 'Google Chrome'): Promise<void> {
88
+ const desired = fullscreen ? 'true' : 'false';
89
+ const script = `
90
+ tell application ${processTarget(matcher, fallbackAppName)} to activate
91
+ tell application "System Events"
92
+ tell process ${processTarget(matcher, fallbackAppName)}
93
+ repeat with candidateWindow in windows
94
+ if ${matcherCondition(matcher)} then
95
+ set currentFullscreen to false
96
+ try
97
+ set currentFullscreen to value of attribute "AXFullScreen" of candidateWindow
98
+ end try
99
+ if currentFullscreen is not ${desired} then
100
+ try
101
+ perform action "AXPress" of (first button of candidateWindow whose subrole is "AXFullScreenButton")
102
+ on error
103
+ click (first button of candidateWindow whose subrole is "AXFullScreenButton")
104
+ end try
105
+ end if
106
+ return
107
+ end if
108
+ end repeat
109
+ end tell
110
+ end tell
111
+ `;
112
+ await this.execute(script);
113
+ }
114
+
115
+ async toggleFullscreen(matcher: WindowMatcher, fallbackAppName = 'Google Chrome'): Promise<void> {
116
+ const script = `
117
+ tell application ${processTarget(matcher, fallbackAppName)} to activate
118
+ tell application "System Events"
119
+ tell process ${processTarget(matcher, fallbackAppName)}
120
+ repeat with candidateWindow in windows
121
+ if ${matcherCondition(matcher)} then
122
+ try
123
+ perform action "AXPress" of (first button of candidateWindow whose subrole is "AXFullScreenButton")
124
+ on error
125
+ click (first button of candidateWindow whose subrole is "AXFullScreenButton")
126
+ end try
127
+ return
128
+ end if
129
+ end repeat
130
+ end tell
131
+ end tell
132
+ `;
133
+ await this.execute(script);
134
+ }
135
+
136
+ async minimize(matcher: WindowMatcher, fallbackAppName = 'Google Chrome'): Promise<void> {
137
+ const script = `
138
+ tell application "System Events"
139
+ tell process ${processTarget(matcher, fallbackAppName)}
140
+ repeat with candidateWindow in windows
141
+ if ${matcherCondition(matcher)} then
142
+ try
143
+ perform action "AXPress" of (first button of candidateWindow whose subrole is "AXMinimizeButton")
144
+ on error
145
+ set value of attribute "AXMinimized" of candidateWindow to true
146
+ end try
147
+ return
148
+ end if
149
+ end repeat
150
+ end tell
151
+ end tell
152
+ `;
153
+ await this.execute(script);
154
+ }
155
+
156
+ async selectFileInOpenDialog(filePath: string, appName = 'Google Chrome', timeoutMs?: number): Promise<void> {
157
+ const script = `
158
+ set targetPath to ${appleScriptString(filePath)}
159
+ tell application ${appleScriptString(appName)} to activate
160
+ tell application "System Events"
161
+ tell process ${appleScriptString(appName)}
162
+ set frontmost to true
163
+ set waited to 0
164
+ repeat while waited < 100
165
+ if (exists window 1) then exit repeat
166
+ delay 0.1
167
+ set waited to waited + 1
168
+ end repeat
169
+ keystroke "g" using {command down, shift down}
170
+ delay 0.2
171
+ set the clipboard to targetPath
172
+ keystroke "v" using {command down}
173
+ delay 0.1
174
+ key code 36
175
+ delay 0.2
176
+ key code 36
177
+ end tell
178
+ end tell
179
+ `;
180
+ await this.execute(script, timeoutMs);
181
+ }
182
+ }
@@ -0,0 +1,11 @@
1
+ import type { SelectFileOptions } from '../types.js';
2
+ import type { FileDialogAccessibilityStrategy } from '../NativeFileDialogService.js';
3
+ import { MacOSAppleScriptClient } from './MacOSAppleScriptClient.js';
4
+
5
+ export class MacOSFileDialogAccessibilityStrategy implements FileDialogAccessibilityStrategy {
6
+ constructor(private readonly client: MacOSAppleScriptClient, private readonly defaultAppName = 'Google Chrome') {}
7
+
8
+ async selectFile(filePath: string, options: SelectFileOptions): Promise<void> {
9
+ await this.client.selectFileInOpenDialog(filePath, options.appName ?? this.defaultAppName, options.timeoutMs);
10
+ }
11
+ }
@@ -0,0 +1,86 @@
1
+ import type { NativeAutomation, NativeAutomationFactoryArgs } from '../types.js';
2
+ import { DefaultNativeActionRegistry, registerDefaultNativeActions } from '../NativeActionRegistry.js';
3
+ import { HumanizedNativeMouse, type HumanizedNativeMouseOptions } from '../NativeMouse.js';
4
+ import { DefaultNativeKeyboard } from '../NativeKeyboard.js';
5
+ import { DefaultNativeWindowService, type NativeWindowServiceDependencies } from '../NativeWindowService.js';
6
+ import { DefaultNativeImageFinder } from '../NativeImageFinder.js';
7
+ import { ImageBasedCoordinateMapper, type ImageBasedCoordinateMapperOptions } from '../NativeCoordinateMapper.js';
8
+ import { DefaultNativeFileDialogService, type NativeFileDialogServiceDependencies } from '../NativeFileDialogService.js';
9
+ import { NutNativeMouseDriver } from '../nut/NutNativeMouseDriver.js';
10
+ import { NutNativeKeyboardDriver } from '../nut/NutNativeKeyboardDriver.js';
11
+ import { NutNativeScreenDriver } from '../nut/NutNativeScreenDriver.js';
12
+ import { NutNativeWindowDriver } from '../nut/NutNativeWindowDriver.js';
13
+ import { FallbackNativeWindowDriver } from '../CompositeNativeWindowDriver.js';
14
+ import { MacOSAccessibilityWindowDriver } from './MacOSAccessibilityWindowDriver.js';
15
+ import { NutNativeImageFinderDriver, type NutNativeImageFinderOptions } from '../nut/NutNativeImageFinder.js';
16
+ import { MacOSAppleScriptClient, type MacOSAppleScriptClientOptions } from './MacOSAppleScriptClient.js';
17
+ import { MacOSFileDialogAccessibilityStrategy } from './MacOSFileDialogAccessibilityStrategy.js';
18
+
19
+ export function createMacOSNativeAutomation(args: NativeAutomationFactoryArgs): NativeAutomation {
20
+ const nativeConfig = args.config.native;
21
+ const mouseDriver = new NutNativeMouseDriver();
22
+ const keyboardDriver = new NutNativeKeyboardDriver();
23
+ const screenDriver = new NutNativeScreenDriver();
24
+ const appleScriptOptions: MacOSAppleScriptClientOptions = {};
25
+ const osascriptTimeoutMs = nativeConfig?.macos?.osascriptTimeoutMs ?? args.config.defaultTimeoutMs;
26
+ if (osascriptTimeoutMs !== undefined) appleScriptOptions.timeoutMs = osascriptTimeoutMs;
27
+ const appleScriptClient = new MacOSAppleScriptClient(appleScriptOptions);
28
+ const windowDriver = new FallbackNativeWindowDriver(
29
+ new MacOSAccessibilityWindowDriver(appleScriptClient, args.logger),
30
+ new NutNativeWindowDriver(),
31
+ args.logger
32
+ );
33
+ const keyboard = new DefaultNativeKeyboard(keyboardDriver);
34
+ const imageDriverOptions: NutNativeImageFinderOptions = { enableNlMatcher: true };
35
+ if (nativeConfig?.templatesDir !== undefined) imageDriverOptions.templatesDir = nativeConfig.templatesDir;
36
+ if (nativeConfig?.imageConfidence !== undefined) imageDriverOptions.confidence = nativeConfig.imageConfidence;
37
+ const imageDriver = new NutNativeImageFinderDriver(imageDriverOptions);
38
+ const images = new DefaultNativeImageFinder(imageDriver, args.config.defaultTimeoutMs);
39
+ const coordinateOptions: ImageBasedCoordinateMapperOptions = {
40
+ images,
41
+ logger: args.logger
42
+ };
43
+ const coordinateTimeoutMs = nativeConfig?.coordinateCalibration?.timeoutMs ?? args.config.defaultTimeoutMs;
44
+ if (coordinateTimeoutMs !== undefined) coordinateOptions.defaultTimeoutMs = coordinateTimeoutMs;
45
+ if (nativeConfig?.coordinateCalibration?.cacheTtlMs !== undefined) {
46
+ coordinateOptions.defaultCacheTtlMs = nativeConfig.coordinateCalibration.cacheTtlMs;
47
+ }
48
+ const coordinates = new ImageBasedCoordinateMapper(coordinateOptions);
49
+ const mouseOptions: HumanizedNativeMouseOptions = { coordinateMapper: coordinates };
50
+ if (nativeConfig?.mouse?.jitterPixels !== undefined) mouseOptions.jitterPixels = nativeConfig.mouse.jitterPixels;
51
+ if (nativeConfig?.mouse?.minJitterPixels !== undefined) mouseOptions.minJitterPixels = nativeConfig.mouse.minJitterPixels;
52
+ if (nativeConfig?.mouse?.moveSpeed !== undefined) mouseOptions.moveSpeed = nativeConfig.mouse.moveSpeed;
53
+ const mouse = new HumanizedNativeMouse(mouseDriver, mouseOptions);
54
+ const windowOptions: NativeWindowServiceDependencies = {
55
+ driver: windowDriver,
56
+ screen: screenDriver,
57
+ mouse,
58
+ images,
59
+ logger: args.logger
60
+ };
61
+ if (args.config.defaultTimeoutMs !== undefined) windowOptions.defaultTimeoutMs = args.config.defaultTimeoutMs;
62
+ const windows = new DefaultNativeWindowService(windowOptions);
63
+ const fileDialogOptions: NativeFileDialogServiceDependencies = {
64
+ keyboard,
65
+ mouse,
66
+ images,
67
+ accessibility: new MacOSFileDialogAccessibilityStrategy(appleScriptClient, nativeConfig?.macos?.targetApplicationName ?? 'Google Chrome'),
68
+ logger: args.logger
69
+ };
70
+ if (args.config.defaultTimeoutMs !== undefined) fileDialogOptions.defaultTimeoutMs = args.config.defaultTimeoutMs;
71
+ const fileDialogs = new DefaultNativeFileDialogService(fileDialogOptions);
72
+
73
+ const automation = {
74
+ windows,
75
+ fileDialogs,
76
+ mouse,
77
+ keyboard,
78
+ coordinates,
79
+ images,
80
+ actions: undefined as unknown as DefaultNativeActionRegistry
81
+ } satisfies Omit<NativeAutomation, 'actions'> & { actions: DefaultNativeActionRegistry };
82
+
83
+ const registry = new DefaultNativeActionRegistry(() => ({ automation, logger: args.logger }));
84
+ automation.actions = registerDefaultNativeActions(registry, automation, args.logger);
85
+ return automation;
86
+ }
@@ -0,0 +1,98 @@
1
+ import fs from 'node:fs/promises';
2
+ import os from 'node:os';
3
+ import path from 'node:path';
4
+ import type { NativeImageFinderDriver } from '../NativeImageFinder.js';
5
+ import type { NativeImageMatch, NativeImageSearchOptions, NativeRegion, NativeTemplateRef } from '../types.js';
6
+ import { loadNlMatcher, loadNut, type NutRegion } from './loadNut.js';
7
+
8
+ export interface NutNativeImageFinderOptions {
9
+ templatesDir?: string;
10
+ confidence?: number;
11
+ enableNlMatcher?: boolean;
12
+ }
13
+
14
+ function toRegion(region: NutRegion): NativeRegion {
15
+ return {
16
+ x: region.left ?? region.x ?? 0,
17
+ y: region.top ?? region.y ?? 0,
18
+ width: region.width,
19
+ height: region.height
20
+ };
21
+ }
22
+
23
+ function templateName(template: string | NativeTemplateRef): string {
24
+ return typeof template === 'string' ? template : template.name;
25
+ }
26
+
27
+ export class NutNativeImageFinderDriver implements NativeImageFinderDriver {
28
+ private initialized = false;
29
+
30
+ constructor(private readonly options: NutNativeImageFinderOptions = {}) {}
31
+
32
+ async findTemplate(template: string | NativeTemplateRef, options: NativeImageSearchOptions = {}): Promise<NativeImageMatch | null> {
33
+ const nut = await this.initialize();
34
+ if (nut.screen.find === undefined || nut.imageResource === undefined) {
35
+ return null;
36
+ }
37
+
38
+ const resourcePath = await this.materializeTemplate(template);
39
+ const searchOptions = this.buildSearchOptions(options);
40
+
41
+ try {
42
+ const region = await nut.screen.find(nut.imageResource(resourcePath), searchOptions);
43
+ const match: NativeImageMatch = { region: toRegion(region) };
44
+ if (options.confidence !== undefined) match.confidence = options.confidence;
45
+ return match;
46
+ } catch {
47
+ return null;
48
+ }
49
+ }
50
+
51
+ private async initialize() {
52
+ const nut = await loadNut();
53
+ if (!this.initialized) {
54
+ if (this.options.enableNlMatcher ?? true) {
55
+ await loadNlMatcher();
56
+ }
57
+ if (this.options.templatesDir !== undefined && nut.screen.config !== undefined) {
58
+ nut.screen.config.resourceDirectory = this.options.templatesDir;
59
+ }
60
+ if (this.options.confidence !== undefined && nut.screen.config !== undefined) {
61
+ nut.screen.config.confidence = this.options.confidence;
62
+ }
63
+ this.initialized = true;
64
+ }
65
+ return nut;
66
+ }
67
+
68
+ private buildSearchOptions(options: NativeImageSearchOptions): Record<string, unknown> | undefined {
69
+ const searchOptions: Record<string, unknown> = {};
70
+ if (options.searchRegion !== undefined) {
71
+ searchOptions.searchRegion = {
72
+ left: options.searchRegion.x,
73
+ top: options.searchRegion.y,
74
+ width: options.searchRegion.width,
75
+ height: options.searchRegion.height
76
+ };
77
+ }
78
+ if (options.confidence !== undefined) {
79
+ searchOptions.confidence = options.confidence;
80
+ }
81
+ return Object.keys(searchOptions).length === 0 ? undefined : searchOptions;
82
+ }
83
+
84
+ private async materializeTemplate(template: string | NativeTemplateRef): Promise<string> {
85
+ if (typeof template === 'string') {
86
+ return this.options.templatesDir === undefined ? template : path.join(this.options.templatesDir, template);
87
+ }
88
+
89
+ if (template.buffer === undefined) {
90
+ return this.options.templatesDir === undefined ? template.name : path.join(this.options.templatesDir, template.name);
91
+ }
92
+
93
+ const fileName = `${template.name.replace(/[^a-z0-9._-]/gi, '_')}`;
94
+ const filePath = path.join(os.tmpdir(), fileName);
95
+ await fs.writeFile(filePath, template.buffer);
96
+ return filePath;
97
+ }
98
+ }
@@ -0,0 +1,38 @@
1
+ import type { NativeKeyboardDriver } from '../drivers.js';
2
+ import { loadNut } from './loadNut.js';
3
+
4
+ function keyName(key: string): string {
5
+ const normalized = key.toLowerCase();
6
+ if (normalized === 'enter' || normalized === 'return') return 'ENTER';
7
+ if (normalized === 'escape' || normalized === 'esc') return 'ESCAPE';
8
+ if (normalized === 'meta' || normalized === 'command' || normalized === 'cmd') return 'LEFT_META';
9
+ if (normalized === 'control' || normalized === 'ctrl') return 'LEFT_CONTROL';
10
+ if (normalized === 'shift') return 'LEFT_SHIFT';
11
+ if (normalized === 'alt' || normalized === 'option') return 'LEFT_ALT';
12
+ return key.toUpperCase();
13
+ }
14
+
15
+ export class NutNativeKeyboardDriver implements NativeKeyboardDriver {
16
+ async type(text: string): Promise<void> {
17
+ const nut = await loadNut();
18
+ await nut.keyboard.type(text);
19
+ }
20
+
21
+ async pressKey(key: string): Promise<void> {
22
+ const nut = await loadNut();
23
+ await nut.keyboard.pressKey(this.resolveKey(nut.Key, key));
24
+ }
25
+
26
+ async pressShortcut(keys: readonly string[]): Promise<void> {
27
+ const nut = await loadNut();
28
+ const resolved = keys.map(key => this.resolveKey(nut.Key, key));
29
+ await nut.keyboard.pressKey(...resolved);
30
+ if (nut.keyboard.releaseKey !== undefined) {
31
+ await nut.keyboard.releaseKey(...resolved.slice().reverse());
32
+ }
33
+ }
34
+
35
+ private resolveKey(keyMap: Record<string, unknown> | undefined, key: string): unknown {
36
+ return keyMap?.[keyName(key)] ?? key;
37
+ }
38
+ }
@@ -0,0 +1,27 @@
1
+ import type { NativeMouseDriver } from '../drivers.js';
2
+ import type { NativeMouseButton, NativePoint } from '../types.js';
3
+ import { loadNut } from './loadNut.js';
4
+
5
+ function buttonName(button: NativeMouseButton): string {
6
+ if (button === 'left') return 'LEFT';
7
+ if (button === 'right') return 'RIGHT';
8
+ return 'MIDDLE';
9
+ }
10
+
11
+ export class NutNativeMouseDriver implements NativeMouseDriver {
12
+ async getPosition(): Promise<NativePoint> {
13
+ const nut = await loadNut();
14
+ return nut.mouse.getPosition();
15
+ }
16
+
17
+ async move(point: NativePoint): Promise<void> {
18
+ const nut = await loadNut();
19
+ await nut.mouse.move(nut.straightTo({ x: point.x, y: point.y }));
20
+ }
21
+
22
+ async click(button: NativeMouseButton): Promise<void> {
23
+ const nut = await loadNut();
24
+ const nutButton = nut.Button?.[buttonName(button)] ?? button;
25
+ await nut.mouse.click(nutButton);
26
+ }
27
+ }
@@ -0,0 +1,14 @@
1
+ import type { NativeScreenDriver } from '../drivers.js';
2
+ import { loadNut } from './loadNut.js';
3
+
4
+ export class NutNativeScreenDriver implements NativeScreenDriver {
5
+ async width(): Promise<number> {
6
+ const nut = await loadNut();
7
+ return nut.screen.width();
8
+ }
9
+
10
+ async height(): Promise<number> {
11
+ const nut = await loadNut();
12
+ return nut.screen.height();
13
+ }
14
+ }
@@ -0,0 +1,61 @@
1
+ import type { NativeWindowDriver, NativeWindowHandle } from '../drivers.js';
2
+ import type { NativePoint, NativeRegion, NativeSize, NativeWindowInfo } from '../types.js';
3
+ import { loadNut, type NutRegion, type NutWindow } from './loadNut.js';
4
+
5
+ function toRegion(region: NutRegion): NativeRegion {
6
+ return {
7
+ x: region.left ?? region.x ?? 0,
8
+ y: region.top ?? region.y ?? 0,
9
+ width: region.width,
10
+ height: region.height
11
+ };
12
+ }
13
+
14
+ class NutNativeWindowHandle implements NativeWindowHandle {
15
+ constructor(private readonly window: NutWindow) {}
16
+
17
+ async info(): Promise<NativeWindowInfo> {
18
+ const [title, region] = await Promise.all([
19
+ Promise.resolve(this.window.title),
20
+ Promise.resolve(this.window.region)
21
+ ]);
22
+ return {
23
+ title,
24
+ region: toRegion(region)
25
+ };
26
+ }
27
+
28
+ async focus(): Promise<void> {
29
+ await this.window.focus();
30
+ }
31
+
32
+ async minimize(): Promise<void> {
33
+ await this.window.minimize();
34
+ }
35
+
36
+ async restore(): Promise<void> {
37
+ await this.window.restore();
38
+ }
39
+
40
+ async move(point: NativePoint): Promise<void> {
41
+ await this.window.move({ x: point.x, y: point.y });
42
+ }
43
+
44
+ async resize(size: NativeSize): Promise<void> {
45
+ await this.window.resize(size);
46
+ }
47
+ }
48
+
49
+ export class NutNativeWindowDriver implements NativeWindowDriver {
50
+ async list(): Promise<NativeWindowHandle[]> {
51
+ const nut = await loadNut();
52
+ const windows = await nut.getWindows();
53
+ return windows.map(window => new NutNativeWindowHandle(window));
54
+ }
55
+
56
+ async active(): Promise<NativeWindowHandle | null> {
57
+ const nut = await loadNut();
58
+ const window = await nut.getActiveWindow();
59
+ return new NutNativeWindowHandle(window);
60
+ }
61
+ }
@@ -0,0 +1,86 @@
1
+ import { NativeDependencyError } from '../errors.js';
2
+
3
+ type DynamicImport = (specifier: string) => Promise<unknown>;
4
+
5
+ const dynamicImport = new Function('specifier', 'return import(specifier)') as DynamicImport;
6
+
7
+ export interface NutPoint {
8
+ x: number;
9
+ y: number;
10
+ }
11
+
12
+ export interface NutRegion {
13
+ left?: number;
14
+ top?: number;
15
+ x?: number;
16
+ y?: number;
17
+ width: number;
18
+ height: number;
19
+ }
20
+
21
+ export interface NutWindow {
22
+ title: Promise<string> | string;
23
+ region: Promise<NutRegion> | NutRegion;
24
+ focus(): Promise<void>;
25
+ minimize(): Promise<void>;
26
+ restore(): Promise<void>;
27
+ move(point: NutPoint): Promise<void>;
28
+ resize(size: { width: number; height: number }): Promise<void>;
29
+ }
30
+
31
+ export interface NutModule {
32
+ mouse: {
33
+ getPosition(): Promise<NutPoint>;
34
+ move(path: unknown): Promise<void>;
35
+ click(button: unknown): Promise<void>;
36
+ };
37
+ keyboard: {
38
+ type(text: string): Promise<void>;
39
+ pressKey(...keys: unknown[]): Promise<void>;
40
+ releaseKey?(...keys: unknown[]): Promise<void>;
41
+ };
42
+ screen: {
43
+ width(): Promise<number>;
44
+ height(): Promise<number>;
45
+ find?(resource: unknown, options?: unknown): Promise<NutRegion>;
46
+ waitFor?(resource: unknown, timeoutMs?: number, intervalMs?: number): Promise<NutRegion>;
47
+ config?: {
48
+ confidence?: number;
49
+ resourceDirectory?: string;
50
+ };
51
+ };
52
+ getWindows(): Promise<NutWindow[]>;
53
+ getActiveWindow(): Promise<NutWindow>;
54
+ straightTo(point: NutPoint): unknown;
55
+ imageResource?(path: string): unknown;
56
+ Button?: Record<string, unknown>;
57
+ Key?: Record<string, unknown>;
58
+ }
59
+
60
+ export interface NlMatcherModule {
61
+ useNlMatcher(): void;
62
+ }
63
+
64
+ let nutModulePromise: Promise<NutModule> | null = null;
65
+ let nlMatcherLoaded = false;
66
+
67
+ export async function loadNut(): Promise<NutModule> {
68
+ nutModulePromise ??= dynamicImport('@nut-tree/nut-js')
69
+ .then(module => module as NutModule)
70
+ .catch(error => {
71
+ nutModulePromise = null;
72
+ throw new NativeDependencyError('@nut-tree/nut-js', error);
73
+ });
74
+ return nutModulePromise;
75
+ }
76
+
77
+ export async function loadNlMatcher(): Promise<void> {
78
+ if (nlMatcherLoaded) return;
79
+ const module = await dynamicImport('@nut-tree/nl-matcher')
80
+ .then(value => value as NlMatcherModule)
81
+ .catch(error => {
82
+ throw new NativeDependencyError('@nut-tree/nl-matcher', error);
83
+ });
84
+ module.useNlMatcher();
85
+ nlMatcherLoaded = true;
86
+ }