@epsilon-asi/actors 0.0.22 → 0.0.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/browser/RuntimeConfig.d.ts +26 -0
  2. package/dist/browser/RuntimeConfig.d.ts.map +1 -1
  3. package/dist/browser/RuntimeConfig.js +29 -1
  4. package/dist/browser/RuntimeConfig.js.map +1 -1
  5. package/dist/core/ActorContext.d.ts +2 -0
  6. package/dist/core/ActorContext.d.ts.map +1 -1
  7. package/dist/core/ActorRunner.d.ts +3 -0
  8. package/dist/core/ActorRunner.d.ts.map +1 -1
  9. package/dist/core/ActorRunner.js +11 -1
  10. package/dist/core/ActorRunner.js.map +1 -1
  11. package/dist/index.d.ts +2 -0
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +2 -0
  14. package/dist/index.js.map +1 -1
  15. package/dist/native/CompositeNativeWindowDriver.d.ts +11 -0
  16. package/dist/native/CompositeNativeWindowDriver.d.ts.map +1 -0
  17. package/dist/native/CompositeNativeWindowDriver.js +31 -0
  18. package/dist/native/CompositeNativeWindowDriver.js.map +1 -0
  19. package/dist/native/NativeActionRegistry.d.ts +14 -0
  20. package/dist/native/NativeActionRegistry.d.ts.map +1 -0
  21. package/dist/native/NativeActionRegistry.js +101 -0
  22. package/dist/native/NativeActionRegistry.js.map +1 -0
  23. package/dist/native/NativeAutomation.d.ts +3 -0
  24. package/dist/native/NativeAutomation.d.ts.map +1 -0
  25. package/dist/native/NativeAutomation.js +12 -0
  26. package/dist/native/NativeAutomation.js.map +1 -0
  27. package/dist/native/NativeCoordinateMapper.d.ts +23 -0
  28. package/dist/native/NativeCoordinateMapper.d.ts.map +1 -0
  29. package/dist/native/NativeCoordinateMapper.js +201 -0
  30. package/dist/native/NativeCoordinateMapper.js.map +1 -0
  31. package/dist/native/NativeFileDialogService.d.ts +26 -0
  32. package/dist/native/NativeFileDialogService.d.ts.map +1 -0
  33. package/dist/native/NativeFileDialogService.js +121 -0
  34. package/dist/native/NativeFileDialogService.js.map +1 -0
  35. package/dist/native/NativeImageFinder.d.ts +12 -0
  36. package/dist/native/NativeImageFinder.d.ts.map +1 -0
  37. package/dist/native/NativeImageFinder.js +29 -0
  38. package/dist/native/NativeImageFinder.js.map +1 -0
  39. package/dist/native/NativeKeyboard.d.ts +10 -0
  40. package/dist/native/NativeKeyboard.d.ts.map +1 -0
  41. package/dist/native/NativeKeyboard.js +16 -0
  42. package/dist/native/NativeKeyboard.js.map +1 -0
  43. package/dist/native/NativeMouse.d.ts +38 -0
  44. package/dist/native/NativeMouse.d.ts.map +1 -0
  45. package/dist/native/NativeMouse.js +82 -0
  46. package/dist/native/NativeMouse.js.map +1 -0
  47. package/dist/native/NativeWindowService.d.ts +31 -0
  48. package/dist/native/NativeWindowService.d.ts.map +1 -0
  49. package/dist/native/NativeWindowService.js +183 -0
  50. package/dist/native/NativeWindowService.js.map +1 -0
  51. package/dist/native/UnsupportedNativeAutomation.d.ts +4 -0
  52. package/dist/native/UnsupportedNativeAutomation.d.ts.map +1 -0
  53. package/dist/native/UnsupportedNativeAutomation.js +77 -0
  54. package/dist/native/UnsupportedNativeAutomation.js.map +1 -0
  55. package/dist/native/WindowMatcher.d.ts +4 -0
  56. package/dist/native/WindowMatcher.d.ts.map +1 -0
  57. package/dist/native/WindowMatcher.js +39 -0
  58. package/dist/native/WindowMatcher.js.map +1 -0
  59. package/dist/native/drivers.d.ts +37 -0
  60. package/dist/native/drivers.d.ts.map +1 -0
  61. package/dist/native/drivers.js +2 -0
  62. package/dist/native/drivers.js.map +1 -0
  63. package/dist/native/errors.d.ts +23 -0
  64. package/dist/native/errors.d.ts.map +1 -0
  65. package/dist/native/errors.js +45 -0
  66. package/dist/native/errors.js.map +1 -0
  67. package/dist/native/index.d.ts +13 -0
  68. package/dist/native/index.d.ts.map +1 -0
  69. package/dist/native/index.js +13 -0
  70. package/dist/native/index.js.map +1 -0
  71. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts +11 -0
  72. package/dist/native/macos/MacOSAccessibilityWindowDriver.d.ts.map +1 -0
  73. package/dist/native/macos/MacOSAccessibilityWindowDriver.js +180 -0
  74. package/dist/native/macos/MacOSAccessibilityWindowDriver.js.map +1 -0
  75. package/dist/native/macos/MacOSAppleScriptClient.d.ts +24 -0
  76. package/dist/native/macos/MacOSAppleScriptClient.d.ts.map +1 -0
  77. package/dist/native/macos/MacOSAppleScriptClient.js +163 -0
  78. package/dist/native/macos/MacOSAppleScriptClient.js.map +1 -0
  79. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts +10 -0
  80. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.d.ts.map +1 -0
  81. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js +12 -0
  82. package/dist/native/macos/MacOSFileDialogAccessibilityStrategy.js.map +1 -0
  83. package/dist/native/macos/MacOSNativeAutomation.d.ts +3 -0
  84. package/dist/native/macos/MacOSNativeAutomation.d.ts.map +1 -0
  85. package/dist/native/macos/MacOSNativeAutomation.js +88 -0
  86. package/dist/native/macos/MacOSNativeAutomation.js.map +1 -0
  87. package/dist/native/nut/NutNativeImageFinder.d.ts +17 -0
  88. package/dist/native/nut/NutNativeImageFinder.d.ts.map +1 -0
  89. package/dist/native/nut/NutNativeImageFinder.js +84 -0
  90. package/dist/native/nut/NutNativeImageFinder.js.map +1 -0
  91. package/dist/native/nut/NutNativeKeyboardDriver.d.ts +8 -0
  92. package/dist/native/nut/NutNativeKeyboardDriver.d.ts.map +1 -0
  93. package/dist/native/nut/NutNativeKeyboardDriver.js +39 -0
  94. package/dist/native/nut/NutNativeKeyboardDriver.js.map +1 -0
  95. package/dist/native/nut/NutNativeMouseDriver.d.ts +8 -0
  96. package/dist/native/nut/NutNativeMouseDriver.d.ts.map +1 -0
  97. package/dist/native/nut/NutNativeMouseDriver.js +24 -0
  98. package/dist/native/nut/NutNativeMouseDriver.js.map +1 -0
  99. package/dist/native/nut/NutNativeScreenDriver.d.ts +6 -0
  100. package/dist/native/nut/NutNativeScreenDriver.d.ts.map +1 -0
  101. package/dist/native/nut/NutNativeScreenDriver.js +12 -0
  102. package/dist/native/nut/NutNativeScreenDriver.js.map +1 -0
  103. package/dist/native/nut/NutNativeWindowDriver.d.ts +6 -0
  104. package/dist/native/nut/NutNativeWindowDriver.d.ts.map +1 -0
  105. package/dist/native/nut/NutNativeWindowDriver.js +53 -0
  106. package/dist/native/nut/NutNativeWindowDriver.js.map +1 -0
  107. package/dist/native/nut/loadNut.d.ts +58 -0
  108. package/dist/native/nut/loadNut.d.ts.map +1 -0
  109. package/dist/native/nut/loadNut.js +25 -0
  110. package/dist/native/nut/loadNut.js.map +1 -0
  111. package/dist/native/types.d.ts +194 -0
  112. package/dist/native/types.d.ts.map +1 -0
  113. package/dist/native/types.js +2 -0
  114. package/dist/native/types.js.map +1 -0
  115. package/dist/native/utils/appleScriptEscape.d.ts +7 -0
  116. package/dist/native/utils/appleScriptEscape.d.ts.map +1 -0
  117. package/dist/native/utils/appleScriptEscape.js +11 -0
  118. package/dist/native/utils/appleScriptEscape.js.map +1 -0
  119. package/dist/native/utils/geometry.d.ts +12 -0
  120. package/dist/native/utils/geometry.d.ts.map +1 -0
  121. package/dist/native/utils/geometry.js +77 -0
  122. package/dist/native/utils/geometry.js.map +1 -0
  123. package/dist/native/utils/redactNative.d.ts +2 -0
  124. package/dist/native/utils/redactNative.d.ts.map +1 -0
  125. package/dist/native/utils/redactNative.js +7 -0
  126. package/dist/native/utils/redactNative.js.map +1 -0
  127. package/dist/native/utils/waitFor.d.ts +7 -0
  128. package/dist/native/utils/waitFor.d.ts.map +1 -0
  129. package/dist/native/utils/waitFor.js +17 -0
  130. package/dist/native/utils/waitFor.js.map +1 -0
  131. package/dist/sites/upwork-com/upwork-com.actor.d.ts +4 -1
  132. package/dist/sites/upwork-com/upwork-com.actor.d.ts.map +1 -1
  133. package/dist/sites/upwork-com/upwork-com.actor.js +31 -11
  134. package/dist/sites/upwork-com/upwork-com.actor.js.map +1 -1
  135. package/dist/sites/upwork-com/upwork-com.types.d.ts +3 -1
  136. package/dist/sites/upwork-com/upwork-com.types.d.ts.map +1 -1
  137. package/dist/sites/upwork-com/upwork-com.types.js.map +1 -1
  138. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts +70 -0
  139. package/dist/sites/upwork-com/util/parseJobApplicationDetails.d.ts.map +1 -0
  140. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js +334 -0
  141. package/dist/sites/upwork-com/util/parseJobApplicationDetails.js.map +1 -0
  142. package/npmrc +5 -0
  143. package/package.json +5 -1
  144. package/src/browser/RuntimeConfig.ts +57 -1
  145. package/src/core/ActorContext.ts +2 -0
  146. package/src/core/ActorRunner.ts +13 -1
  147. package/src/index.ts +2 -0
  148. package/src/native/CompositeNativeWindowDriver.ts +30 -0
  149. package/src/native/NativeActionRegistry.ts +114 -0
  150. package/src/native/NativeAutomation.ts +15 -0
  151. package/src/native/NativeCoordinateMapper.ts +258 -0
  152. package/src/native/NativeFileDialogService.ts +138 -0
  153. package/src/native/NativeImageFinder.ts +33 -0
  154. package/src/native/NativeKeyboard.ts +18 -0
  155. package/src/native/NativeMouse.ts +116 -0
  156. package/src/native/NativeWindowService.ts +229 -0
  157. package/src/native/UnsupportedNativeAutomation.ts +92 -0
  158. package/src/native/WindowMatcher.ts +31 -0
  159. package/src/native/drivers.ts +38 -0
  160. package/src/native/errors.ts +51 -0
  161. package/src/native/index.ts +12 -0
  162. package/src/native/macos/MacOSAccessibilityWindowDriver.ts +183 -0
  163. package/src/native/macos/MacOSAppleScriptClient.ts +182 -0
  164. package/src/native/macos/MacOSFileDialogAccessibilityStrategy.ts +11 -0
  165. package/src/native/macos/MacOSNativeAutomation.ts +86 -0
  166. package/src/native/nut/NutNativeImageFinder.ts +98 -0
  167. package/src/native/nut/NutNativeKeyboardDriver.ts +38 -0
  168. package/src/native/nut/NutNativeMouseDriver.ts +27 -0
  169. package/src/native/nut/NutNativeScreenDriver.ts +14 -0
  170. package/src/native/nut/NutNativeWindowDriver.ts +61 -0
  171. package/src/native/nut/loadNut.ts +86 -0
  172. package/src/native/types.ts +224 -0
  173. package/src/native/utils/appleScriptEscape.ts +11 -0
  174. package/src/native/utils/geometry.ts +88 -0
  175. package/src/native/utils/redactNative.ts +6 -0
  176. package/src/native/utils/waitFor.ts +25 -0
  177. package/src/sites/upwork-com/upwork-com.actor.ts +47 -14
  178. package/src/sites/upwork-com/upwork-com.types.ts +4 -1
  179. package/src/sites/upwork-com/util/parseJobApplicationDetails.ts +622 -0
  180. package/tests/fixtures/makeContext.ts +7 -2
  181. package/tests/fixtures/native/FakeNativeAutomation.ts +138 -0
  182. package/tests/unit/browser/RuntimeConfig.native.test.ts +63 -0
  183. package/tests/unit/core/ActorRunner.native.test.ts +69 -0
  184. package/tests/unit/native/MacOSAppleScriptClient.test.ts +35 -0
  185. package/tests/unit/native/NativeActionRegistry.test.ts +34 -0
  186. package/tests/unit/native/NativeCoordinateMapper.test.ts +92 -0
  187. package/tests/unit/native/NativeFileDialogService.test.ts +91 -0
  188. package/tests/unit/native/NativeMouse.test.ts +91 -0
  189. package/tests/unit/native/NativeWindowService.test.ts +87 -0
  190. package/tests/unit/native/WindowMatcher.test.ts +32 -0
  191. package/tests/unit/native/appleScriptEscape.test.ts +9 -0
  192. package/tests/unit/sites/myvistage-com.login.test.ts +1 -1
  193. package/tests/unit/sites/myvistage-com.postComment.test.ts +0 -1
  194. package/tests/unit/sites/upwork-com.login.test.ts +1 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parseJobApplicationDetails.js","sourceRoot":"","sources":["../../../../src/sites/upwork-com/util/parseJobApplicationDetails.ts"],"names":[],"mappings":"AAyFA;;;;;;uGAMuG;AAEvG,SAAS,+BAA+B,CACpC,QAA2B,EAC3B,SAAwB;IAExB,MAAM,IAAI,GAAe,QAAQ,IAAI,QAAQ,CAAC;IAE9C,MAAM,SAAS,GAAG,CAAC,KAAqB,EAAU,EAAE;QAChD,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;aACf,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,KAAqB,EAAiB,EAAE;QACtD,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3C,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE;QACtC,MAAM,IAAI,GAAG,KAAK;aACb,WAAW,EAAE;aACb,SAAS,CAAC,MAAM,CAAC;aACjB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;aACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;aACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAElB,OAAO,IAAI,IAAI,UAAU,CAAC;IAC9B,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE;QACxC,MAAM,GAAG,GAAI,UAA0E,CAAC,GAAG,CAAC;QAE5F,IAAI,GAAG,EAAE,MAAM,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC,CAAC;IAEF,MAAM,iBAAiB,GAAG,CACtB,WAAmB,EACD,EAAE;QACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACvB,IAAI,CAAC,gBAAgB,CAAc,oBAAoB,CAAC,CAC3D,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CACzB,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,KAAK,WAAW,CAAC,WAAW,EAAE,CAChF,CAAC;QAEF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,CACH,OAAO,CAAC,OAAO,CAAc,iCAAiC,CAAC;YAC/D,OAAO,CAAC,OAAO,CAAc,qBAAqB,CAAC;YACnD,OAAO,CAAC,OAAO,CAAc,YAAY,CAAC;YAC1C,OAAO,CAAC,aAAa,CACxB,CAAC;IACN,CAAC,CAAC;IAEF,MAAM,iBAAiB,GACnB,IAAI,CAAC,aAAa,CAAc,iCAAiC,CAAC;QAClE,IAAI,CAAC,aAAa,CAAc,qBAAqB,CAAC;QACtD,iBAAiB,CAAC,oBAAoB,CAAC,CAAC;IAE5C,MAAM,2BAA2B,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE/D,MAAM,KAAK,GAAG,iBAAiB,IAAI,IAAI,CAAC;IAExC,MAAM,gBAAgB,GAAG,CAAC,SAAkB,EAAY,EAAE;QACtD,OAAO,KAAK,CAAC,IAAI,CACb,SAAS,CAAC,gBAAgB,CACtB,8DAA8D,CACjE,CACJ;aACI,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;aACtC,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC;IACzE,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CACnB,OAAuB,EACE,EAAE;QAC3B,IAAI,CAAC,OAAO;YAAE,OAAO,SAAS,CAAC;QAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAE1C,IAAI,GAAG,KAAK,UAAU;YAAE,OAAO,UAAU,CAAC;QAC1C,IAAI,GAAG,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QACtC,IAAI,GAAG,KAAK,OAAO;YAAE,OAAO,OAAO,CAAC;QAEpC,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,CAAC,OAAuB,EAAU,EAAE;QACxD,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,IACI,OAAO,YAAY,mBAAmB;YACtC,OAAO,YAAY,gBAAgB;YACnC,OAAO,YAAY,iBAAiB,EACtC,CAAC;YACC,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,CAAC,OAAuB,EAAiB,EAAE;QAC5D,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAEhD,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACnD,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,OAAuB,EAAiB,EAAE;QACvD,IAAI,CAAC,CAAC,OAAO,YAAY,mBAAmB,CAAC,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,CAClB,IAAgC,EAChC,SAAkB,EAClB,OAAuB,EACvB,MAAgB,EACF,EAAE;QAChB,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,OAAO,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAEjD,IAAI,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1E,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,CAClB,MAKC,EAC4B,EAAE;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAAc,oBAAoB,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAG1C;YACI,UAAU;YACV,+CAA+C;YAC/C,QAAQ;SACX,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAElD,MAAM,cAAc,GAChB,OAAO,CAAC,YAAY,CAAC,iBAAiB,CAAC;YACvC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC;YAClC,IAAI,CAAC;QAET,MAAM,EAAE,GACJ,MAAM,CAAC,IAAI,KAAK,cAAc;YAC1B,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,mBAAmB,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAEnE,IAAI,iBAAiB,GAAkB,IAAI,CAAC;QAC5C,IAAI,aAAa,GAAkB,IAAI,CAAC;QACxC,IAAI,eAAe,GAAkB,IAAI,CAAC;QAE1C,IAAI,MAAM,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACjC,iBAAiB;gBACb,gEAAgE,CAAC;YAErE,aAAa;gBACT,0DAA0D,CAAC;YAE/D,eAAe;gBACX,6DAA6D,CAAC;QACtE,CAAC;aAAM,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;YAE7B,iBAAiB;gBACb,GAAG,MAAM,CAAC,YAAY,4BAA4B,GAAG,GAAG,CAAC;YAE7D,aAAa;gBACT,GAAG,iBAAiB,QAAQ,CAAC;YAEjC,eAAe;gBACX,GAAG,iBAAiB,IAAI,UAAU,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACjC,eAAe,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QAClD,CAAC;QAED,OAAO;YACH,EAAE;YAEF,IAAI,EAAE,MAAM,CAAC,IAAI;YAEjB,MAAM;YAEN,KAAK,EAAE,MAAM,CAAC,KAAK;YAEnB,WAAW;YAEX,SAAS,EAAE;gBACP,SAAS,EAAE,iBAAiB;gBAC5B,KAAK,EAAE,aAAa;gBACpB,OAAO,EAAE,eAAe;aAC3B;YAED,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;YAEvE,WAAW,EAAE,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,IAAI;YAExD,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC;YAEtC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC;YAEtB,SAAS,EAAE,YAAY,CAAC,OAAO,CAAC;YAEhC,cAAc;YAEd,aAAa,EAAE,MAAM;SACxB,CAAC;IACN,CAAC,CAAC;IAEF;;uGAEmG;IAGnG,MAAM,oBAAoB,GACtB,KAAK,CAAC,aAAa,CAAc,gCAAgC,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAc,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACvE,SAAS,CAAC,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CACrF;QACD,IAAI,CAAC;IAGT,MAAM,WAAW,GAAG,oBAAoB;QACpC,CAAC,CAAC,aAAa,CAAC;YACZ,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,oBAAoB;YAC/B,YAAY,EAAE,oDAAoD;SACrE,CAAC;QACF,CAAC,CAAC,IAAI,CAAC;IAEX;;uGAEmG;IAGnG,MAAM,kBAAkB,GACpB,KAAK,CAAC,aAAa,CAAc,2CAA2C,CAAC;QAC7E,KAAK,CAAC,aAAa,CAAc,iBAAiB,CAAC,CAAC;IAGxD,MAAM,wBAAwB,GAAG,kBAAkB;QAC/C,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAc,aAAa,CAAC,CAAC;aACxE,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACX,OAAO,CACH,EAAE,CAAC,aAAa,CACZ;YACI,UAAU;YACV,+CAA+C;YAC/C,QAAQ;SACX,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CACJ,CACJ;aACA,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YACX,MAAM,SAAS,GAAG,SAAS,CAAC,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElF,OAAO,CACH,SAAS,KAAK,cAAc;gBAC5B,SAAS,KAAK,aAAa,CAC9B,CAAC;QACN,CAAC,CAAC;QACN,CAAC,CAAC,EAAE,CAAC;IAGT,MAAM,eAAe,GAAG,wBAAwB;SAC3C,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CACtB,aAAa,CAAC;QACV,IAAI,EAAE,iBAAiB;QACvB,KAAK;QACL,SAAS;QACT,YAAY,EACR,2EAA2E;KAClF,CAAC,CACL;SACA,MAAM,CAAC,CAAC,QAAQ,EAAsC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjF;;uGAEmG;IAGnG,MAAM,eAAe,GACjB,KAAK,CAAC,aAAa,CAAc,mBAAmB,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAc,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACvE,SAAS,CAAC,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC,WAAW,EAAE,KAAK,aAAa,CACpF;QACD,IAAI,CAAC;IAGT,MAAM,eAAe,GAAG,SAAS,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAGhE,MAAM,SAAS,GACX,eAAe,EAAE,aAAa,CAAmB,oBAAoB,CAAC;QACtE,IAAI,CAAC;IAGT,MAAM,iBAAiB,GACnB,eAAe,EAAE,aAAa,CAAc,gBAAgB,CAAC;QAC7D,IAAI,CAAC;IAGT,MAAM,kBAAkB,GAAG,SAAS,CAChC,iBAAiB,EAAE,YAAY,CAAC,cAAc,CAAC,IAAI,EAAE,CACxD;SACI,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;SAC/C,MAAM,CAAC,OAAO,CAAC,CAAC;IAErB,MAAM,YAAY,GAAG;QACjB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,GAAG,eAAe;KACrB,CAAC;IAEF,OAAO;QACH,2BAA2B;QAE3B,SAAS;QAET,WAAW;QAEX,eAAe;QAEf,YAAY;QAIZ,MAAM,EAAE;YACJ,KAAK,EAAE,YAAY,CAAC,MAAM;YAC1B,MAAM,EAAE,eAAe,CAAC,MAAM;YAC9B,cAAc,EAAE,OAAO,CAAC,WAAW,CAAC;SACvC;KACJ,CAAC;AACN,CAAC;AAED;;uGAEuG;AAEvG,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAC9C,IAAU,EACV,OAEC;IAED,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,MAAM,CAAC;IAE/C,MAAM,IAAI;SACL,eAAe,CACZ;QACI,iCAAiC;QACjC,qBAAqB;QACrB,oBAAoB;QACpB,iBAAiB;KACpB,CAAC,IAAI,CAAC,IAAI,CAAC,EACZ;QACI,OAAO,EAAE,SAAS;KACrB,CACJ;SACA,KAAK,CAAC,GAAG,EAAE;QACR,4EAA4E;QAC5E,qEAAqE;IACzE,CAAC,CAAC,CAAC;IAEP,OAAO,IAAI,CAAC,QAAQ,CAChB,+BAA+B,EAC/B,IAAI,EACJ,IAAI,CAAC,GAAG,EAAE,CACb,CAAC;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oCAAoC,CACtD,IAA4B,EAC5B,YAA2B,IAAI;IAE/B,OAAO,IAAI,CAAC,QAAQ,CAChB,+BAA+B,EAC/B,SAAS,CACZ,CAAC;AACN,CAAC;AAiBD,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC7C,IAAU,EACV,OAAsC;IAEtC,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;IAEpD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC5C,MAAM,MAAM,GACR,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEnC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACzC,SAAS;QACb,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;AACL,CAAC;AAED;;uGAEuG;AAEvG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgCE"}
package/npmrc ADDED
@@ -0,0 +1,5 @@
1
+
2
+ //registry.npmjs.org/:_authToken=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
3
+ @epsilon-asi:registry=https://registry.npmjs.org/
4
+ @nut-tree:registry=https://pkg.nutjs.dev/
5
+ //pkg.nutjs.dev/:_authToken=ffcb6d37-3cab-43e8-bfce-680de3cf5b95
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@epsilon-asi/actors",
3
- "version": "0.0.22",
3
+ "version": "0.0.33",
4
4
  "description": "A TypeScript Puppeteer actor framework using existing Chrome profiles and ghost-cursor.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -27,6 +27,10 @@
27
27
  "clean": "rm -rf dist coverage"
28
28
  },
29
29
  "dependencies": {
30
+ "@nut-tree-fork/nut-js": "^4.2.6",
31
+ "@nut-tree/bolt": "^3.1.1",
32
+ "@nut-tree/nl-matcher": "^5.0.1",
33
+ "@nut-tree/nut-js": "^5.1.2",
30
34
  "cheerio": "^1.2.0",
31
35
  "ghost-cursor": "^1.4.2",
32
36
  "puppeteer-core": "24.43.1"
@@ -77,6 +77,32 @@ export interface RemoteDebuggingChromeConfig extends BrowserSessionBehavior, Rem
77
77
 
78
78
  export type BrowserRuntimeConfig = ExistingChromeProfileConfig | RemoteDebuggingChromeConfig;
79
79
 
80
+ export interface NativeRuntimeConfig {
81
+ enabled?: boolean;
82
+ platform?: 'macos';
83
+ templatesDir?: string;
84
+ imageConfidence?: number;
85
+ mouse?: {
86
+ jitterPixels?: number;
87
+ minJitterPixels?: number;
88
+ moveSpeed?: number;
89
+ randomSeed?: number;
90
+ };
91
+ macos?: {
92
+ targetApplicationName?: string;
93
+ targetBundleId?: string;
94
+ osascriptTimeoutMs?: number;
95
+ preferAppleScriptForWindows?: boolean;
96
+ preferVisualTemplatesForDialogs?: boolean;
97
+ };
98
+ coordinateCalibration?: {
99
+ enabled?: boolean;
100
+ markerSizePx?: number;
101
+ timeoutMs?: number;
102
+ cacheTtlMs?: number;
103
+ };
104
+ }
105
+
80
106
  export interface RuntimeConfig {
81
107
  browser: BrowserRuntimeConfig;
82
108
  defaultTimeoutMs?: number;
@@ -84,6 +110,7 @@ export interface RuntimeConfig {
84
110
  interaction?: {
85
111
  typing?: HumanTypingOptions;
86
112
  };
113
+ native?: NativeRuntimeConfig;
87
114
  }
88
115
 
89
116
  export function expandHome(input: string): string {
@@ -131,6 +158,34 @@ function normalizeExistingProfileConfig(browser: ExistingChromeProfileConfig): E
131
158
  };
132
159
  }
133
160
 
161
+ function normalizeNativeRuntimeConfig(config: NativeRuntimeConfig): NativeRuntimeConfig {
162
+ return {
163
+ ...config,
164
+ enabled: config.enabled ?? true,
165
+ platform: config.platform ?? 'macos',
166
+ imageConfidence: config.imageConfidence ?? 0.9,
167
+ mouse: {
168
+ ...config.mouse,
169
+ jitterPixels: config.mouse?.jitterPixels ?? 3,
170
+ minJitterPixels: config.mouse?.minJitterPixels ?? 2
171
+ },
172
+ macos: {
173
+ ...config.macos,
174
+ targetApplicationName: config.macos?.targetApplicationName ?? 'Google Chrome',
175
+ osascriptTimeoutMs: config.macos?.osascriptTimeoutMs ?? 15_000,
176
+ preferAppleScriptForWindows: config.macos?.preferAppleScriptForWindows ?? true,
177
+ preferVisualTemplatesForDialogs: config.macos?.preferVisualTemplatesForDialogs ?? false
178
+ },
179
+ coordinateCalibration: {
180
+ ...config.coordinateCalibration,
181
+ enabled: config.coordinateCalibration?.enabled ?? true,
182
+ markerSizePx: config.coordinateCalibration?.markerSizePx ?? 22,
183
+ timeoutMs: config.coordinateCalibration?.timeoutMs ?? 15_000,
184
+ cacheTtlMs: config.coordinateCalibration?.cacheTtlMs ?? 60_000
185
+ }
186
+ };
187
+ }
188
+
134
189
  function normalizeRemoteDebuggingConfig(browser: RemoteDebuggingChromeConfig): RemoteDebuggingChromeConfig {
135
190
  return {
136
191
  ...browser,
@@ -151,6 +206,7 @@ export function normalizeRuntimeConfig(config: RuntimeConfig): RuntimeConfig {
151
206
  : normalizeRemoteDebuggingConfig(config.browser),
152
207
  defaultTimeoutMs: config.defaultTimeoutMs ?? 15_000,
153
208
  navigationTimeoutMs: config.navigationTimeoutMs ?? 30_000,
154
- ...(config.interaction !== undefined ? { interaction: config.interaction } : {})
209
+ ...(config.interaction !== undefined ? { interaction: config.interaction } : {}),
210
+ ...(config.native !== undefined ? { native: normalizeNativeRuntimeConfig(config.native) } : {})
155
211
  };
156
212
  }
@@ -8,6 +8,7 @@ import type { HumanInteractor } from '../interaction/HumanInteractor.js';
8
8
  import type { Navigator } from '../interaction/Navigation.js';
9
9
  import type { PageAdapter } from '../interaction/PageAdapter.js';
10
10
  import type { Logger } from '../logging/Logger.js';
11
+ import type { NativeAutomation } from '../native/types.js';
11
12
 
12
13
  export interface ActorMetadata {
13
14
  id: string;
@@ -31,4 +32,5 @@ export interface ActorContext {
31
32
  pagination: Pagination;
32
33
  auth: AuthController;
33
34
  logger: Logger;
35
+ native: NativeAutomation;
34
36
  }
@@ -16,6 +16,8 @@ import type { HumanInteractor } from '../interaction/HumanInteractor.js';
16
16
  import { Navigator } from '../interaction/Navigation.js';
17
17
  import { PuppeteerPageAdapter, type PageAdapter } from '../interaction/PageAdapter.js';
18
18
  import type { Logger } from '../logging/Logger.js';
19
+ import { createNativeAutomation } from '../native/NativeAutomation.js';
20
+ import type { NativeAutomation, NativeAutomationFactory } from '../native/types.js';
19
21
  import { ConsoleLogger } from '../logging/ConsoleLogger.js';
20
22
  import type { Actor, ActorTaskMap } from './Actor.js';
21
23
  import type { ActorContext, AuthController } from './ActorContext.js';
@@ -28,6 +30,7 @@ export interface ActorRunnerOptions {
28
30
  logger?: Logger;
29
31
  interactorFactory?: (page: PageLike) => HumanInteractor;
30
32
  pageAdapterFactory?: (page: PageLike) => PageAdapter;
33
+ nativeAutomationFactory?: NativeAutomationFactory;
31
34
  loginFlow?: LoginFlow;
32
35
  authStateDetector?: AuthStateDetector;
33
36
  registry?: ActorRegistry;
@@ -52,6 +55,7 @@ export class ActorRunner {
52
55
  private readonly logger: Logger;
53
56
  private readonly interactorFactory: (page: PageLike) => HumanInteractor;
54
57
  private readonly pageAdapterFactory: (page: PageLike) => PageAdapter;
58
+ private readonly nativeAutomationFactory: NativeAutomationFactory;
55
59
  private readonly loginFlow: LoginFlow;
56
60
  private readonly authStateDetector: AuthStateDetector;
57
61
  protected readonly registry: ActorRegistry | undefined;
@@ -67,6 +71,7 @@ export class ActorRunner {
67
71
  options.config.interaction?.typing === undefined ? {} : { typing: options.config.interaction.typing }
68
72
  ));
69
73
  this.pageAdapterFactory = options.pageAdapterFactory ?? (page => new PuppeteerPageAdapter(page));
74
+ this.nativeAutomationFactory = options.nativeAutomationFactory ?? createNativeAutomation;
70
75
  this.authStateDetector = options.authStateDetector ?? new AuthStateDetector();
71
76
  this.loginFlow = options.loginFlow ?? new LoginFlow(this.credentialsProvider, this.authStateDetector);
72
77
  this.registry = options.registry;
@@ -145,6 +150,12 @@ export class ActorRunner {
145
150
  const nav = new Navigator(page, cursor, actor.baseUrl);
146
151
  const extract = new Extractor(page);
147
152
  const pagination = new Pagination(page, cursor);
153
+ const native: NativeAutomation = this.nativeAutomationFactory({
154
+ session,
155
+ page: session.page,
156
+ config: this.options.config,
157
+ logger: this.logger
158
+ });
148
159
 
149
160
  let context!: ActorContext;
150
161
 
@@ -176,7 +187,8 @@ export class ActorRunner {
176
187
  extract,
177
188
  pagination,
178
189
  auth,
179
- logger: this.logger
190
+ logger: this.logger,
191
+ native
180
192
  };
181
193
 
182
194
  return context;
package/src/index.ts CHANGED
@@ -5,5 +5,7 @@ export * from './errors/index.js';
5
5
  export * from './extraction/index.js';
6
6
  export * from './interaction/index.js';
7
7
  export * from './logging/index.js';
8
+ export * from './native/index.js';
9
+ export * from './native/index.js'
8
10
  export * from './sites/index.js';
9
11
  export * from './utils/index.js';
@@ -0,0 +1,30 @@
1
+ import type { Logger } from '../logging/Logger.js';
2
+ import type { NativeWindowDriver, NativeWindowHandle } from './drivers.js';
3
+
4
+ export class FallbackNativeWindowDriver implements NativeWindowDriver {
5
+ constructor(
6
+ private readonly primary: NativeWindowDriver,
7
+ private readonly fallback: NativeWindowDriver,
8
+ private readonly logger?: Logger
9
+ ) {}
10
+
11
+ async list(): Promise<NativeWindowHandle[]> {
12
+ try {
13
+ return await this.primary.list();
14
+ } catch (error) {
15
+ this.logger?.debug('Primary native window driver failed; using fallback driver.', { error });
16
+ return this.fallback.list();
17
+ }
18
+ }
19
+
20
+ async active(): Promise<NativeWindowHandle | null> {
21
+ if (this.primary.active !== undefined) {
22
+ try {
23
+ return await this.primary.active();
24
+ } catch (error) {
25
+ this.logger?.debug('Primary native active-window lookup failed; using fallback driver.', { error });
26
+ }
27
+ }
28
+ return this.fallback.active === undefined ? null : this.fallback.active();
29
+ }
30
+ }
@@ -0,0 +1,114 @@
1
+ import type { Logger } from '../logging/Logger.js';
2
+ import type {
3
+ NativeAction,
4
+ NativeActionContext,
5
+ NativeActionMetadata,
6
+ NativeActionRegistry as NativeActionRegistryInterface,
7
+ NativeAutomation
8
+ } from './types.js';
9
+ import { NativeAutomationError } from './errors.js';
10
+
11
+ export function defineNativeAction<TInput, TOutput>(definition: NativeAction<TInput, TOutput>): NativeAction<TInput, TOutput> {
12
+ return definition;
13
+ }
14
+
15
+ export class DefaultNativeActionRegistry implements NativeActionRegistryInterface {
16
+ private readonly actions = new Map<string, NativeAction<unknown, unknown>>();
17
+
18
+ constructor(private readonly contextFactory: () => NativeActionContext) {}
19
+
20
+ register<TInput, TOutput>(action: NativeAction<TInput, TOutput>): this {
21
+ if (this.actions.has(action.name)) {
22
+ throw new NativeAutomationError(`Native action already registered: ${action.name}.`, { actionName: action.name });
23
+ }
24
+ this.actions.set(action.name, action as NativeAction<unknown, unknown>);
25
+ return this;
26
+ }
27
+
28
+ get<TInput = unknown, TOutput = unknown>(name: string): NativeAction<TInput, TOutput> {
29
+ const action = this.actions.get(name);
30
+ if (action === undefined) {
31
+ throw new NativeAutomationError(`Native action is not registered: ${name}.`, { actionName: name });
32
+ }
33
+ return action as NativeAction<TInput, TOutput>;
34
+ }
35
+
36
+ async run<TInput = unknown, TOutput = unknown>(name: string, input: TInput): Promise<TOutput> {
37
+ const action = this.get<TInput, TOutput>(name);
38
+ return action.run(this.contextFactory(), input);
39
+ }
40
+
41
+ list(): NativeActionMetadata[] {
42
+ return Array.from(this.actions.values()).map(action => {
43
+ const metadata: NativeActionMetadata = { name: action.name };
44
+ if (action.description !== undefined) metadata.description = action.description;
45
+ return metadata;
46
+ });
47
+ }
48
+ }
49
+
50
+ export function registerDefaultNativeActions(registry: DefaultNativeActionRegistry, automation: NativeAutomation, logger: Logger): DefaultNativeActionRegistry {
51
+ const actionContext = (): NativeActionContext => ({ automation, logger });
52
+ void actionContext;
53
+
54
+ registry
55
+ .register(defineNativeAction({
56
+ name: 'window.isOpen',
57
+ description: 'Return whether a native window matching the given matcher is open.',
58
+ run: async (context, input) => context.automation.windows.isOpen(input as never)
59
+ }))
60
+ .register(defineNativeAction({
61
+ name: 'window.focus',
62
+ description: 'Focus a native window matching the given matcher.',
63
+ run: async (context, input) => context.automation.windows.focus(input as never)
64
+ }))
65
+ .register(defineNativeAction({
66
+ name: 'window.ensureInView',
67
+ description: 'Restore, focus, and ensure a native window is visible.',
68
+ run: async (context, input) => context.automation.windows.ensureInView(input as never)
69
+ }))
70
+ .register(defineNativeAction({
71
+ name: 'window.setFullscreen',
72
+ description: 'Set a native window fullscreen state.',
73
+ run: async (context, input) => {
74
+ const value = input as { matcher: never; fullscreen: boolean };
75
+ return context.automation.windows.setFullscreen(value.matcher, value.fullscreen);
76
+ }
77
+ }))
78
+ .register(defineNativeAction({
79
+ name: 'window.toggleFullscreen',
80
+ description: 'Toggle a native window fullscreen state.',
81
+ run: async (context, input) => context.automation.windows.toggleFullscreen(input as never)
82
+ }))
83
+ .register(defineNativeAction({
84
+ name: 'window.minimize',
85
+ description: 'Minimize a native window.',
86
+ run: async (context, input) => context.automation.windows.minimize(input as never)
87
+ }))
88
+ .register(defineNativeAction({
89
+ name: 'fileDialog.selectFile',
90
+ description: 'Select a file in the active native open/upload dialog.',
91
+ run: async (context, input) => {
92
+ const value = input as { filePath: string; options?: never };
93
+ await context.automation.fileDialogs.selectFile(value.filePath, value.options);
94
+ }
95
+ }))
96
+ .register(defineNativeAction({
97
+ name: 'coordinates.calibrate',
98
+ description: 'Calibrate browser viewport coordinates to native screen coordinates.',
99
+ run: async (context, input) => {
100
+ const value = input as { page: never; options?: never };
101
+ return context.automation.coordinates.calibrate(value.page, value.options);
102
+ }
103
+ }))
104
+ .register(defineNativeAction({
105
+ name: 'mouse.clickBrowserPoint',
106
+ description: 'Click a browser viewport point through native OS mouse automation.',
107
+ run: async (context, input) => {
108
+ const value = input as { page: never; point: never; options?: never };
109
+ await context.automation.mouse.clickBrowserPoint(value.page, value.point, value.options);
110
+ }
111
+ }));
112
+
113
+ return registry;
114
+ }
@@ -0,0 +1,15 @@
1
+ import type { NativeAutomation, NativeAutomationFactory, NativeAutomationFactoryArgs } from './types.js';
2
+ import { createUnsupportedNativeAutomation } from './UnsupportedNativeAutomation.js';
3
+ import { createMacOSNativeAutomation } from './macos/MacOSNativeAutomation.js';
4
+
5
+ export const createNativeAutomation: NativeAutomationFactory = (args: NativeAutomationFactoryArgs): NativeAutomation => {
6
+ if (args.config.native?.enabled === false) {
7
+ return createUnsupportedNativeAutomation(process.platform, args.logger);
8
+ }
9
+
10
+ if (process.platform === 'darwin') {
11
+ return createMacOSNativeAutomation(args);
12
+ }
13
+
14
+ return createUnsupportedNativeAutomation(process.platform, args.logger);
15
+ };
@@ -0,0 +1,258 @@
1
+ import type { PageLike } from '../browser/PuppeteerLike.js';
2
+ import type { Logger } from '../logging/Logger.js';
3
+ import type {
4
+ BrowserPoint,
5
+ BrowserViewportSnapshot,
6
+ CalibrationOptions,
7
+ CoordinateMapOptions,
8
+ CoordinateTransform,
9
+ NativeCoordinateMapper,
10
+ NativeImageFinder,
11
+ NativeTemplateRef,
12
+ NativeWindowService,
13
+ ScreenPoint
14
+ } from './types.js';
15
+ import { CoordinateCalibrationError } from './errors.js';
16
+ import { centerOf } from './utils/geometry.js';
17
+
18
+ interface MarkerSpec {
19
+ id: string;
20
+ point: BrowserPoint;
21
+ size: number;
22
+ color: string;
23
+ }
24
+
25
+ interface MarkerInstallPayload {
26
+ markers: MarkerSpec[];
27
+ }
28
+
29
+ export interface ImageBasedCoordinateMapperOptions {
30
+ images: NativeImageFinder;
31
+ windows?: NativeWindowService;
32
+ logger?: Logger;
33
+ defaultTimeoutMs?: number;
34
+ defaultCacheTtlMs?: number;
35
+ }
36
+
37
+ const DEFAULT_MARKER_A: BrowserPoint = { x: 120, y: 120 };
38
+ const DEFAULT_MARKER_B: BrowserPoint = { x: 420, y: 320 };
39
+ const DEFAULT_MARKER_SIZE = 22;
40
+
41
+ function assertFinite(value: number, name: string): void {
42
+ if (!Number.isFinite(value)) {
43
+ throw new CoordinateCalibrationError(`Coordinate calibration produced a non-finite ${name}.`, { [name]: value });
44
+ }
45
+ }
46
+
47
+ async function getViewportSnapshot(page: PageLike): Promise<BrowserViewportSnapshot> {
48
+ return page.evaluate<BrowserViewportSnapshot>(() => {
49
+ const view = globalThis.window;
50
+ return {
51
+ screenX: view.screenX,
52
+ screenY: view.screenY,
53
+ outerWidth: view.outerWidth,
54
+ outerHeight: view.outerHeight,
55
+ innerWidth: view.innerWidth,
56
+ innerHeight: view.innerHeight,
57
+ devicePixelRatio: view.devicePixelRatio
58
+ };
59
+ });
60
+ }
61
+
62
+ async function installMarkers(page: PageLike, markers: MarkerSpec[]): Promise<void> {
63
+ const payload: MarkerInstallPayload = { markers };
64
+ await page.evaluate<void>((rawPayload: unknown) => {
65
+ const { markers: markerSpecs } = rawPayload as MarkerInstallPayload;
66
+ const existing = document.getElementById('__paf_native_coordinate_markers__');
67
+ existing?.remove();
68
+
69
+ const root = document.createElement('div');
70
+ root.id = '__paf_native_coordinate_markers__';
71
+ root.style.position = 'fixed';
72
+ root.style.left = '0';
73
+ root.style.top = '0';
74
+ root.style.width = '0';
75
+ root.style.height = '0';
76
+ root.style.zIndex = '2147483647';
77
+ root.style.pointerEvents = 'none';
78
+
79
+ for (const marker of markerSpecs) {
80
+ const element = document.createElement('div');
81
+ element.id = marker.id;
82
+ element.setAttribute('data-paf-native-calibration-marker', marker.id);
83
+ element.style.position = 'fixed';
84
+ element.style.left = `${marker.point.x}px`;
85
+ element.style.top = `${marker.point.y}px`;
86
+ element.style.width = `${marker.size}px`;
87
+ element.style.height = `${marker.size}px`;
88
+ element.style.background = marker.color;
89
+ element.style.border = '3px solid #000';
90
+ element.style.boxSizing = 'border-box';
91
+ element.style.boxShadow = '0 0 0 2px #fff';
92
+ root.appendChild(element);
93
+ }
94
+
95
+ document.documentElement.appendChild(root);
96
+ }, payload);
97
+ }
98
+
99
+ async function removeMarkers(page: PageLike): Promise<void> {
100
+ await page.evaluate<void>(() => {
101
+ document.getElementById('__paf_native_coordinate_markers__')?.remove();
102
+ });
103
+ }
104
+
105
+ async function captureMarkerTemplate(page: PageLike, marker: MarkerSpec): Promise<NativeTemplateRef> {
106
+ const margin = 8;
107
+ const screenshot = await page.screenshot({
108
+ type: 'png',
109
+ clip: {
110
+ x: marker.point.x - margin,
111
+ y: marker.point.y - margin,
112
+ width: marker.size + margin * 2,
113
+ height: marker.size + margin * 2
114
+ }
115
+ });
116
+
117
+ if (typeof screenshot === 'string') {
118
+ return { name: `${marker.id}.png`, buffer: Buffer.from(screenshot, 'base64') };
119
+ }
120
+
121
+ return { name: `${marker.id}.png`, buffer: screenshot };
122
+ }
123
+
124
+ export class ImageBasedCoordinateMapper implements NativeCoordinateMapper {
125
+ private cached: CoordinateTransform | null = null;
126
+ private cacheExpiresAt = 0;
127
+
128
+ constructor(private readonly options: ImageBasedCoordinateMapperOptions) {}
129
+
130
+ async calibrate(page: PageLike, options: CalibrationOptions = {}): Promise<CoordinateTransform> {
131
+ const markerSize = options.markerSizePx ?? DEFAULT_MARKER_SIZE;
132
+ const markerA: MarkerSpec = {
133
+ id: 'paf-native-calibration-a',
134
+ point: options.markerA ?? DEFAULT_MARKER_A,
135
+ size: markerSize,
136
+ color: '#ff00ff'
137
+ };
138
+ const markerB: MarkerSpec = {
139
+ id: 'paf-native-calibration-b',
140
+ point: options.markerB ?? DEFAULT_MARKER_B,
141
+ size: markerSize,
142
+ color: '#00ffff'
143
+ };
144
+
145
+ await installMarkers(page, [markerA, markerB]);
146
+
147
+ try {
148
+ const viewport = await getViewportSnapshot(page);
149
+ const [templateA, templateB] = await Promise.all([
150
+ captureMarkerTemplate(page, markerA),
151
+ captureMarkerTemplate(page, markerB)
152
+ ]);
153
+
154
+ const imageOptions: { timeoutMs?: number; intervalMs?: number } = {};
155
+ const timeoutMs = options.timeoutMs ?? this.options.defaultTimeoutMs;
156
+ if (timeoutMs !== undefined) imageOptions.timeoutMs = timeoutMs;
157
+ if (options.intervalMs !== undefined) imageOptions.intervalMs = options.intervalMs;
158
+
159
+ const [matchA, matchB] = await Promise.all([
160
+ this.options.images.waitForTemplate(templateA, imageOptions),
161
+ this.options.images.waitForTemplate(templateB, imageOptions)
162
+ ]);
163
+
164
+ const screenA = centerOf(matchA.region);
165
+ const screenB = centerOf(matchB.region);
166
+ const browserA = {
167
+ x: markerA.point.x + markerA.size / 2,
168
+ y: markerA.point.y + markerA.size / 2
169
+ };
170
+ const browserB = {
171
+ x: markerB.point.x + markerB.size / 2,
172
+ y: markerB.point.y + markerB.size / 2
173
+ };
174
+
175
+ const scaleX = (screenB.x - screenA.x) / (browserB.x - browserA.x);
176
+ const scaleY = (screenB.y - screenA.y) / (browserB.y - browserA.y);
177
+ const offsetX = screenA.x - scaleX * browserA.x;
178
+ const offsetY = screenA.y - scaleY * browserA.y;
179
+
180
+ this.validateTransform({ offsetX, offsetY, scaleX, scaleY }, viewport);
181
+
182
+ const transform: CoordinateTransform = {
183
+ offsetX,
184
+ offsetY,
185
+ scaleX,
186
+ scaleY,
187
+ calibratedAt: Date.now(),
188
+ viewport: {
189
+ innerWidth: viewport.innerWidth,
190
+ innerHeight: viewport.innerHeight,
191
+ devicePixelRatio: viewport.devicePixelRatio
192
+ }
193
+ };
194
+
195
+ if (this.options.windows !== undefined && options.windowMatcher !== undefined) {
196
+ const window = await this.options.windows.find(options.windowMatcher);
197
+ if (window !== null) transform.window = window;
198
+ }
199
+
200
+ this.cached = transform;
201
+ this.cacheExpiresAt = transform.calibratedAt + (options.cacheTtlMs ?? this.options.defaultCacheTtlMs ?? 60_000);
202
+ return transform;
203
+ } finally {
204
+ await removeMarkers(page).catch(error => {
205
+ this.options.logger?.warn('Failed to remove native coordinate calibration markers.', { error });
206
+ });
207
+ }
208
+ }
209
+
210
+ async browserToScreen(point: BrowserPoint, options: CoordinateMapOptions = {}): Promise<ScreenPoint> {
211
+ const transform = await this.getTransform(options);
212
+ return {
213
+ x: transform.offsetX + transform.scaleX * point.x,
214
+ y: transform.offsetY + transform.scaleY * point.y
215
+ };
216
+ }
217
+
218
+ async screenToBrowser(point: ScreenPoint, options: CoordinateMapOptions = {}): Promise<BrowserPoint> {
219
+ const transform = await this.getTransform(options);
220
+ return {
221
+ x: (point.x - transform.offsetX) / transform.scaleX,
222
+ y: (point.y - transform.offsetY) / transform.scaleY
223
+ };
224
+ }
225
+
226
+ invalidate(): void {
227
+ this.cached = null;
228
+ this.cacheExpiresAt = 0;
229
+ }
230
+
231
+ private async getTransform(options: CoordinateMapOptions): Promise<CoordinateTransform> {
232
+ const useCache = options.useCache ?? true;
233
+ if (useCache && this.cached !== null && Date.now() < this.cacheExpiresAt) {
234
+ return this.cached;
235
+ }
236
+
237
+ if (options.page === undefined) {
238
+ throw new CoordinateCalibrationError('Coordinate mapping requires a calibrated transform or a page for calibration.');
239
+ }
240
+
241
+ return this.calibrate(options.page);
242
+ }
243
+
244
+ private validateTransform(transform: Pick<CoordinateTransform, 'offsetX' | 'offsetY' | 'scaleX' | 'scaleY'>, viewport: BrowserViewportSnapshot): void {
245
+ assertFinite(transform.offsetX, 'offsetX');
246
+ assertFinite(transform.offsetY, 'offsetY');
247
+ assertFinite(transform.scaleX, 'scaleX');
248
+ assertFinite(transform.scaleY, 'scaleY');
249
+
250
+ if (transform.scaleX < 0.5 || transform.scaleX > 3.5 || transform.scaleY < 0.5 || transform.scaleY > 3.5) {
251
+ throw new CoordinateCalibrationError('Coordinate calibration produced an implausible scale.', {
252
+ scaleX: transform.scaleX,
253
+ scaleY: transform.scaleY,
254
+ viewport
255
+ });
256
+ }
257
+ }
258
+ }