@elizaos/plugin-computeruse 2.0.3-beta.2 → 2.0.3-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/actions/clipboard.d.ts +22 -0
  2. package/dist/actions/clipboard.d.ts.map +1 -0
  3. package/dist/actions/helpers.d.ts +33 -0
  4. package/dist/actions/helpers.d.ts.map +1 -0
  5. package/dist/actions/progress.d.ts +26 -0
  6. package/dist/actions/progress.d.ts.map +1 -0
  7. package/dist/actions/use-computer-agent.d.ts +113 -0
  8. package/dist/actions/use-computer-agent.d.ts.map +1 -0
  9. package/dist/actions/use-computer.d.ts +3 -0
  10. package/dist/actions/use-computer.d.ts.map +1 -0
  11. package/dist/actions/window-handlers.d.ts +11 -0
  12. package/dist/actions/window-handlers.d.ts.map +1 -0
  13. package/dist/actions/window.d.ts +11 -0
  14. package/dist/actions/window.d.ts.map +1 -0
  15. package/dist/actor/actor.d.ts +84 -0
  16. package/dist/actor/actor.d.ts.map +1 -0
  17. package/dist/actor/agent-callbacks.d.ts +128 -0
  18. package/dist/actor/agent-callbacks.d.ts.map +1 -0
  19. package/dist/actor/agent-loop.d.ts +134 -0
  20. package/dist/actor/agent-loop.d.ts.map +1 -0
  21. package/dist/actor/aosp-input-actor.d.ts +87 -0
  22. package/dist/actor/aosp-input-actor.d.ts.map +1 -0
  23. package/dist/actor/brain.d.ts +195 -0
  24. package/dist/actor/brain.d.ts.map +1 -0
  25. package/dist/actor/cascade.d.ts +92 -0
  26. package/dist/actor/cascade.d.ts.map +1 -0
  27. package/dist/actor/computer-interface.d.ts +276 -0
  28. package/dist/actor/computer-interface.d.ts.map +1 -0
  29. package/dist/actor/dispatch.d.ts +24 -0
  30. package/dist/actor/dispatch.d.ts.map +1 -0
  31. package/dist/actor/index.d.ts +12 -0
  32. package/dist/actor/index.d.ts.map +1 -0
  33. package/dist/actor/types.d.ts +94 -0
  34. package/dist/actor/types.d.ts.map +1 -0
  35. package/dist/approval-manager.d.ts +29 -0
  36. package/dist/approval-manager.d.ts.map +1 -0
  37. package/dist/index.d.ts +46 -0
  38. package/dist/index.d.ts.map +1 -0
  39. package/dist/index.js +13649 -0
  40. package/dist/index.js.map +68 -0
  41. package/dist/mcp/index.d.ts +8 -0
  42. package/dist/mcp/index.d.ts.map +1 -0
  43. package/dist/mcp/server.d.ts +42 -0
  44. package/dist/mcp/server.d.ts.map +1 -0
  45. package/dist/mcp/tools.d.ts +53 -0
  46. package/dist/mcp/tools.d.ts.map +1 -0
  47. package/dist/mobile/android-bridge.d.ts +263 -0
  48. package/dist/mobile/android-bridge.d.ts.map +1 -0
  49. package/dist/mobile/android-scene.d.ts +52 -0
  50. package/dist/mobile/android-scene.d.ts.map +1 -0
  51. package/dist/mobile/android-trajectory.d.ts +66 -0
  52. package/dist/mobile/android-trajectory.d.ts.map +1 -0
  53. package/dist/mobile/index.d.ts +19 -0
  54. package/dist/mobile/index.d.ts.map +1 -0
  55. package/dist/mobile/ios-app-intent-registry.d.ts +20 -0
  56. package/dist/mobile/ios-app-intent-registry.d.ts.map +1 -0
  57. package/dist/mobile/ios-bridge.d.ts +359 -0
  58. package/dist/mobile/ios-bridge.d.ts.map +1 -0
  59. package/dist/mobile/ios-computer-interface.d.ts +160 -0
  60. package/dist/mobile/ios-computer-interface.d.ts.map +1 -0
  61. package/dist/mobile/mobile-computer-interface.d.ts +142 -0
  62. package/dist/mobile/mobile-computer-interface.d.ts.map +1 -0
  63. package/dist/mobile/mobile-screen-capture.d.ts +64 -0
  64. package/dist/mobile/mobile-screen-capture.d.ts.map +1 -0
  65. package/dist/mobile/ocr-provider.d.ts +187 -0
  66. package/dist/mobile/ocr-provider.d.ts.map +1 -0
  67. package/dist/mobile/ocr-provider.js +111 -0
  68. package/dist/mobile/ocr-provider.js.map +10 -0
  69. package/dist/osworld/action-converter.d.ts +38 -0
  70. package/dist/osworld/action-converter.d.ts.map +1 -0
  71. package/dist/osworld/adapter.d.ts +79 -0
  72. package/dist/osworld/adapter.d.ts.map +1 -0
  73. package/dist/osworld/types.d.ts +69 -0
  74. package/dist/osworld/types.d.ts.map +1 -0
  75. package/dist/parity/index.d.ts +9 -0
  76. package/dist/parity/index.d.ts.map +1 -0
  77. package/dist/parity/parity-matrix.d.ts +82 -0
  78. package/dist/parity/parity-matrix.d.ts.map +1 -0
  79. package/dist/parity/screenspot.d.ts +56 -0
  80. package/dist/parity/screenspot.d.ts.map +1 -0
  81. package/dist/platform/a11y.d.ts +64 -0
  82. package/dist/platform/a11y.d.ts.map +1 -0
  83. package/dist/platform/browser.d.ts +61 -0
  84. package/dist/platform/browser.d.ts.map +1 -0
  85. package/dist/platform/capabilities.d.ts +33 -0
  86. package/dist/platform/capabilities.d.ts.map +1 -0
  87. package/dist/platform/capture.d.ts +65 -0
  88. package/dist/platform/capture.d.ts.map +1 -0
  89. package/dist/platform/clipboard.d.ts +24 -0
  90. package/dist/platform/clipboard.d.ts.map +1 -0
  91. package/dist/platform/coords.d.ts +73 -0
  92. package/dist/platform/coords.d.ts.map +1 -0
  93. package/dist/platform/desktop.d.ts +56 -0
  94. package/dist/platform/desktop.d.ts.map +1 -0
  95. package/dist/platform/displays.d.ts +97 -0
  96. package/dist/platform/displays.d.ts.map +1 -0
  97. package/dist/platform/driver.d.ts +49 -0
  98. package/dist/platform/driver.d.ts.map +1 -0
  99. package/dist/platform/file-ops.d.ts +27 -0
  100. package/dist/platform/file-ops.d.ts.map +1 -0
  101. package/dist/platform/helpers.d.ts +60 -0
  102. package/dist/platform/helpers.d.ts.map +1 -0
  103. package/dist/platform/launch.d.ts +54 -0
  104. package/dist/platform/launch.d.ts.map +1 -0
  105. package/dist/platform/normalized-coords.d.ts +46 -0
  106. package/dist/platform/normalized-coords.d.ts.map +1 -0
  107. package/dist/platform/nut-driver.d.ts +86 -0
  108. package/dist/platform/nut-driver.d.ts.map +1 -0
  109. package/dist/platform/permissions.d.ts +33 -0
  110. package/dist/platform/permissions.d.ts.map +1 -0
  111. package/dist/platform/process-list.d.ts +32 -0
  112. package/dist/platform/process-list.d.ts.map +1 -0
  113. package/dist/platform/ps-host.d.ts +77 -0
  114. package/dist/platform/ps-host.d.ts.map +1 -0
  115. package/dist/platform/screenshot-errors.d.ts +54 -0
  116. package/dist/platform/screenshot-errors.d.ts.map +1 -0
  117. package/dist/platform/screenshot-quality.d.ts +11 -0
  118. package/dist/platform/screenshot-quality.d.ts.map +1 -0
  119. package/dist/platform/screenshot.d.ts +16 -0
  120. package/dist/platform/screenshot.d.ts.map +1 -0
  121. package/dist/platform/security.d.ts +20 -0
  122. package/dist/platform/security.d.ts.map +1 -0
  123. package/dist/platform/terminal.d.ts +38 -0
  124. package/dist/platform/terminal.d.ts.map +1 -0
  125. package/dist/platform/wayland-portal.d.ts +25 -0
  126. package/dist/platform/wayland-portal.d.ts.map +1 -0
  127. package/dist/platform/windows-list.d.ts +78 -0
  128. package/dist/platform/windows-list.d.ts.map +1 -0
  129. package/dist/providers/computer-state.d.ts +9 -0
  130. package/dist/providers/computer-state.d.ts.map +1 -0
  131. package/dist/providers/scene.d.ts +21 -0
  132. package/dist/providers/scene.d.ts.map +1 -0
  133. package/dist/register-routes.d.ts +2 -0
  134. package/dist/register-routes.d.ts.map +1 -0
  135. package/dist/register-routes.js +13836 -0
  136. package/dist/register-routes.js.map +71 -0
  137. package/dist/routes/computer-use-compat-routes.d.ts +29 -0
  138. package/dist/routes/computer-use-compat-routes.d.ts.map +1 -0
  139. package/dist/routes/computer-use-routes.d.ts +3 -0
  140. package/dist/routes/computer-use-routes.d.ts.map +1 -0
  141. package/dist/routes/sandbox-routes.d.ts +53 -0
  142. package/dist/routes/sandbox-routes.d.ts.map +1 -0
  143. package/dist/sandbox/docker-backend.d.ts +69 -0
  144. package/dist/sandbox/docker-backend.d.ts.map +1 -0
  145. package/dist/sandbox/index.d.ts +62 -0
  146. package/dist/sandbox/index.d.ts.map +1 -0
  147. package/dist/sandbox/qemu-backend.d.ts +48 -0
  148. package/dist/sandbox/qemu-backend.d.ts.map +1 -0
  149. package/dist/sandbox/remote-guest.d.ts +72 -0
  150. package/dist/sandbox/remote-guest.d.ts.map +1 -0
  151. package/dist/sandbox/sandbox-driver.d.ts +41 -0
  152. package/dist/sandbox/sandbox-driver.d.ts.map +1 -0
  153. package/dist/sandbox/surface-types.d.ts +17 -0
  154. package/dist/sandbox/surface-types.d.ts.map +1 -0
  155. package/dist/sandbox/types.d.ts +138 -0
  156. package/dist/sandbox/types.d.ts.map +1 -0
  157. package/dist/sandbox/wsb-backend.d.ts +48 -0
  158. package/dist/sandbox/wsb-backend.d.ts.map +1 -0
  159. package/dist/scene/a11y-provider.d.ts +83 -0
  160. package/dist/scene/a11y-provider.d.ts.map +1 -0
  161. package/dist/scene/apps.d.ts +39 -0
  162. package/dist/scene/apps.d.ts.map +1 -0
  163. package/dist/scene/dhash.d.ts +105 -0
  164. package/dist/scene/dhash.d.ts.map +1 -0
  165. package/dist/scene/ocr-adapter.d.ts +64 -0
  166. package/dist/scene/ocr-adapter.d.ts.map +1 -0
  167. package/dist/scene/scene-builder.d.ts +107 -0
  168. package/dist/scene/scene-builder.d.ts.map +1 -0
  169. package/dist/scene/scene-types.d.ts +70 -0
  170. package/dist/scene/scene-types.d.ts.map +1 -0
  171. package/dist/scene/screen-state.d.ts +105 -0
  172. package/dist/scene/screen-state.d.ts.map +1 -0
  173. package/dist/scene/serialize.d.ts +28 -0
  174. package/dist/scene/serialize.d.ts.map +1 -0
  175. package/dist/security/browser-script-policy.d.ts +9 -0
  176. package/dist/security/browser-script-policy.d.ts.map +1 -0
  177. package/dist/services/computer-use-service.d.ts +142 -0
  178. package/dist/services/computer-use-service.d.ts.map +1 -0
  179. package/dist/services/desktop-control.d.ts +35 -0
  180. package/dist/services/desktop-control.d.ts.map +1 -0
  181. package/dist/services/index.d.ts +7 -0
  182. package/dist/services/index.d.ts.map +1 -0
  183. package/dist/services/vision-context-provider.d.ts +32 -0
  184. package/dist/services/vision-context-provider.d.ts.map +1 -0
  185. package/dist/types.d.ts +385 -0
  186. package/dist/types.d.ts.map +1 -0
  187. package/package.json +16 -5
  188. package/registry-entry.json +74 -0
@@ -0,0 +1,79 @@
1
+ /**
2
+ * OSWorld benchmark adapter.
3
+ *
4
+ * Bridges the xlang-ai/OSWorld evaluation framework with plugin-computeruse.
5
+ * Provides the observation/action loop expected by OSWorld:
6
+ *
7
+ * 1. getObservation() — capture screenshot + optional a11y tree
8
+ * 2. executeAction() — execute an OSWorld action (computer_13 or pyautogui)
9
+ * 3. step() — combined execute + observe (gymnasium-style)
10
+ *
11
+ * Usage:
12
+ * const adapter = new OSWorldAdapter(computerUseService);
13
+ * const obs = await adapter.getObservation("Open the settings app");
14
+ * const result = await adapter.executeAction({ action_type: "CLICK", x: 100, y: 200 });
15
+ * // or with pyautogui:
16
+ * const result2 = await adapter.executePyAutoGUI("pyautogui.click(100, 200)");
17
+ */
18
+ import type { ComputerUseService } from "../services/computer-use-service.js";
19
+ import type { OSWorldAction, OSWorldAgentConfig, OSWorldObservation, OSWorldStepResult } from "./types.js";
20
+ export declare class OSWorldAdapter {
21
+ private service;
22
+ private config;
23
+ private stepCount;
24
+ private pointerState;
25
+ private trajectory;
26
+ constructor(service: ComputerUseService, config?: Partial<OSWorldAgentConfig>);
27
+ /**
28
+ * Capture the current screen state as an OSWorld observation.
29
+ */
30
+ getObservation(instruction: string): Promise<OSWorldObservation>;
31
+ /**
32
+ * Execute an OSWorld computer_13 action.
33
+ * Returns true if the action was executed, false for control flow (WAIT/DONE/FAIL).
34
+ */
35
+ executeAction(action: OSWorldAction): Promise<{
36
+ executed: boolean;
37
+ done: boolean;
38
+ failed: boolean;
39
+ }>;
40
+ /**
41
+ * Execute a pyautogui Python code string.
42
+ */
43
+ executePyAutoGUI(code: string): Promise<{
44
+ executed: boolean;
45
+ done: boolean;
46
+ failed: boolean;
47
+ }>;
48
+ /**
49
+ * Execute an action and return the next observation (gymnasium step).
50
+ * Matches the signature: step(action) → (obs, reward, done, info)
51
+ */
52
+ step(action: OSWorldAction | string, instruction: string): Promise<OSWorldStepResult>;
53
+ /**
54
+ * Reset the adapter for a new task.
55
+ */
56
+ reset(): void;
57
+ /**
58
+ * Get the full trajectory of actions and observations.
59
+ */
60
+ getTrajectory(): {
61
+ action: OSWorldAction | string;
62
+ observation: OSWorldObservation;
63
+ timestamp: number;
64
+ }[];
65
+ /**
66
+ * Get current step count.
67
+ */
68
+ getStepCount(): number;
69
+ /**
70
+ * Check if a11y tree extraction is available.
71
+ */
72
+ isA11yAvailable(): boolean;
73
+ /**
74
+ * Get adapter config.
75
+ */
76
+ getConfig(): OSWorldAgentConfig;
77
+ private sleep;
78
+ }
79
+ //# sourceMappingURL=adapter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/osworld/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAM9E,OAAO,KAAK,EACV,aAAa,EACb,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,UAAU,CAIV;gBAGN,OAAO,EAAE,kBAAkB,EAC3B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IActC;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA2BtE;;;OAGG;IACG,aAAa,CACjB,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAyBjE;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IA2BjE;;;OAGG;IACG,IAAI,CACR,MAAM,EAAE,aAAa,GAAG,MAAM,EAC9B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,iBAAiB,CAAC;IAuC7B;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,aAAa;gBA5KH,aAAa,GAAG,MAAM;qBACjB,kBAAkB;mBACpB,MAAM;;IA8KnB;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,eAAe,IAAI,OAAO;IAI1B;;OAEG;IACH,SAAS,IAAI,kBAAkB;IAM/B,OAAO,CAAC,KAAK;CAGd"}
@@ -0,0 +1,69 @@
1
+ /**
2
+ * OSWorld benchmark type definitions.
3
+ *
4
+ * These types match the interface expected by the xlang-ai/OSWorld
5
+ * evaluation framework. The benchmark uses two action spaces:
6
+ *
7
+ * 1. "pyautogui" — Python code strings (e.g. "pyautogui.click(100, 200)")
8
+ * 2. "computer_13" — Structured action dicts with action_type field
9
+ *
10
+ * Our adapter supports both and converts them to plugin-computeruse actions.
11
+ */
12
+ export interface OSWorldObservation {
13
+ /** Base64-encoded PNG screenshot */
14
+ screenshot: string;
15
+ /** Text accessibility tree of the focused window (optional) */
16
+ accessibility_tree?: string | null;
17
+ /** Terminal output if relevant (optional) */
18
+ terminal?: string | null;
19
+ /** Task instruction */
20
+ instruction: string;
21
+ }
22
+ /**
23
+ * OSWorld computer_13 action types (15 enumerated actions including WAIT/DONE/FAIL).
24
+ */
25
+ export type OSWorldActionType = "CLICK" | "MOVE_TO" | "DOUBLE_CLICK" | "RIGHT_CLICK" | "SCROLL" | "DRAG_TO" | "TYPING" | "PRESS" | "KEY_DOWN" | "KEY_UP" | "HOTKEY" | "MOUSE_DOWN" | "MOUSE_UP" | "WAIT" | "DONE" | "FAIL";
26
+ /**
27
+ * Structured action in the computer_13 action space.
28
+ */
29
+ export interface OSWorldAction {
30
+ action_type: OSWorldActionType;
31
+ x?: number;
32
+ y?: number;
33
+ text?: string;
34
+ keys?: string[];
35
+ key?: string;
36
+ dx?: number;
37
+ dy?: number;
38
+ button?: "left" | "right" | "middle";
39
+ direction?: "up" | "down" | "left" | "right";
40
+ amount?: number;
41
+ }
42
+ export type OSWorldObservationType = "screenshot" | "a11y_tree" | "screenshot_a11y_tree" | "som";
43
+ export interface OSWorldTaskConfig {
44
+ id: string;
45
+ instruction: string;
46
+ config?: Record<string, unknown>;
47
+ domain?: string;
48
+ evaluator?: Record<string, unknown>;
49
+ }
50
+ export interface OSWorldStepResult {
51
+ observation: OSWorldObservation;
52
+ reward: number;
53
+ done: boolean;
54
+ info: Record<string, unknown>;
55
+ }
56
+ export interface OSWorldAgentConfig {
57
+ /** Action space to use: "pyautogui" (code strings) or "computer_13" (structured actions) */
58
+ actionSpace: "pyautogui" | "computer_13";
59
+ /** Observation type */
60
+ observationType: OSWorldObservationType;
61
+ /** Maximum trajectory length before giving up */
62
+ maxTrajectoryLength: number;
63
+ /** Whether to include a11y tree in observations */
64
+ includeA11yTree: boolean;
65
+ /** Screenshot delay after actions (ms) */
66
+ screenshotDelayMs: number;
67
+ }
68
+ export declare const DEFAULT_AGENT_CONFIG: OSWorldAgentConfig;
69
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/osworld/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,MAAM,WAAW,kBAAkB;IACjC,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,uBAAuB;IACvB,WAAW,EAAE,MAAM,CAAC;CACrB;AAID;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,OAAO,GACP,SAAS,GACT,cAAc,GACd,aAAa,GACb,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,MAAM,GACN,MAAM,GACN,MAAM,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,iBAAiB,CAAC;IAC/B,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;IACrC,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAID,MAAM,MAAM,sBAAsB,GAC9B,YAAY,GACZ,WAAW,GACX,sBAAsB,GACtB,KAAK,CAAC;AAIV,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAID,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAID,MAAM,WAAW,kBAAkB;IACjC,4FAA4F;IAC5F,WAAW,EAAE,WAAW,GAAG,aAAa,CAAC;IACzC,uBAAuB;IACvB,eAAe,EAAE,sBAAsB,CAAC;IACxC,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mDAAmD;IACnD,eAAe,EAAE,OAAO,CAAC;IACzB,0CAA0C;IAC1C,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,eAAO,MAAM,oBAAoB,EAAE,kBAMlC,CAAC"}
@@ -0,0 +1,9 @@
1
+ /**
2
+ * trycua/cua parity tooling (#9170 M14) — public surface.
3
+ *
4
+ * The machine-checkable capability matrix + its validator, and the ScreenSpot
5
+ * grounding harness. The OSWorld benchmark adapter lives under `src/osworld/`.
6
+ */
7
+ export { type OsCoverage, type OsName, PARITY_MATRIX, type ParityCapability, type ParityCoverageByOs, type ParityStatus, type ParityValidationProblem, type ParityValidationResult, parityCoverageByOs, parityMatrixSummary, validateParityCoverage, validateParityMatrix, } from "./parity-matrix.js";
8
+ export { pointInBbox, type ScreenSpotPrediction, type ScreenSpotSample, type ScreenSpotSampleResult, type ScreenSpotScore, scoreScreenSpot, } from "./screenspot.js";
9
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parity/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,KAAK,UAAU,EACf,KAAK,MAAM,EACX,aAAa,EACb,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,WAAW,EACX,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,eAAe,EACpB,eAAe,GAChB,MAAM,iBAAiB,CAAC"}
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Machine-checkable trycua/cua parity matrix (#9170 M14).
3
+ *
4
+ * The tracking issue maps every cua computer-use capability to its elizaOS
5
+ * status. That map is only trustworthy if it can't silently drift from the
6
+ * code, so this module encodes it as DATA plus a validator: every capability
7
+ * marked `have` with a promoted action verb must actually be registered on the
8
+ * plugin. `validateParityMatrix(actionNames)` is wired into the test suite, so
9
+ * adding a verb to the matrix without registering it (or vice-versa) fails CI.
10
+ *
11
+ * Per-OS coverage records where each capability is exercised (scenario-runner
12
+ * cases + the gated real-driver lanes). `covered` = has evidence today,
13
+ * `planned` = on the M14 backlog, `na` = intentionally out of scope for that OS.
14
+ */
15
+ export type ParityStatus = "have" | "partial" | "na";
16
+ export type OsName = "windows" | "linux" | "macos" | "aosp";
17
+ export type OsCoverage = "covered" | "planned" | "na";
18
+ export interface ParityCapability {
19
+ /** cua capability / verb id. */
20
+ id: string;
21
+ /** Promoted elizaOS action verb (when `have`/`partial`), else omitted. */
22
+ elizaVerb?: string;
23
+ status: ParityStatus;
24
+ /** Milestone that delivered it (M7–M13), for traceability. */
25
+ milestone?: string;
26
+ os: Record<OsName, OsCoverage>;
27
+ note?: string;
28
+ }
29
+ /** The canonical capability → status map. */
30
+ export declare const PARITY_MATRIX: readonly ParityCapability[];
31
+ export interface ParityValidationProblem {
32
+ capability: string;
33
+ problem: string;
34
+ }
35
+ export interface ParityValidationResult {
36
+ ok: boolean;
37
+ problems: ParityValidationProblem[];
38
+ /** Count of `have` capabilities whose verb was confirmed registered. */
39
+ confirmed: number;
40
+ }
41
+ /**
42
+ * Cross-check the matrix against the live action surface. Every `have`/`partial`
43
+ * capability whose `elizaVerb` is a promoted action (COMPUTER_USE_* / WINDOW_* /
44
+ * CLIPBOARD_*) must appear in `actionNames`. Pure — the caller passes the
45
+ * plugin's registered action names.
46
+ */
47
+ export declare function validateParityMatrix(actionNames: readonly string[]): ParityValidationResult;
48
+ /** Summary counts for reporting / dashboards. */
49
+ export declare function parityMatrixSummary(): {
50
+ have: number;
51
+ partial: number;
52
+ na: number;
53
+ total: number;
54
+ };
55
+ /** Per-OS coverage rollup for the parity matrix. */
56
+ export interface ParityCoverageByOs {
57
+ os: OsName;
58
+ covered: number;
59
+ planned: number;
60
+ na: number;
61
+ }
62
+ /**
63
+ * Roll up per-OS coverage across every `have`/`partial` capability. `na`
64
+ * capabilities are excluded (they are not part of any OS's parity surface).
65
+ * Pure — derives only from PARITY_MATRIX. Useful for a "what still needs a
66
+ * macOS/Linux real-lane case" dashboard and the M14 coverage report.
67
+ */
68
+ export declare function parityCoverageByOs(): ParityCoverageByOs[];
69
+ /**
70
+ * Structural invariants on the per-OS coverage records — the `os` field was
71
+ * free-form data with no guard, so a typo or a capability that claims
72
+ * `covered` on a platform it can't run on could drift in silently. Pure; the
73
+ * test suite asserts `ok`. Invariants:
74
+ * 1. every `have`/`partial` capability declares all four OS keys with a valid
75
+ * OsCoverage value;
76
+ * 2. an `na` capability is `os: na` on every axis (it is out of scope, not
77
+ * "covered/planned" anywhere);
78
+ * 3. a non-`na` capability must have at least one OS that is `covered` or
79
+ * `planned` (a verb that is `na` on every OS should be status `na`).
80
+ */
81
+ export declare function validateParityCoverage(): ParityValidationResult;
82
+ //# sourceMappingURL=parity-matrix.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parity-matrix.d.ts","sourceRoot":"","sources":["../../src/parity/parity-matrix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;AACrD,MAAM,MAAM,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AAC5D,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC;AAEtD,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAgBD,6CAA6C;AAC7C,eAAO,MAAM,aAAa,EAAE,SAAS,gBAAgB,EAyQpD,CAAC;AAMF,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,EAAE,uBAAuB,EAAE,CAAC;IACpC,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;CACnB;AAKD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,SAAS,MAAM,EAAE,GAC7B,sBAAsB,CA+BxB;AAED,iDAAiD;AACjD,wBAAgB,mBAAmB,IAAI;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf,CAUA;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,IAAI,kBAAkB,EAAE,CAsBzD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,IAAI,sBAAsB,CAyC/D"}
@@ -0,0 +1,56 @@
1
+ /**
2
+ * ScreenSpot grounding harness (#9170 M14).
3
+ *
4
+ * ScreenSpot scores click-grounding: given an instruction + screenshot, a model
5
+ * predicts a click point; the sample is correct iff the point lands inside the
6
+ * ground-truth element bbox. This module is the scorer + harness — it consumes a
7
+ * grounder function (e.g. the M9 Set-of-Marks marks → `center`, or the M10
8
+ * `predictClick` seam) and reports point-in-bbox accuracy.
9
+ *
10
+ * The dataset itself is NOT vendored (the real ScreenSpot images are large and
11
+ * licensed separately); a runner provides samples + a grounder and gets a score.
12
+ * Pure scoring is unit-tested with synthetic samples.
13
+ */
14
+ export interface ScreenSpotSample {
15
+ id: string;
16
+ instruction: string;
17
+ /** Ground-truth target bbox `[x, y, w, h]` in image pixels. */
18
+ bbox: [number, number, number, number];
19
+ imageWidth: number;
20
+ imageHeight: number;
21
+ /** Optional source group (e.g. "icon" / "text" / platform) for breakdowns. */
22
+ group?: string;
23
+ }
24
+ export interface ScreenSpotPrediction {
25
+ x: number;
26
+ y: number;
27
+ }
28
+ /** A predicted point lands inside the bbox (inclusive of edges). Pure. */
29
+ export declare function pointInBbox(point: ScreenSpotPrediction, bbox: readonly [number, number, number, number]): boolean;
30
+ export interface ScreenSpotSampleResult {
31
+ id: string;
32
+ group?: string;
33
+ predicted: ScreenSpotPrediction | null;
34
+ correct: boolean;
35
+ }
36
+ export interface ScreenSpotScore {
37
+ total: number;
38
+ correct: number;
39
+ /** correct / total (0 when total is 0). */
40
+ accuracy: number;
41
+ /** Per-group accuracy breakdown. */
42
+ byGroup: Record<string, {
43
+ total: number;
44
+ correct: number;
45
+ accuracy: number;
46
+ }>;
47
+ results: ScreenSpotSampleResult[];
48
+ }
49
+ /**
50
+ * Run a grounder over the samples and score point-in-bbox accuracy. The grounder
51
+ * returns a predicted click point (or `null` when it abstains — counted wrong).
52
+ * Samples run sequentially so a real grounder's per-call I/O is not blasted in
53
+ * parallel.
54
+ */
55
+ export declare function scoreScreenSpot(samples: readonly ScreenSpotSample[], predict: (sample: ScreenSpotSample) => Promise<ScreenSpotPrediction | null> | ScreenSpotPrediction | null): Promise<ScreenSpotScore>;
56
+ //# sourceMappingURL=screenspot.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"screenspot.d.ts","sourceRoot":"","sources":["../../src/parity/screenspot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAED,0EAA0E;AAC1E,wBAAgB,WAAW,CACzB,KAAK,EAAE,oBAAoB,EAC3B,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAC9C,OAAO,CAGT;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACvC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,sBAAsB,EAAE,CAAC;CACnC;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,OAAO,EAAE,CACP,MAAM,EAAE,gBAAgB,KACrB,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,oBAAoB,GAAG,IAAI,GACtE,OAAO,CAAC,eAAe,CAAC,CAsC1B"}
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Cross-platform accessibility tree extraction.
3
+ *
4
+ * OSWorld benchmarks expect an optional accessibility tree as part of the
5
+ * observation. This module extracts a simplified a11y tree from the desktop
6
+ * using native platform tools.
7
+ *
8
+ * macOS — System Accessibility API via osascript / swift
9
+ * Linux — AT-SPI via python3-atspi (X11 + GNOME-Wayland). Wayland-only
10
+ * environments without AT-SPI fall back to a structured snapshot
11
+ * built from `listWindows()` + `listProcesses()` and tagged with
12
+ * `source: "compositor-fallback"`.
13
+ * Windows — UIAutomation via PowerShell
14
+ */
15
+ export interface A11yNode {
16
+ role: string;
17
+ name: string;
18
+ description?: string;
19
+ value?: string;
20
+ bounds?: {
21
+ x: number;
22
+ y: number;
23
+ width: number;
24
+ height: number;
25
+ };
26
+ children?: A11yNode[];
27
+ }
28
+ /**
29
+ * Structured snapshot returned by the Wayland compositor fallback. Callers
30
+ * inspect `source` to discriminate AT-SPI quality from the cheaper
31
+ * window-list join used when AT-SPI is unavailable.
32
+ */
33
+ export interface WaylandA11ySnapshot {
34
+ source: "compositor-fallback";
35
+ role: "desktop";
36
+ name: string;
37
+ focusedWindow: {
38
+ app: string;
39
+ title: string;
40
+ id: string;
41
+ } | null;
42
+ focusedApp: {
43
+ pid: number;
44
+ name: string;
45
+ } | null;
46
+ children: Array<{
47
+ role: "window";
48
+ name: string;
49
+ app: string;
50
+ id: string;
51
+ }>;
52
+ }
53
+ /**
54
+ * Extract the accessibility tree of the focused window / screen.
55
+ * Returns a simplified plain-text accessibility tree suitable for LLM consumption.
56
+ *
57
+ * Returns null if a11y data is unavailable on the current platform.
58
+ */
59
+ export declare function extractA11yTree(): string | null;
60
+ /**
61
+ * Check if a11y tree extraction is available on this platform.
62
+ */
63
+ export declare function isA11yAvailable(): boolean;
64
+ //# sourceMappingURL=a11y.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"a11y.d.ts","sourceRoot":"","sources":["../../src/platform/a11y.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;CACvB;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,qBAAqB,CAAC;IAC9B,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjE,UAAU,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjD,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;KACZ,CAAC,CAAC;CACJ;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,IAAI,MAAM,GAAG,IAAI,CAkB/C;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAezC"}
@@ -0,0 +1,61 @@
1
+ /**
2
+ * Browser automation via Puppeteer Core + Chrome DevTools Protocol.
3
+ *
4
+ * Ported from coasty-ai/open-computer-use browser-automation.ts (Apache 2.0).
5
+ *
6
+ * Uses puppeteer-core (not full puppeteer) to avoid bundling Chromium.
7
+ * Auto-detects installed Chrome, Edge, or Brave at launch.
8
+ * Each session uses a temp user data directory to prevent conflicts.
9
+ */
10
+ import type { BrowserInfo, BrowserState, BrowserTab, ClickableElement } from "../types.js";
11
+ export declare function setBrowserRuntimeOptions(options: {
12
+ headless?: boolean;
13
+ }): void;
14
+ /**
15
+ * Check if a Chromium-based browser is available.
16
+ */
17
+ export declare function isBrowserAvailable(): boolean;
18
+ export declare function openBrowser(url?: string): Promise<BrowserState>;
19
+ export declare function closeBrowser(): Promise<void>;
20
+ export declare function navigateBrowser(url: string): Promise<BrowserState>;
21
+ export declare function clickBrowser(selector?: string, coordinate?: [number, number], text?: string): Promise<void>;
22
+ export declare function typeBrowser(text: string, selector?: string): Promise<void>;
23
+ export declare function scrollBrowser(direction: "up" | "down", amount?: number): Promise<void>;
24
+ export declare function getBrowserState(): Promise<BrowserState>;
25
+ export declare function getBrowserContext(): Promise<BrowserState>;
26
+ export declare function getBrowserInfo(): Promise<BrowserInfo>;
27
+ export declare function getBrowserDom(): Promise<string>;
28
+ export declare function getBrowserClickables(): Promise<ClickableElement[]>;
29
+ export declare function screenshotBrowser(): Promise<string>;
30
+ export declare function executeBrowser(_code: string): Promise<string>;
31
+ export declare function waitBrowser(selector?: string, text?: string, timeout?: number): Promise<void>;
32
+ export declare function browserWait(selector?: string, text?: string, timeout?: number): Promise<{
33
+ success: boolean;
34
+ message?: string;
35
+ error?: string;
36
+ }>;
37
+ export declare function listBrowserTabs(): Promise<BrowserTab[]>;
38
+ export declare function openBrowserTab(url?: string): Promise<BrowserTab>;
39
+ export declare function closeBrowserTab(tabId: string): Promise<void>;
40
+ export declare function switchBrowserTab(tabId: string): Promise<BrowserState>;
41
+ export declare const browser_open: typeof openBrowser;
42
+ export declare const browser_connect: typeof openBrowser;
43
+ export declare const browser_navigate: typeof navigateBrowser;
44
+ export declare const browser_click: typeof clickBrowser;
45
+ export declare const browser_type: typeof typeBrowser;
46
+ export declare const browser_scroll: typeof scrollBrowser;
47
+ export declare const browser_close: typeof closeBrowser;
48
+ export declare const browser_execute: typeof executeBrowser;
49
+ export declare const browser_screenshot: typeof screenshotBrowser;
50
+ export declare const browser_dom: typeof getBrowserDom;
51
+ export declare const browser_get_dom: typeof getBrowserDom;
52
+ export declare const browser_get_clickables: typeof getBrowserClickables;
53
+ export declare const browser_state: typeof getBrowserState;
54
+ export declare const browser_get_context: typeof getBrowserContext;
55
+ export declare const browser_info: typeof getBrowserInfo;
56
+ export declare const browser_wait: typeof browserWait;
57
+ export declare const browser_list_tabs: typeof listBrowserTabs;
58
+ export declare const browser_open_tab: typeof openBrowserTab;
59
+ export declare const browser_close_tab: typeof closeBrowserTab;
60
+ export declare const browser_switch_tab: typeof switchBrowserTab;
61
+ //# sourceMappingURL=browser.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/platform/browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAOH,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EACjB,MAAM,aAAa,CAAC;AA+BrB,wBAAgB,wBAAwB,CAAC,OAAO,EAAE;IAChD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,GAAG,IAAI,CAIP;AAsFD;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,OAAO,CAE5C;AAkBD,wBAAsB,WAAW,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAwErE;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAmBlD;AAID,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CASxE;AAID,wBAAsB,YAAY,CAChC,QAAQ,CAAC,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC,CAmCf;AAID,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CASf;AAID,wBAAsB,aAAa,CACjC,SAAS,EAAE,IAAI,GAAG,MAAM,EACxB,MAAM,SAAM,GACX,OAAO,CAAC,IAAI,CAAC,CAIf;AAID,wBAAsB,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC,CAQ7D;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,YAAY,CAAC,CAE/D;AAED,wBAAsB,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,CAmB3D;AAID,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAKrD;AAID,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAmCxE;AAID,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,CAKzD;AAID,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEnE;AA+BD,wBAAsB,WAAW,CAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,SAAO,GACb,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,WAAW,CAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,SAAO,GACb,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBjE;AAID,wBAAsB,eAAe,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAa7D;AAED,wBAAsB,cAAc,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CActE;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAWlE;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAc3E;AAED,eAAO,MAAM,YAAY,oBAAc,CAAC;AACxC,eAAO,MAAM,eAAe,oBAAc,CAAC;AAC3C,eAAO,MAAM,gBAAgB,wBAAkB,CAAC;AAChD,eAAO,MAAM,aAAa,qBAAe,CAAC;AAC1C,eAAO,MAAM,YAAY,oBAAc,CAAC;AACxC,eAAO,MAAM,cAAc,sBAAgB,CAAC;AAC5C,eAAO,MAAM,aAAa,qBAAe,CAAC;AAC1C,eAAO,MAAM,eAAe,uBAAiB,CAAC;AAC9C,eAAO,MAAM,kBAAkB,0BAAoB,CAAC;AACpD,eAAO,MAAM,WAAW,sBAAgB,CAAC;AACzC,eAAO,MAAM,eAAe,sBAAgB,CAAC;AAC7C,eAAO,MAAM,sBAAsB,6BAAuB,CAAC;AAC3D,eAAO,MAAM,aAAa,wBAAkB,CAAC;AAC7C,eAAO,MAAM,mBAAmB,0BAAoB,CAAC;AACrD,eAAO,MAAM,YAAY,uBAAiB,CAAC;AAC3C,eAAO,MAAM,YAAY,oBAAc,CAAC;AACxC,eAAO,MAAM,iBAAiB,wBAAkB,CAAC;AACjD,eAAO,MAAM,gBAAgB,uBAAiB,CAAC;AAC/C,eAAO,MAAM,iBAAiB,wBAAkB,CAAC;AACjD,eAAO,MAAM,kBAAkB,yBAAmB,CAAC"}
@@ -0,0 +1,33 @@
1
+ import type { PlatformCapabilities } from "../types.js";
2
+ import type { PlatformOS } from "./helpers.js";
3
+ export interface CapabilityDetectionOptions {
4
+ osName: PlatformOS;
5
+ commandExists: (command: string) => boolean;
6
+ isBrowserAvailable: () => boolean;
7
+ shell?: string;
8
+ }
9
+ /**
10
+ * Per-capability parity classification, kept in lock-step with
11
+ * `plugins/plugin-computeruse/src/mobile/parity-status.md`.
12
+ *
13
+ * verified — exercised on real hardware (Linux/macOS in CI today)
14
+ * code-parity — feature-equivalent code path exists, runtime untested
15
+ * unavailable — surface present but not available in this delivery model
16
+ * blocked — OS does not allow the operation in our delivery model
17
+ *
18
+ * iOS / Android live in `mobile/parity-status.md`; they are not desktop OSes
19
+ * and don't pass through `detectPlatformCapabilities`.
20
+ */
21
+ export type ParityStatus = "verified" | "code-parity" | "unavailable" | "blocked";
22
+ export interface ParityNote {
23
+ readonly status: ParityStatus;
24
+ readonly note?: string;
25
+ }
26
+ /**
27
+ * Static parity table for desktop targets, derived from the per-file audit.
28
+ * Linux is the reference implementation.
29
+ */
30
+ export declare const DESKTOP_PARITY: Readonly<Record<PlatformOS, Readonly<Record<keyof PlatformCapabilities, ParityNote>>>>;
31
+ export declare function parityFor(osName: PlatformOS, capability: keyof PlatformCapabilities): ParityNote;
32
+ export declare function detectPlatformCapabilities(options: CapabilityDetectionOptions): PlatformCapabilities;
33
+ //# sourceMappingURL=capabilities.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"capabilities.d.ts","sourceRoot":"","sources":["../../src/platform/capabilities.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,UAAU,CAAC;IACnB,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IAC5C,kBAAkB,EAAE,MAAM,OAAO,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,aAAa,GACb,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,eAAO,MAAM,cAAc,EAAE,QAAQ,CACnC,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,oBAAoB,EAAE,UAAU,CAAC,CAAC,CAAC,CA4C7E,CAAC;AAEF,wBAAgB,SAAS,CACvB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,oBAAoB,GACrC,UAAU,CAEZ;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,0BAA0B,GAClC,oBAAoB,CAoFtB"}
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Per-display capture (WS5).
3
+ *
4
+ * Canonical capture entry point. Replaces `screenshot.ts` for any caller that
5
+ * cares about multi-monitor or DPI. The legacy `captureScreenshot()` keeps
6
+ * working (single-display, primary only) for back-compat.
7
+ *
8
+ * Returned `frame` is a PNG buffer at backing-store resolution:
9
+ * - macOS retina — 2× the logical bounds (e.g. 5120×2880 for a 2560×1440
10
+ * display reported as scaleFactor 2).
11
+ * - Windows/Linux — pixel-equivalent to logical bounds.
12
+ *
13
+ * Implementation notes:
14
+ * - macOS uses `screencapture -D <displayIndex+1>` to pick a specific
15
+ * display (1-indexed). For ScreenCaptureKit follow-up, replace this
16
+ * dispatch with the Swift sidecar.
17
+ * - Linux/Wayland prefers the xdg-desktop-portal screenshot sidecar. X11
18
+ * uses `import` (ImageMagick) or `scrot` cropped to the display's xrandr
19
+ * rect.
20
+ * - Windows uses PowerShell + System.Drawing to crop the virtual desktop
21
+ * to the screen bounds.
22
+ *
23
+ * Every code path is conditional on the current platform — a Linux test host
24
+ * can exercise the Linux paths; macOS / Windows paths are typechecked but
25
+ * exercised only in CI on those OSes.
26
+ */
27
+ import type { ScreenRegion } from "../types.js";
28
+ import type { DisplayInfo } from "./displays.js";
29
+ export interface DisplayCapture {
30
+ display: DisplayInfo;
31
+ /** PNG bytes at backing-store resolution. */
32
+ frame: Buffer;
33
+ }
34
+ /**
35
+ * Capture every attached display in parallel. Returns one entry per display.
36
+ * Errors on any display are surfaced as rejected promises — caller decides
37
+ * whether to retry or partial-fail.
38
+ */
39
+ export declare function captureAllDisplays(): Promise<DisplayCapture[]>;
40
+ /**
41
+ * Capture a specific display by id.
42
+ */
43
+ export declare function captureDisplay(displayId: number): Promise<DisplayCapture>;
44
+ /**
45
+ * Capture a region within a specific display. Region coordinates are LOCAL
46
+ * to that display.
47
+ */
48
+ export declare function captureDisplayRegion(displayId: number, region: ScreenRegion): Promise<DisplayCapture>;
49
+ /**
50
+ * Convenience: capture the primary display.
51
+ */
52
+ export declare function capturePrimaryDisplay(): Promise<DisplayCapture>;
53
+ /**
54
+ * Coerce a capture region to integer pixels and reject empty/degenerate sizes.
55
+ *
56
+ * `New-Object System.Drawing.Bitmap(w, h)` throws an opaque "Parameter is not
57
+ * valid" GDI+ error for a zero/negative/non-integer width or height, so we
58
+ * validate up front and surface a clear message. `x`/`y` may legitimately be
59
+ * negative — secondary monitors placed left of / above the primary live in the
60
+ * negative quadrant of the Windows virtual-desktop coordinate space — so only
61
+ * the dimensions are bounds-checked. Exported for cross-platform unit tests.
62
+ */
63
+ export declare function normalizeCaptureRegion(region: ScreenRegion): ScreenRegion;
64
+ export { captureScreenshot } from "./screenshot.js";
65
+ //# sourceMappingURL=capture.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../../src/platform/capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAkBjD,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,WAAW,CAAC;IACrB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAGpE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,CA4CzB;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,cAAc,CAAC,CA6BzB;AAED;;GAEG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,cAAc,CAAC,CAErE;AA4GD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CAmBzE;AAuDD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC"}
@@ -0,0 +1,24 @@
1
+ export declare class ClipboardUnavailableError extends Error {
2
+ readonly tool: string;
3
+ constructor(tool: string, hint: string);
4
+ }
5
+ type LinuxBackend = "wayland" | "x11";
6
+ declare function detectLinuxBackend(): LinuxBackend;
7
+ interface ClipboardCommand {
8
+ command: string;
9
+ args: readonly string[];
10
+ }
11
+ interface ClipboardPlan {
12
+ read: ClipboardCommand;
13
+ write: ClipboardCommand;
14
+ }
15
+ declare function pickPlan(): ClipboardPlan;
16
+ export declare function readClipboard(): Promise<string>;
17
+ export declare function writeClipboard(text: string): Promise<void>;
18
+ /** Internal hook for unit tests — re-exported for parity with helpers.ts. */
19
+ export declare const __testing: {
20
+ detectLinuxBackend: typeof detectLinuxBackend;
21
+ pickPlan: typeof pickPlan;
22
+ };
23
+ export {};
24
+ //# sourceMappingURL=clipboard.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"clipboard.d.ts","sourceRoot":"","sources":["../../src/platform/clipboard.ts"],"names":[],"mappings":"AA6BA,qBAAa,yBAA0B,SAAQ,KAAK;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBACV,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;CAKvC;AAED,KAAK,YAAY,GAAG,SAAS,GAAG,KAAK,CAAC;AAEtC,iBAAS,kBAAkB,IAAI,YAAY,CAI1C;AAED,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,UAAU,aAAa;IACrB,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,gBAAgB,CAAC;CACzB;AAED,iBAAS,QAAQ,IAAI,aAAa,CAiDjC;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAmBrD;AAED,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8ChE;AAED,6EAA6E;AAC7E,eAAO,MAAM,SAAS;;;CAGrB,CAAC"}