@melra/computer-runtime 0.3.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +53 -0
- package/dist/index.d.ts +194 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1306 -0
- package/dist/index.js.map +1 -0
- package/dist/index.test.d.ts +2 -0
- package/dist/index.test.d.ts.map +1 -0
- package/dist/index.test.js +259 -0
- package/dist/index.test.js.map +1 -0
- package/dist/windows.test.d.ts +2 -0
- package/dist/windows.test.d.ts.map +1 -0
- package/dist/windows.test.js +129 -0
- package/dist/windows.test.js.map +1 -0
- package/package.json +32 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# @melra/computer-runtime
|
|
2
|
+
|
|
3
|
+
Screen and input control for [MELRA](https://github.com/XAGI-Lab/melra):
|
|
4
|
+
capabilities, inspect, screenshot, click, move, drag, type, key, and scroll,
|
|
5
|
+
driven through the host's own automation tooling.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @melra/computer-runtime
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { ComputerOperationSchema } from "@melra/protocol";
|
|
13
|
+
import { ComputerRuntime } from "@melra/computer-runtime";
|
|
14
|
+
|
|
15
|
+
const computer = new ComputerRuntime({ artifactDirectory });
|
|
16
|
+
const report = await computer.execute(
|
|
17
|
+
ComputerOperationSchema.parse({ kind: "computer", action: "capabilities" }),
|
|
18
|
+
);
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Ask `capabilities` first. Desktop control depends on platform tooling and on
|
|
22
|
+
permissions the OS grants to the process — on macOS that means Accessibility and
|
|
23
|
+
Screen Recording — so the honest answer on many machines is that some actions are
|
|
24
|
+
unavailable. `capabilities` reports what this host can actually do instead of
|
|
25
|
+
letting a call fail halfway through a sequence; every other action is refused
|
|
26
|
+
with `computer_use_unavailable` when the host cannot support it.
|
|
27
|
+
|
|
28
|
+
Coordinates are validated against the reported screen bounds before an event is
|
|
29
|
+
synthesised, and typed text is escaped for the underlying tool rather than
|
|
30
|
+
concatenated into a script — `escapeSendKeys` is exported for that reason.
|
|
31
|
+
|
|
32
|
+
`inspect` is the read that makes the mutations checkable. It reports the
|
|
33
|
+
frontmost application, the focused window's title, and the display geometry, so
|
|
34
|
+
a task can declare "the frontmost application is Safari" as a post-condition
|
|
35
|
+
with `result_equals` rather than accepting an adapter's `success: true`. Fields
|
|
36
|
+
the platform cannot observe — a window title without macOS Accessibility
|
|
37
|
+
permission, an application on a bare X session — come back absent rather than
|
|
38
|
+
empty, because an empty string would verify as an observation.
|
|
39
|
+
|
|
40
|
+
On macOS `inspect` also reports `secureInput`, and `type` and `key` are refused
|
|
41
|
+
while it is held. Secure keyboard entry means the window server is dropping
|
|
42
|
+
synthetic keystrokes, so the alternative is a task that verifies as a success
|
|
43
|
+
having typed nothing into a focused password field.
|
|
44
|
+
|
|
45
|
+
This is the widest-blast-radius capability in MELRA: it drives the same input
|
|
46
|
+
devices the human is using, and it is not confined to a workspace the way files
|
|
47
|
+
and terminals are. Screenshots capture whatever is on screen, including windows
|
|
48
|
+
belonging to other applications, so treat the artifacts as sensitive.
|
|
49
|
+
|
|
50
|
+
Requires Node.js 22 or newer. Full documentation:
|
|
51
|
+
[github.com/XAGI-Lab/melra](https://github.com/XAGI-Lab/melra)
|
|
52
|
+
|
|
53
|
+
Apache-2.0
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import type { ComputerOperation } from "@melra/protocol";
|
|
2
|
+
export interface ComputerCapabilities {
|
|
3
|
+
platform: NodeJS.Platform;
|
|
4
|
+
adapter: "macos-native" | "linux-xdotool" | "windows-powershell" | "unavailable";
|
|
5
|
+
available: boolean;
|
|
6
|
+
screenshot: boolean;
|
|
7
|
+
pointer: boolean;
|
|
8
|
+
keyboard: boolean;
|
|
9
|
+
scroll: boolean;
|
|
10
|
+
/** `inspect` can report the focused window and display geometry. */
|
|
11
|
+
inspect: boolean;
|
|
12
|
+
/** `inspect` can list addressable elements, so `target` resolves to a point. */
|
|
13
|
+
elements: boolean;
|
|
14
|
+
/**
|
|
15
|
+
* A screenshot can be read for words, so `target` resolves on a desktop with
|
|
16
|
+
* no accessibility tree. Weaker than `elements` in kind, not only in quality:
|
|
17
|
+
* a tree states what a control *is*, while a reading only says what it looks
|
|
18
|
+
* like, which is why every element it produces carries a `confidence`.
|
|
19
|
+
*/
|
|
20
|
+
ocr: boolean;
|
|
21
|
+
/** `drag` can hold the button down between two points. */
|
|
22
|
+
drag: boolean;
|
|
23
|
+
coordinateSpaces: Array<"normalized" | "pixel">;
|
|
24
|
+
limitations: string[];
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* One addressable thing on the desktop, in pixel space. `role` is the platform's
|
|
28
|
+
* own word for what it is (`AXButton` on macOS, a UI Automation control type on
|
|
29
|
+
* Windows) rather than a normalised vocabulary: translating it would mean
|
|
30
|
+
* guessing at intent, and a caller that can read `inspect` output can match what
|
|
31
|
+
* it saw there.
|
|
32
|
+
*/
|
|
33
|
+
export interface DesktopElement {
|
|
34
|
+
role: string;
|
|
35
|
+
name?: string;
|
|
36
|
+
x: number;
|
|
37
|
+
y: number;
|
|
38
|
+
width: number;
|
|
39
|
+
height: number;
|
|
40
|
+
/**
|
|
41
|
+
* How sure the reading is, `0`–`1`. Present only on an element read off a
|
|
42
|
+
* screenshot; an element the platform reported has no confidence because it
|
|
43
|
+
* is not a reading. Absent therefore means "stated", never "unsure".
|
|
44
|
+
*/
|
|
45
|
+
confidence?: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* What the desktop looked like at a moment in time. `inspect` returns this, and
|
|
49
|
+
* a caller can hold a task to it with `result_equals` — "the frontmost
|
|
50
|
+
* application is Safari" is a post-condition an adapter cannot fake by
|
|
51
|
+
* returning `success: true`.
|
|
52
|
+
*
|
|
53
|
+
* Every field is optional because the platforms degrade differently: macOS
|
|
54
|
+
* needs Accessibility permission to name a window at all, X11 needs a window
|
|
55
|
+
* manager that sets `_NET_ACTIVE_WINDOW`, and a headless session has no
|
|
56
|
+
* frontmost anything. A missing field is "not observable here", never "empty".
|
|
57
|
+
*/
|
|
58
|
+
export interface DesktopObservation {
|
|
59
|
+
application?: string;
|
|
60
|
+
windowTitle?: string;
|
|
61
|
+
displayWidth?: number;
|
|
62
|
+
displayHeight?: number;
|
|
63
|
+
/**
|
|
64
|
+
* The OS has given one process exclusive keyboard access — a password field
|
|
65
|
+
* is focused, or a lock screen is up. Synthetic keystrokes do not reach it.
|
|
66
|
+
*/
|
|
67
|
+
secureInput?: boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Addressable elements of the frontmost window, when the platform exposes an
|
|
70
|
+
* accessibility tree — or, where it does not, words read off a screenshot,
|
|
71
|
+
* each carrying a `confidence`. Absent — not empty — when neither was
|
|
72
|
+
* possible, because "this window has no buttons" and "I cannot see this
|
|
73
|
+
* window's buttons" have to stay distinguishable to a caller deciding whether
|
|
74
|
+
* to fall back to coordinates.
|
|
75
|
+
*/
|
|
76
|
+
elements?: DesktopElement[];
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Words read off a screenshot, in the display's own pixel space.
|
|
80
|
+
*
|
|
81
|
+
* Tesseract's TSV carries one row per token with a `level` column: `1` is the
|
|
82
|
+
* whole page, `5` is a word. Only words are kept, because a line row spanning
|
|
83
|
+
* "Save Cancel" has a centre point that lands between two buttons.
|
|
84
|
+
*
|
|
85
|
+
* The page row's width is what the image measured, and a Retina capture is
|
|
86
|
+
* larger than the display it came from by the backing scale factor, so every
|
|
87
|
+
* box is scaled by the ratio between them. A capture with no readable page row
|
|
88
|
+
* yields nothing rather than coordinates at an assumed scale.
|
|
89
|
+
*
|
|
90
|
+
* Exported for the same reason as `matchElement`: the parsing is the part worth
|
|
91
|
+
* testing without a screen to point at.
|
|
92
|
+
*/
|
|
93
|
+
export declare function parseOcrElements(tsv: string, displayWidth: number): DesktopElement[];
|
|
94
|
+
/**
|
|
95
|
+
* Resolves a named target against an observed element list.
|
|
96
|
+
*
|
|
97
|
+
* Ambiguity is an error rather than a first-match, because the failure mode of
|
|
98
|
+
* guessing is clicking the wrong "Delete" and there is no undo for a desktop
|
|
99
|
+
* action. Exact names beat substrings for the same reason: a window with both
|
|
100
|
+
* "Save" and "Save As…" must resolve "Save" to the one the caller wrote. An
|
|
101
|
+
* element read off a screenshot has to clear `MIN_TARGET_CONFIDENCE` on top of
|
|
102
|
+
* matching, since a doubtful reading is a doubtful click and the same reasoning
|
|
103
|
+
* applies — worth reporting from `inspect`, not worth acting on.
|
|
104
|
+
*
|
|
105
|
+
* Exported because the matching rules are the part worth testing directly —
|
|
106
|
+
* everything around them needs a live desktop.
|
|
107
|
+
*/
|
|
108
|
+
export declare function matchElement(elements: DesktopElement[], target: {
|
|
109
|
+
role?: string | undefined;
|
|
110
|
+
name?: string | undefined;
|
|
111
|
+
}): DesktopElement;
|
|
112
|
+
export interface ComputerAdapter {
|
|
113
|
+
capabilities(): Promise<ComputerCapabilities>;
|
|
114
|
+
execute(operation: ComputerOperation, artifactDirectory: string, signal?: AbortSignal): Promise<Record<string, unknown>>;
|
|
115
|
+
}
|
|
116
|
+
export interface ComputerRuntimeOptions {
|
|
117
|
+
artifactDirectory: string;
|
|
118
|
+
adapter?: ComputerAdapter;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Reads secure-input state out of an `ioreg` dump of the console-session
|
|
122
|
+
* dictionary. `undefined` means the dump held no session at all, which is "not
|
|
123
|
+
* observable here" rather than "off".
|
|
124
|
+
*
|
|
125
|
+
* Exported for the test: the negative case is the one a developer machine
|
|
126
|
+
* naturally produces, so the positive case — a real password field taking the
|
|
127
|
+
* keyboard — can only be exercised against captured output.
|
|
128
|
+
*/
|
|
129
|
+
export declare function parseSecureInput(stdout: string): boolean | undefined;
|
|
130
|
+
/**
|
|
131
|
+
* `SendKeys` reads `+^%~(){}[]` as modifiers and grouping rather than as
|
|
132
|
+
* literal characters, so a password containing `+` types a Shift chord and a
|
|
133
|
+
* `(` opens a group that never closes. Each one is wrapped in braces, which is
|
|
134
|
+
* how `SendKeys` spells "the literal character".
|
|
135
|
+
*
|
|
136
|
+
* Exported because it is the part of the Windows adapter worth testing on any
|
|
137
|
+
* platform: the escaping is pure, and getting it wrong silently corrupts typed
|
|
138
|
+
* text rather than failing.
|
|
139
|
+
*/
|
|
140
|
+
export declare function escapeSendKeys(text: string): string;
|
|
141
|
+
/**
|
|
142
|
+
* A recorded desktop: what the platform reported, frozen to a file.
|
|
143
|
+
*
|
|
144
|
+
* Steps are keyed by action rather than ordered, because the runtime issues
|
|
145
|
+
* calls a caller never wrote — resolving a named `target` inspects first — and a
|
|
146
|
+
* positional script would drift the moment that happened.
|
|
147
|
+
*/
|
|
148
|
+
export interface ComputerTrace {
|
|
149
|
+
capabilities: ComputerCapabilities;
|
|
150
|
+
steps: Record<string, Record<string, unknown>>;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Replays a recorded desktop instead of touching the real one.
|
|
154
|
+
*
|
|
155
|
+
* Computer use is the one adapter whose evaluations cannot run honestly in CI:
|
|
156
|
+
* asserting what a click does means either taking hold of the mouse on the
|
|
157
|
+
* machine running the suite, or stopping at the approval and never testing what
|
|
158
|
+
* happens after it. Replaying a recording exercises the whole path — resolution,
|
|
159
|
+
* execution, verification, receipt — against a desktop that is identical on
|
|
160
|
+
* every machine.
|
|
161
|
+
*
|
|
162
|
+
* An action the recording does not contain is refused rather than defaulted, so
|
|
163
|
+
* a trace that has drifted from the scenario using it fails loudly.
|
|
164
|
+
*/
|
|
165
|
+
export declare class ReplayComputerAdapter implements ComputerAdapter {
|
|
166
|
+
private readonly trace;
|
|
167
|
+
constructor(trace: ComputerTrace);
|
|
168
|
+
capabilities(): Promise<ComputerCapabilities>;
|
|
169
|
+
execute(operation: ComputerOperation, _artifactDirectory: string, signal?: AbortSignal): Promise<Record<string, unknown>>;
|
|
170
|
+
}
|
|
171
|
+
export declare function createSystemComputerAdapter(): ComputerAdapter;
|
|
172
|
+
export declare class ComputerRuntime {
|
|
173
|
+
private readonly options;
|
|
174
|
+
private readonly adapter;
|
|
175
|
+
constructor(options: ComputerRuntimeOptions);
|
|
176
|
+
execute(operation: ComputerOperation, signal?: AbortSignal): Promise<Record<string, unknown>>;
|
|
177
|
+
/** The frontmost window's elements, or a refusal if the platform saw none. */
|
|
178
|
+
private elements;
|
|
179
|
+
/**
|
|
180
|
+
* An `inspect` observation, with words read off a screenshot when the platform
|
|
181
|
+
* reported no element tree.
|
|
182
|
+
*
|
|
183
|
+
* The fallback lives here rather than in each adapter so all four inherit it,
|
|
184
|
+
* and it only runs when the tree came back empty: a tree states what a control
|
|
185
|
+
* *is*, while a reading only says what it looks like, so the tree wins
|
|
186
|
+
* wherever there is one. Absence of a display width is a hard stop rather than
|
|
187
|
+
* an assumed scale — without it there is nothing to map a Retina capture back
|
|
188
|
+
* onto, and a wrong scale means a click that lands somewhere nobody named.
|
|
189
|
+
*/
|
|
190
|
+
private observe;
|
|
191
|
+
/** Takes a screenshot through the adapter and reads its words. */
|
|
192
|
+
private readScreen;
|
|
193
|
+
}
|
|
194
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAIzD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,OAAO,EAAE,cAAc,GAAG,eAAe,GAAG,oBAAoB,GAAG,aAAa,CAAC;IACjF,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,oEAAoE;IACpE,OAAO,EAAE,OAAO,CAAC;IACjB,gFAAgF;IAChF,QAAQ,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,GAAG,EAAE,OAAO,CAAC;IACb,0DAA0D;IAC1D,IAAI,EAAE,OAAO,CAAC;IACd,gBAAgB,EAAE,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,cAAc,EAAE,CAAC;CAC7B;AA6GD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,MAAM,EACX,YAAY,EAAE,MAAM,GACnB,cAAc,EAAE,CA8BlB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,cAAc,EAAE,EAC1B,MAAM,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GAC/D,cAAc,CAwChB;AA0BD,MAAM,WAAW,eAAe;IAC9B,YAAY,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC9C,OAAO,CACL,SAAS,EAAE,iBAAiB,EAC5B,iBAAiB,EAAE,MAAM,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AA8LD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAOpE;AAmcD;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AA+ZD;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,YAAY,EAAE,oBAAoB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAChD;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,qBAAsB,YAAW,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAAL,KAAK,EAAE,aAAa;IAE3C,YAAY,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAI7C,OAAO,CACX,SAAS,EAAE,iBAAiB,EAC5B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAQpC;AAED,wBAAgB,2BAA2B,IAAI,eAAe,CAK7D;AAED,qBAAa,eAAe;IAGd,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAEb,OAAO,EAAE,sBAAsB;IAItD,OAAO,CACX,SAAS,EAAE,iBAAiB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAqFnC,8EAA8E;YAChE,QAAQ;IAatB;;;;;;;;;;OAUG;YACW,OAAO;IAmBrB,kEAAkE;YACpD,UAAU;CAuBzB"}
|