@phone-use/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +103 -0
- package/dist/backend-Cbr2tIN-.d.mts +316 -0
- package/dist/device-BzPnHvQy.mjs +284 -0
- package/dist/device-BzPnHvQy.mjs.map +1 -0
- package/dist/index.d.mts +689 -0
- package/dist/index.mjs +1848 -0
- package/dist/index.mjs.map +1 -0
- package/dist/testing.d.mts +127 -0
- package/dist/testing.mjs +188 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +44 -0
- package/src/actions.ts +545 -0
- package/src/backend.ts +185 -0
- package/src/backends/agent-device.ts +242 -0
- package/src/backends/ios.ts +262 -0
- package/src/config.ts +43 -0
- package/src/device.ts +98 -0
- package/src/errors.ts +177 -0
- package/src/exec.ts +48 -0
- package/src/index.ts +86 -0
- package/src/lifecycle.ts +349 -0
- package/src/observe.ts +1093 -0
- package/src/secrets.ts +67 -0
- package/src/testing.ts +239 -0
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
import { A as Snapshot, C as AlertAction, D as PressTarget, E as OpenAppResult, O as Rect, S as ALL_CAPABILITIES, T as Capability, _ as PhoneUseErrorDetails, a as listBackends, b as UnsupportedCapabilityError, c as CommonDeviceConfig, d as AbortedError, f as ActionFailedError, g as PhoneUseErrorCode, h as PhoneUseError, i as getBackendFactory, j as SnapshotNode, k as ScrollDirection, l as DeviceConfig, m as DeviceNotFoundError, n as BaseDeviceBackend, o as registerBackend, p as DeviceInUseError, r as DeviceBackend, s as AndroidDeviceConfig, t as BackendFactory, u as IosDeviceConfig, v as SessionNotFoundError, w as BackendAlertResult, x as toPhoneUseError, y as TimeoutError } from "./backend-Cbr2tIN-.mjs";
|
|
2
|
+
//#region src/observe.d.ts
|
|
3
|
+
/** One compressed observation: the frontmost app plus the rendered element list. */
|
|
4
|
+
type Observation = {
|
|
5
|
+
/** Frontmost app name, when known. */
|
|
6
|
+
app?: string | undefined;
|
|
7
|
+
/** Frontmost app bundle id, when known. */
|
|
8
|
+
bundleId?: string | undefined;
|
|
9
|
+
/** Whether the element list was truncated. */
|
|
10
|
+
truncated: boolean;
|
|
11
|
+
/** The compressed, human/LLM-readable element listing. */
|
|
12
|
+
elements: string;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Post-action evidence from the driver's verify pass: whether the
|
|
16
|
+
* accessibility tree changed, without paying for a full follow-up snapshot.
|
|
17
|
+
*/
|
|
18
|
+
type ActionEvidence = {
|
|
19
|
+
/** Did the tree fingerprint change across the action. */
|
|
20
|
+
changed?: boolean | undefined;
|
|
21
|
+
/** Human-readable verdict detail. */
|
|
22
|
+
detail?: string | undefined;
|
|
23
|
+
};
|
|
24
|
+
/** The delta renderer's baseline: last rendered app, element keys, and lines. */
|
|
25
|
+
type RenderState = {
|
|
26
|
+
app?: string | undefined;
|
|
27
|
+
keys: string[];
|
|
28
|
+
lineByKey: Map<string, string>;
|
|
29
|
+
};
|
|
30
|
+
/** A structured interactive element extracted from the snapshot cache. */
|
|
31
|
+
type UiElement = {
|
|
32
|
+
/** Snapshot-scoped element ref (normalized to the `@N` form). */
|
|
33
|
+
ref: string;
|
|
34
|
+
/** Visible label (or accessibility identifier when the label is empty). */
|
|
35
|
+
label: string;
|
|
36
|
+
/** Semantic role, e.g. "Button", "Cell", "Switch". */
|
|
37
|
+
role: string;
|
|
38
|
+
/** Current value ("1"/"0" for switches, field contents, ...). */
|
|
39
|
+
value?: string | undefined;
|
|
40
|
+
/** On-screen geometry when known. */
|
|
41
|
+
rect?: Rect | undefined;
|
|
42
|
+
/** Accessibility identifier when the app exposes one. */
|
|
43
|
+
id?: string | undefined;
|
|
44
|
+
/** false when the element is disabled. */
|
|
45
|
+
enabled?: boolean | undefined;
|
|
46
|
+
/** Reason interaction is blocked, when the tree reports one. */
|
|
47
|
+
blocked?: string | undefined;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* Does a label match a query — by substring, or a strict punctuation/spacing-
|
|
51
|
+
* tolerant fuzzy match? Shared by the task layer's cached-map lookups so
|
|
52
|
+
* ask/toggle tolerate rewording the same way findElement does.
|
|
53
|
+
*/
|
|
54
|
+
declare function labelMatches(label: string, query: string): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Outcome of the resolution ladder. Exactly one of: a match (`el` set, `via`
|
|
57
|
+
* reporting the winning rung), an ambiguity (`el` null + `candidates` listing
|
|
58
|
+
* the distinct matches — the caller disambiguates with role/near), or a miss
|
|
59
|
+
* (`el` null, no candidates).
|
|
60
|
+
*/
|
|
61
|
+
type Resolution = {
|
|
62
|
+
/** The winning element, or null on ambiguity/miss. */
|
|
63
|
+
el: UiElement | null;
|
|
64
|
+
/** Match provenance — which rung won (id, exact label, substring, fuzzy). */
|
|
65
|
+
via?: string | undefined;
|
|
66
|
+
/** On ambiguity: the distinct elements that tied. */
|
|
67
|
+
candidates?: UiElement[] | undefined;
|
|
68
|
+
};
|
|
69
|
+
/** Disambiguators accepted by the resolution ladder. */
|
|
70
|
+
type ResolveOpts = {
|
|
71
|
+
/** Restrict matches to this role (case-insensitive). */
|
|
72
|
+
role?: string | undefined;
|
|
73
|
+
/** Label of another element; pick the candidate geometrically closest to it. */
|
|
74
|
+
near?: string | undefined;
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* One screen's worth of the resolution ladder: id exact → exact label →
|
|
78
|
+
* substring → fuzzy above a strict bar. This is the per-iteration body of
|
|
79
|
+
* resolveElement's scroll loop, extracted so callers holding a fresh cache
|
|
80
|
+
* (item-4 auto-wait) can match without scrolling. Returns `{ el: null }` with
|
|
81
|
+
* no candidates when no rung matched at all.
|
|
82
|
+
*/
|
|
83
|
+
declare function matchInElements(els: UiElement[], query: string, opts: ResolveOpts): Resolution;
|
|
84
|
+
/** Outcome of a system-alert interaction (see `DeviceCore.handleAlert`). */
|
|
85
|
+
type AlertOutcome = {
|
|
86
|
+
/** Was an alert showing at all. */
|
|
87
|
+
present: boolean;
|
|
88
|
+
/** Was it cleared (accept/dismiss actions only). */
|
|
89
|
+
handled?: boolean | undefined;
|
|
90
|
+
/** The button that was tapped, when handled. */
|
|
91
|
+
button?: string | undefined;
|
|
92
|
+
/** Title/message/buttons summary of the alert. */
|
|
93
|
+
description?: string | undefined;
|
|
94
|
+
};
|
|
95
|
+
/** Render any thrown value as a one-line message (appends `details.hint` when present). */
|
|
96
|
+
declare function describeError(error: unknown): string;
|
|
97
|
+
/**
|
|
98
|
+
* The device core: one instance = one device's observe/resolve/act state,
|
|
99
|
+
* driving one {@link DeviceBackend}. Everything here is portable — no
|
|
100
|
+
* runtime-specific globals and no image libraries — so it runs under Node.
|
|
101
|
+
* The harness's DeviceContext subclasses this and adds the cursor +
|
|
102
|
+
* live-viewer layer via the onCacheUpdated hook.
|
|
103
|
+
*/
|
|
104
|
+
declare class DeviceCore {
|
|
105
|
+
/** The backend this core drives. */
|
|
106
|
+
readonly backend: DeviceBackend;
|
|
107
|
+
protected cachedNodes: SnapshotNode[];
|
|
108
|
+
protected cachedViewport: {
|
|
109
|
+
width: number;
|
|
110
|
+
height: number;
|
|
111
|
+
};
|
|
112
|
+
protected lastApp: {
|
|
113
|
+
app?: string | undefined;
|
|
114
|
+
bundleId?: string | undefined;
|
|
115
|
+
};
|
|
116
|
+
private lastRender;
|
|
117
|
+
protected cacheAt: number;
|
|
118
|
+
constructor(backend: DeviceBackend);
|
|
119
|
+
protected onCacheUpdated(): void;
|
|
120
|
+
/**
|
|
121
|
+
* Canonical post-action report for LLM tool results, shared by every tool
|
|
122
|
+
* surface (agent tools + MCP): verdict from the action's own evidence, then a
|
|
123
|
+
* delta-rendered view of the screen it left behind.
|
|
124
|
+
*/
|
|
125
|
+
renderActionResult(prefix: string, evidence?: ActionEvidence, refresh?: boolean): Promise<string>;
|
|
126
|
+
/**
|
|
127
|
+
* Render the current cached screen for the LLM. full=true (or a structural
|
|
128
|
+
* change since last render) yields the complete compressed tree; otherwise a
|
|
129
|
+
* compact delta. Always updates the baseline.
|
|
130
|
+
*/
|
|
131
|
+
renderObservation(full?: boolean): string;
|
|
132
|
+
private cacheSnapshot;
|
|
133
|
+
private refreshCache;
|
|
134
|
+
private cacheSignature;
|
|
135
|
+
/** Milliseconds since the cache was last refreshed (Infinity before first). */
|
|
136
|
+
cacheAgeMs(): number;
|
|
137
|
+
/** Public fingerprint of the cached tree — the settle/verify signal. */
|
|
138
|
+
stateSignature(): string;
|
|
139
|
+
/** The cached screen as a compressed {@link Observation} (no new snapshot). */
|
|
140
|
+
currentElements(): Observation;
|
|
141
|
+
/**
|
|
142
|
+
* The frontmost app name from the last snapshot — cheap label without paying
|
|
143
|
+
* the full tree compression (used by the delta renderer's callers).
|
|
144
|
+
*/
|
|
145
|
+
currentApp(): string | undefined;
|
|
146
|
+
/**
|
|
147
|
+
* Structured interactive elements from the current cache — the crawler taps
|
|
148
|
+
* these by label (refs are only valid within one snapshot).
|
|
149
|
+
*/
|
|
150
|
+
interactiveElements(): UiElement[];
|
|
151
|
+
/**
|
|
152
|
+
* Editable text inputs from the current cache. These roles are deliberately
|
|
153
|
+
* excluded from interactiveElements() (they aren't "tap" targets), so the
|
|
154
|
+
* input primitive needs its own accessor to find a search bar / text field to
|
|
155
|
+
* focus. includeMultiline adds TextView bodies for form/compose filling.
|
|
156
|
+
*/
|
|
157
|
+
inputFields(includeMultiline?: boolean): UiElement[];
|
|
158
|
+
/**
|
|
159
|
+
* Run the resolution ladder against the current cache only — no scrolling,
|
|
160
|
+
* no fresh snapshot. resolveElement drives this per scroll step.
|
|
161
|
+
*/
|
|
162
|
+
resolveInCache(query: string, opts?: ResolveOpts): Resolution;
|
|
163
|
+
/** The full ladder: scroll to top, then match + scroll down until found or stable. */
|
|
164
|
+
resolveElement(query: string, opts?: ResolveOpts): Promise<Resolution>;
|
|
165
|
+
/**
|
|
166
|
+
* Compatibility wrapper: single best element or null (read paths — ask/read a
|
|
167
|
+
* value — where picking the first match is low-risk). Tap paths use
|
|
168
|
+
* resolveElement directly and honor the ambiguity contract.
|
|
169
|
+
*/
|
|
170
|
+
findElement(labelSubstring: string): Promise<UiElement | null>;
|
|
171
|
+
/**
|
|
172
|
+
* Read a labeled value. iOS list rows fold the value into the label
|
|
173
|
+
* ("iOS Version, 26.1") or expose it as a Switch value ("1"/"0"); handle both.
|
|
174
|
+
*/
|
|
175
|
+
readField(labelSubstring: string): Promise<string | null>;
|
|
176
|
+
/**
|
|
177
|
+
* A structural fingerprint of the current screen that is stable across
|
|
178
|
+
* dynamic content (times, battery, values) — it keys the crawler's graph
|
|
179
|
+
* nodes so the same screen is recognized regardless of transient text.
|
|
180
|
+
*/
|
|
181
|
+
screenSignature(): string;
|
|
182
|
+
/** Navigation-bar title of the cached screen ('' when absent). */
|
|
183
|
+
screenTitle(): string;
|
|
184
|
+
/** Take one fresh snapshot into the cache and return the compressed observation. */
|
|
185
|
+
observe(): Promise<Observation>;
|
|
186
|
+
/**
|
|
187
|
+
* Open an app by name/bundle id. relaunch forces a fresh launch (clean
|
|
188
|
+
* initial screen) instead of just foregrounding — iOS keeps an app's
|
|
189
|
+
* navigation state across foregrounding, so primitives that need a known
|
|
190
|
+
* starting screen pass relaunch=true.
|
|
191
|
+
*/
|
|
192
|
+
openApp(app: string, relaunch?: boolean): Promise<string>;
|
|
193
|
+
/**
|
|
194
|
+
* Level-2 of the action ladder: deep links beat tap sequences when a URL route
|
|
195
|
+
* exists (maps://, app schemes, https:// universal links). XCTest sessions are
|
|
196
|
+
* app-scoped, so a link that opens a different app must re-scope the session
|
|
197
|
+
* to that app or observations keep tracking the old one.
|
|
198
|
+
*/
|
|
199
|
+
openUrl(url: string, app?: string): Promise<string>;
|
|
200
|
+
private currentBundleId;
|
|
201
|
+
/** List installed app bundle ids. */
|
|
202
|
+
listApps(): Promise<string[]>;
|
|
203
|
+
private tapAndDiff;
|
|
204
|
+
/** Tap an element ref, self-diffing the cache to report whether the screen changed. */
|
|
205
|
+
press(ref: string): Promise<ActionEvidence>;
|
|
206
|
+
private visibleMidpoint;
|
|
207
|
+
protected onScreen(rect?: Rect): boolean;
|
|
208
|
+
/**
|
|
209
|
+
* Open an app and walk its nav stack back to the root (dismissing modals), so
|
|
210
|
+
* map-based navigation always starts from a known origin.
|
|
211
|
+
*/
|
|
212
|
+
goToRoot(app: string): Promise<void>;
|
|
213
|
+
/** Height of the cached viewport in points. */
|
|
214
|
+
viewportHeight(): number;
|
|
215
|
+
/**
|
|
216
|
+
* Vertical span of interactive content in the current cache. Used to decide
|
|
217
|
+
* whether scrolling is even necessary — scroll gestures cost ~2s each, so
|
|
218
|
+
* skipping them on screens that already fit is the single biggest crawl
|
|
219
|
+
* speedup.
|
|
220
|
+
*/
|
|
221
|
+
contentBounds(): {
|
|
222
|
+
minY: number;
|
|
223
|
+
maxY: number;
|
|
224
|
+
};
|
|
225
|
+
/**
|
|
226
|
+
* Tapping the status bar scrolls the active scroll view to the top — native
|
|
227
|
+
* iOS behavior, one fast tap instead of multiple ~2s scroll gestures. Falls
|
|
228
|
+
* back to gesture scrolling if the tap doesn't take.
|
|
229
|
+
*/
|
|
230
|
+
scrollToTop(): Promise<void>;
|
|
231
|
+
/**
|
|
232
|
+
* Tap an element by its label, scrolling it into view first if it's
|
|
233
|
+
* off-screen. The crawler and the map navigator use this so a target below
|
|
234
|
+
* the fold (a long Settings list) is still reachable. Re-resolves the ref
|
|
235
|
+
* after each scroll.
|
|
236
|
+
*/
|
|
237
|
+
tapLabel(label: string): Promise<boolean>;
|
|
238
|
+
/**
|
|
239
|
+
* Vision-path fallback: tap raw coordinates when the accessibility tree is
|
|
240
|
+
* missing or wrong (canvas, games, custom controls). Coordinates are in the
|
|
241
|
+
* same space as observe()'s rects and the screenshot pixels (@1x points).
|
|
242
|
+
*/
|
|
243
|
+
pressAt(x: number, y: number): Promise<ActionEvidence>;
|
|
244
|
+
/**
|
|
245
|
+
* Coordinate drag: touch down at (x,y), move by (dx,dy). The primitive for
|
|
246
|
+
* controls a tap can't operate — picker wheels (drag vertically on the wheel
|
|
247
|
+
* column), sliders, and custom carousels. Same coordinate space as rects.
|
|
248
|
+
*/
|
|
249
|
+
pan(x: number, y: number, dx: number, dy: number, durationMs?: number): Promise<void>;
|
|
250
|
+
/**
|
|
251
|
+
* Set-of-Marks visual observation: screenshot with `@ref` labels drawn on the
|
|
252
|
+
* elements, so a vision model can ground itself in pixels and still act by ref.
|
|
253
|
+
*/
|
|
254
|
+
screenshotWithRefs(path: string): Promise<string>;
|
|
255
|
+
protected currentViewport(): Promise<{
|
|
256
|
+
width: number;
|
|
257
|
+
height: number;
|
|
258
|
+
}>;
|
|
259
|
+
/** Long-press an element ref. */
|
|
260
|
+
longPress(ref: string, durationMs?: number): Promise<void>;
|
|
261
|
+
/** Focus a field and replace its text, self-diffing the cache for evidence. */
|
|
262
|
+
fill(ref: string, text: string): Promise<ActionEvidence>;
|
|
263
|
+
/** Type into whatever currently has keyboard focus. */
|
|
264
|
+
typeText(text: string): Promise<void>;
|
|
265
|
+
/**
|
|
266
|
+
* Press the keyboard's return/go key. Submits a search bar that acts on
|
|
267
|
+
* Return (Safari's address bar, web forms) rather than filtering results as
|
|
268
|
+
* you type.
|
|
269
|
+
*/
|
|
270
|
+
pressReturn(): Promise<void>;
|
|
271
|
+
/** Scroll the active scroll view one step. */
|
|
272
|
+
scroll(direction: 'up' | 'down' | 'left' | 'right'): Promise<void>;
|
|
273
|
+
/** Block until `text` appears on screen; returns a confirmation note. */
|
|
274
|
+
waitForText(text: string, timeoutMs?: number): Promise<string>;
|
|
275
|
+
private alertFromCache;
|
|
276
|
+
/**
|
|
277
|
+
* System dialogs (permissions, sign-in prompts) block everything else; the
|
|
278
|
+
* driver exposes them as a first-class action instead of hoping a tap lands.
|
|
279
|
+
*/
|
|
280
|
+
handleAlert(action: 'get' | 'accept' | 'dismiss'): Promise<AlertOutcome>;
|
|
281
|
+
/**
|
|
282
|
+
* Clear the launch permission gauntlet — real apps stack location /
|
|
283
|
+
* notification / tracking prompts on first open, each blocking the app.
|
|
284
|
+
* Grants by default so the crawl sees the most surface. Returns the buttons
|
|
285
|
+
* tapped. Bounded so a non-clearing dialog can't loop forever.
|
|
286
|
+
*/
|
|
287
|
+
clearBlockingAlerts(action?: 'accept' | 'dismiss', max?: number): Promise<string[]>;
|
|
288
|
+
/** Go to the home screen. */
|
|
289
|
+
goHome(): Promise<void>;
|
|
290
|
+
/** Navigate back (nav-bar back / hardware back). */
|
|
291
|
+
goBack(): Promise<void>;
|
|
292
|
+
/** Save a screenshot to `path`; returns the written path. */
|
|
293
|
+
screenshot(path: string): Promise<string>;
|
|
294
|
+
/** Close the backend's transport session. */
|
|
295
|
+
closeSession(): Promise<void>;
|
|
296
|
+
protected findNode(ref: string): SnapshotNode | undefined;
|
|
297
|
+
protected ensureCache(): Promise<void>;
|
|
298
|
+
}
|
|
299
|
+
//#endregion
|
|
300
|
+
//#region src/secrets.d.ts
|
|
301
|
+
/**
|
|
302
|
+
* `%variable%` secret substitution (docs/19 §API shape, Stagehand's pattern):
|
|
303
|
+
* the model/caller plans against NAMES; values are injected at the last moment
|
|
304
|
+
* before backend.fill/typeText and never rendered into Actions, results,
|
|
305
|
+
* observations, or traces. Redaction is best-effort belt-and-braces; the hard
|
|
306
|
+
* guarantee is at the substitution point — values never enter stored Actions
|
|
307
|
+
* by construction.
|
|
308
|
+
*/
|
|
309
|
+
declare class SecretStore {
|
|
310
|
+
private readonly values;
|
|
311
|
+
constructor(values?: Record<string, string>);
|
|
312
|
+
/**
|
|
313
|
+
* Store a secret under `name`. Rejects values shorter than 4 chars — a
|
|
314
|
+
* 2-char secret would redact innocent UI text everywhere.
|
|
315
|
+
*/
|
|
316
|
+
set(name: string, value: string): void;
|
|
317
|
+
/** The stored secret NAMES (never the values). */
|
|
318
|
+
names(): string[];
|
|
319
|
+
/** %name% → value. Unknown %x% stays literal. */
|
|
320
|
+
substitute(text: string): string;
|
|
321
|
+
/** value → %name% across outbound text (messages, rendered observations). */
|
|
322
|
+
redact(text: string): string;
|
|
323
|
+
/** Per-call vars layered over the store (call-scoped, never persisted). */
|
|
324
|
+
withOverrides(vars?: Record<string, string>): SecretStore;
|
|
325
|
+
}
|
|
326
|
+
//#endregion
|
|
327
|
+
//#region src/actions.d.ts
|
|
328
|
+
/**
|
|
329
|
+
* A RE-RESOLVABLE element query — how a portable {@link Action} names its
|
|
330
|
+
* target. Resolved against the live tree by the resolution ladder at act()
|
|
331
|
+
* time; no stale handles.
|
|
332
|
+
*/
|
|
333
|
+
type ElementQuery = {
|
|
334
|
+
/** Label to match (exact → substring → fuzzy, the ladder's rungs). */
|
|
335
|
+
label?: string | undefined;
|
|
336
|
+
/** Accessibility identifier — rung 0 of the ladder, wins when present. */
|
|
337
|
+
id?: string | undefined;
|
|
338
|
+
/** Disambiguator: restrict matches to this role (e.g. "Button"). */
|
|
339
|
+
role?: string | undefined;
|
|
340
|
+
/** Disambiguator: label of another element; pick the geometrically closest match. */
|
|
341
|
+
near?: string | undefined;
|
|
342
|
+
};
|
|
343
|
+
/** Verbs a portable {@link Action} can carry. */
|
|
344
|
+
type ActionVerb = 'tap' | 'longPress' | 'fill' | 'type' | 'pressKey' | 'scroll' | 'openApp' | 'openUrl' | 'back' | 'home' | 'alert' | 'waitForText';
|
|
345
|
+
/**
|
|
346
|
+
* The observe→act seam (docs/19 §API shape; docs/20 item 4): a portable action
|
|
347
|
+
* descriptor carrying a re-resolvable {@link ElementQuery}. `observe()` returns
|
|
348
|
+
* these; `act()` re-resolves against the live tree and executes with no
|
|
349
|
+
* re-inference. A compiled skill is a stored `Action[]`.
|
|
350
|
+
*/
|
|
351
|
+
type Action = {
|
|
352
|
+
/** Versioned, documented UNSTABLE pre-1.0 (docs/20 open-question 3). */
|
|
353
|
+
formatVersion: 0;
|
|
354
|
+
/** What to do. */
|
|
355
|
+
verb: ActionVerb;
|
|
356
|
+
/** Target query for element-directed verbs (tap/longPress/fill). */
|
|
357
|
+
target?: ElementQuery | undefined;
|
|
358
|
+
/** Verb parameters (text, direction, app, url, ...). */
|
|
359
|
+
params?: {
|
|
360
|
+
/** fill/type text — may contain %name% secret references. */
|
|
361
|
+
text?: string | undefined;
|
|
362
|
+
direction?: ScrollDirection | undefined;
|
|
363
|
+
app?: string | undefined;
|
|
364
|
+
url?: string | undefined;
|
|
365
|
+
durationMs?: number | undefined;
|
|
366
|
+
key?: 'return' | undefined;
|
|
367
|
+
alertAction?: 'accept' | 'dismiss' | undefined;
|
|
368
|
+
submit?: boolean | undefined;
|
|
369
|
+
relaunch?: boolean | undefined;
|
|
370
|
+
} | undefined;
|
|
371
|
+
/**
|
|
372
|
+
* Provenance from observe/record time. ADVISORY ONLY — act() always
|
|
373
|
+
* re-resolves; this exists for traces, drift diagnosis, and human review.
|
|
374
|
+
*/
|
|
375
|
+
observed?: {
|
|
376
|
+
via?: string | undefined;
|
|
377
|
+
label?: string | undefined;
|
|
378
|
+
role?: string | undefined;
|
|
379
|
+
rect?: Rect | undefined;
|
|
380
|
+
app?: string | undefined;
|
|
381
|
+
screenTitle?: string | undefined;
|
|
382
|
+
} | undefined;
|
|
383
|
+
};
|
|
384
|
+
/** The stored-Action[] artifact (public TYPE, unstable FORMAT — see docs/20). */
|
|
385
|
+
type CompiledSkill = {
|
|
386
|
+
formatVersion: 0;
|
|
387
|
+
name: string;
|
|
388
|
+
description?: string | undefined;
|
|
389
|
+
params?: string[] | undefined;
|
|
390
|
+
precondition?: string | undefined;
|
|
391
|
+
actions: Action[];
|
|
392
|
+
};
|
|
393
|
+
/** An interactive element as surfaced by `observe()` (alias of {@link UiElement}). */
|
|
394
|
+
type ObservedElement = UiElement;
|
|
395
|
+
/** What `observe()` returns: elements, rendered text, and portable actions. */
|
|
396
|
+
type ObserveResult = {
|
|
397
|
+
/** Always true for a completed observation. */
|
|
398
|
+
success: boolean;
|
|
399
|
+
/** Human-readable summary of the observation. */
|
|
400
|
+
message: string;
|
|
401
|
+
/** Frontmost app name, when known. */
|
|
402
|
+
app?: string | undefined;
|
|
403
|
+
/** Frontmost app bundle id, when known. */
|
|
404
|
+
bundleId?: string | undefined;
|
|
405
|
+
/** Navigation-bar title of the current screen, when present. */
|
|
406
|
+
screenTitle?: string | undefined;
|
|
407
|
+
/** Structured channel (secret-redacted values). */
|
|
408
|
+
elements: ObservedElement[];
|
|
409
|
+
/** Compressed text channel (secret-redacted). */
|
|
410
|
+
rendered: string;
|
|
411
|
+
/** Portable descriptors: a tap per tappable, a fill per input field. */
|
|
412
|
+
actions: Action[];
|
|
413
|
+
};
|
|
414
|
+
/**
|
|
415
|
+
* Structured outcome of every action verb. Never-throw contract: device-legible
|
|
416
|
+
* failures (not found, ambiguous, wait deadline, gesture failure) come back as
|
|
417
|
+
* `{success: false, ...}`; only infrastructure errors (abort, closed session,
|
|
418
|
+
* unsupported capability) throw PhoneUseError subclasses.
|
|
419
|
+
*/
|
|
420
|
+
type ActionResult = {
|
|
421
|
+
/** Did the action execute as intended. */
|
|
422
|
+
success: boolean;
|
|
423
|
+
/** Human-readable outcome (secret-redacted). */
|
|
424
|
+
message: string;
|
|
425
|
+
/** tapAndDiff verdict where applicable: did the screen actually change. */
|
|
426
|
+
changed?: boolean | undefined;
|
|
427
|
+
/** How the target resolved (match provenance, ref, label, role, rect). */
|
|
428
|
+
resolved?: {
|
|
429
|
+
via: string;
|
|
430
|
+
ref: string;
|
|
431
|
+
label: string;
|
|
432
|
+
role: string;
|
|
433
|
+
rect?: Rect | undefined;
|
|
434
|
+
} | undefined;
|
|
435
|
+
/** The ambiguity contract, surfaced structurally. */
|
|
436
|
+
candidates?: ObservedElement[] | undefined;
|
|
437
|
+
/** Auto-wait cost when the slow path ran (elapsed ms, poll count). */
|
|
438
|
+
waited?: {
|
|
439
|
+
ms: number;
|
|
440
|
+
polls: number;
|
|
441
|
+
} | undefined;
|
|
442
|
+
/** Set on structured failures with an error flavor (e.g. TIMEOUT). */
|
|
443
|
+
code?: PhoneUseErrorCode | undefined;
|
|
444
|
+
};
|
|
445
|
+
/** Per-call options accepted by every action verb. */
|
|
446
|
+
type ActOptions = {
|
|
447
|
+
/** Abort the call; the in-flight gesture may still land (state indeterminate). */
|
|
448
|
+
signal?: AbortSignal | undefined;
|
|
449
|
+
/** Auto-wait deadline (default 5000 ms). */
|
|
450
|
+
timeoutMs?: number | undefined;
|
|
451
|
+
/** Per-call secret overrides, layered on the device store. */
|
|
452
|
+
vars?: Record<string, string> | undefined;
|
|
453
|
+
};
|
|
454
|
+
/** Synthesize portable actions from the CURRENT cache (observe-time). */
|
|
455
|
+
declare function toActions(core: DeviceCore): Action[];
|
|
456
|
+
/**
|
|
457
|
+
* Take one fresh snapshot and assemble the full {@link ObserveResult}:
|
|
458
|
+
* deduped element channel, secret-redacted rendered text, and a portable
|
|
459
|
+
* Action per tappable / input field.
|
|
460
|
+
*/
|
|
461
|
+
declare function buildObserveResult(core: DeviceCore, secrets?: SecretStore): Promise<ObserveResult>;
|
|
462
|
+
/**
|
|
463
|
+
* THE dispatcher: resolve → auto-wait → execute → diff. One brain — used by
|
|
464
|
+
* device.tap/type/act, and (items 5-6) by the harness and the skill runner.
|
|
465
|
+
*/
|
|
466
|
+
declare function executeAction(core: DeviceCore, action: Action, opts?: ActOptions, secrets?: SecretStore): Promise<ActionResult>;
|
|
467
|
+
//#endregion
|
|
468
|
+
//#region src/backends/agent-device.d.ts
|
|
469
|
+
/**
|
|
470
|
+
* Build the agent-device backend: the ONE place agent-device is called. Device
|
|
471
|
+
* pinning is per-request in agent-device, so the backend holds a selection
|
|
472
|
+
* object and spreads it into every call. With no config, selection is `{}` and
|
|
473
|
+
* the client is default-constructed — requests are byte-identical to the
|
|
474
|
+
* pre-seam process-global path (booted-sim auto-detect). Every method
|
|
475
|
+
* normalizes errors via `toPhoneUseError`; no agent-device type or error ever
|
|
476
|
+
* escapes this module.
|
|
477
|
+
*/
|
|
478
|
+
declare function createAgentDeviceBackend(config?: DeviceConfig): DeviceBackend;
|
|
479
|
+
//#endregion
|
|
480
|
+
//#region src/lifecycle.d.ts
|
|
481
|
+
/** The platform a {@link Device} runs on. */
|
|
482
|
+
type DevicePlatform = 'ios' | 'android';
|
|
483
|
+
/** Lifecycle state of a {@link Device} handle. */
|
|
484
|
+
type DeviceStatus = 'running' | 'closed';
|
|
485
|
+
/**
|
|
486
|
+
* The Device lifecycle handle (docs/20 item 3; docs/19 §Lifecycle): id, pinned
|
|
487
|
+
* backend, close/dispose, idle lease + reaper — plus the action verb surface
|
|
488
|
+
* (item 4) layered onto the same type. `ios.launch()` and `ios.connect()`
|
|
489
|
+
* return it; the android engine (item 7b) will share `createDeviceHandle`.
|
|
490
|
+
*/
|
|
491
|
+
interface Device {
|
|
492
|
+
/** udid (iOS) / serial (Android). */
|
|
493
|
+
readonly id: string;
|
|
494
|
+
/** Which platform this device runs. */
|
|
495
|
+
readonly platform: DevicePlatform;
|
|
496
|
+
/** Simulator/device name when known. */
|
|
497
|
+
readonly name?: string | undefined;
|
|
498
|
+
/** Name of the backend driving this device. */
|
|
499
|
+
readonly backendName: string;
|
|
500
|
+
/** The backend's declared capability set. */
|
|
501
|
+
readonly capabilities: ReadonlySet<Capability>;
|
|
502
|
+
/**
|
|
503
|
+
* The pinned backend — `new DeviceContext(device.backend)` works today.
|
|
504
|
+
* Backend calls through this handle count as activity for the idle lease.
|
|
505
|
+
*/
|
|
506
|
+
readonly backend: DeviceBackend;
|
|
507
|
+
/** true when launch() created the device — close() then also deletes it. */
|
|
508
|
+
readonly createdByUs: boolean;
|
|
509
|
+
/** Current lifecycle state. */
|
|
510
|
+
readonly status: DeviceStatus;
|
|
511
|
+
/** Sugar for `status === 'closed'`. */
|
|
512
|
+
readonly isClosed: boolean;
|
|
513
|
+
/** Re-arm the idle lease (ms overrides the configured window for this arm only). */
|
|
514
|
+
extendLease(ms?: number): void;
|
|
515
|
+
/** Canonical, idempotent shutdown. `await using` is sugar over this. */
|
|
516
|
+
close(): Promise<void>;
|
|
517
|
+
/** `await using` support — delegates to {@link Device.close}. */
|
|
518
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
519
|
+
/** Look at the screen: elements + rendered text + portable Action[]. */
|
|
520
|
+
observe(opts?: {
|
|
521
|
+
signal?: AbortSignal | undefined;
|
|
522
|
+
}): Promise<ObserveResult>;
|
|
523
|
+
/** Tap by label/id query. Auto-waits; never throws for normal outcomes. */
|
|
524
|
+
tap(target: string | ElementQuery, opts?: ActOptions): Promise<ActionResult>;
|
|
525
|
+
/** Type text (optionally into a field resolved by query); %name% secrets substituted. */
|
|
526
|
+
type(text: string, opts?: ActOptions & {
|
|
527
|
+
field?: string | ElementQuery | undefined;
|
|
528
|
+
submit?: boolean | undefined;
|
|
529
|
+
}): Promise<ActionResult>;
|
|
530
|
+
/** Execute a portable Action deterministically — no re-inference. */
|
|
531
|
+
act(action: Action, opts?: ActOptions): Promise<ActionResult>;
|
|
532
|
+
/** App management: open by name/deep link, list installed, current app. */
|
|
533
|
+
readonly apps: {
|
|
534
|
+
/** Open an app by name/bundle id, or a deep link when `url` is set. */
|
|
535
|
+
open(app: string, opts?: {
|
|
536
|
+
relaunch?: boolean | undefined;
|
|
537
|
+
url?: string | undefined;
|
|
538
|
+
signal?: AbortSignal | undefined;
|
|
539
|
+
}): Promise<ActionResult>;
|
|
540
|
+
/** List installed app bundle ids. */
|
|
541
|
+
list(opts?: {
|
|
542
|
+
signal?: AbortSignal | undefined;
|
|
543
|
+
}): Promise<string[]>;
|
|
544
|
+
/** The frontmost app name from the last observation (no new snapshot). */
|
|
545
|
+
current(): string | undefined;
|
|
546
|
+
};
|
|
547
|
+
/** Screen-level verbs: scroll, screenshot, waitForText, alert, back, home. */
|
|
548
|
+
readonly screen: {
|
|
549
|
+
/** Scroll the active scroll view one step. */
|
|
550
|
+
scroll(direction: ScrollDirection, opts?: {
|
|
551
|
+
signal?: AbortSignal | undefined;
|
|
552
|
+
}): Promise<ActionResult>;
|
|
553
|
+
/** Save a screenshot to `path`. */
|
|
554
|
+
screenshot(opts: {
|
|
555
|
+
path: string;
|
|
556
|
+
signal?: AbortSignal | undefined;
|
|
557
|
+
}): Promise<{
|
|
558
|
+
success: boolean;
|
|
559
|
+
message: string;
|
|
560
|
+
path?: string | undefined;
|
|
561
|
+
}>;
|
|
562
|
+
/** Block until `text` appears on screen or the timeout elapses. */
|
|
563
|
+
waitForText(text: string, opts?: {
|
|
564
|
+
timeoutMs?: number | undefined;
|
|
565
|
+
signal?: AbortSignal | undefined;
|
|
566
|
+
}): Promise<ActionResult>;
|
|
567
|
+
/** Read ('get'), accept, or dismiss a blocking system alert. */
|
|
568
|
+
alert(action: 'get' | 'accept' | 'dismiss', opts?: {
|
|
569
|
+
signal?: AbortSignal | undefined;
|
|
570
|
+
}): Promise<ActionResult>;
|
|
571
|
+
/** Navigate back. */
|
|
572
|
+
back(opts?: {
|
|
573
|
+
signal?: AbortSignal | undefined;
|
|
574
|
+
}): Promise<ActionResult>;
|
|
575
|
+
/** Go to the home screen. */
|
|
576
|
+
home(opts?: {
|
|
577
|
+
signal?: AbortSignal | undefined;
|
|
578
|
+
}): Promise<ActionResult>;
|
|
579
|
+
};
|
|
580
|
+
/** %name% secret store — values substituted at execution, redacted everywhere else. */
|
|
581
|
+
readonly secrets: SecretStore;
|
|
582
|
+
}
|
|
583
|
+
/** Inputs to {@link createDeviceHandle} — what an engine supplies per device. */
|
|
584
|
+
type CreateDeviceHandleOptions = {
|
|
585
|
+
/** udid (iOS) / serial (Android). */
|
|
586
|
+
id: string;
|
|
587
|
+
/** Which platform the device runs. */
|
|
588
|
+
platform: DevicePlatform;
|
|
589
|
+
/** Simulator/device name when known. */
|
|
590
|
+
name?: string | undefined;
|
|
591
|
+
/** The backend pinned to this device. */
|
|
592
|
+
backend: DeviceBackend;
|
|
593
|
+
/** true when the engine created the device (close() then also deletes it). */
|
|
594
|
+
createdByUs: boolean;
|
|
595
|
+
/** Idle window in ms; false disables the lease. Default 180_000 (3m). */
|
|
596
|
+
idleTimeoutMs?: number | false | undefined;
|
|
597
|
+
/** Observer for reaper-initiated closes (the SDK never logs). */
|
|
598
|
+
onIdleClose?: ((device: Device) => void) | undefined;
|
|
599
|
+
/** Initial %name% secret values. */
|
|
600
|
+
secrets?: Record<string, string> | undefined;
|
|
601
|
+
/** Platform teardown: shutdown (+ delete when createdByUs). */
|
|
602
|
+
doClose: () => Promise<void>;
|
|
603
|
+
};
|
|
604
|
+
/**
|
|
605
|
+
* Assemble a Device handle over a backend: lease/reaper, verb surface,
|
|
606
|
+
* close/dispose semantics. Engine authors (ios here, android in item 7b,
|
|
607
|
+
* phone-backend-* third parties) build on this; tests fabricate devices with
|
|
608
|
+
* it over a FakeBackend.
|
|
609
|
+
*/
|
|
610
|
+
declare function createDeviceHandle(opts: CreateDeviceHandleOptions): Device;
|
|
611
|
+
//#endregion
|
|
612
|
+
//#region src/backends/ios.d.ts
|
|
613
|
+
type CommonIosOptions = {
|
|
614
|
+
/** Custom simulator device set directory (maps to `simctl --set`). */
|
|
615
|
+
simulatorDeviceSet?: string | undefined;
|
|
616
|
+
/** agent-device session/daemon pinning, passed through to the backend. */
|
|
617
|
+
session?: string | undefined;
|
|
618
|
+
daemonBaseUrl?: string | undefined;
|
|
619
|
+
daemonAuthToken?: string | undefined;
|
|
620
|
+
/** Idle lease window in ms (false disables). Default 180_000 (3 min). */
|
|
621
|
+
idleTimeoutMs?: number | false | undefined;
|
|
622
|
+
/** Observer for reaper-initiated closes. */
|
|
623
|
+
onIdleClose?: ((device: Device) => void) | undefined;
|
|
624
|
+
/** Initial %name% secret values (see Device.secrets). */
|
|
625
|
+
secrets?: Record<string, string> | undefined;
|
|
626
|
+
/**
|
|
627
|
+
* Probe the agent-device daemon right away (one listApps) so a missing
|
|
628
|
+
* daemon fails at launch instead of on first use. Default false: sessions
|
|
629
|
+
* open lazily and a probe requires the daemon to exist.
|
|
630
|
+
*/
|
|
631
|
+
failFast?: boolean | undefined;
|
|
632
|
+
};
|
|
633
|
+
/** Options for `ios.launch()` — device type, runtime, name, boot ceiling. */
|
|
634
|
+
type IosLaunchOptions = CommonIosOptions & {
|
|
635
|
+
/** simctl device type, e.g. "iPhone 16" (the default). */
|
|
636
|
+
deviceType?: string | undefined;
|
|
637
|
+
/** simctl runtime id; omitted → newest compatible. */
|
|
638
|
+
runtime?: string | undefined;
|
|
639
|
+
/** Simulator name; default `phone-use-<hex>` (the orphan-discovery prefix). */
|
|
640
|
+
name?: string | undefined;
|
|
641
|
+
/** Boot wait ceiling for `simctl bootstatus` (default 120_000 ms). */
|
|
642
|
+
bootTimeoutMs?: number | undefined;
|
|
643
|
+
};
|
|
644
|
+
/** Options for `ios.connect()`. */
|
|
645
|
+
type IosConnectOptions = CommonIosOptions & {
|
|
646
|
+
/** Boot wait ceiling when connect has to boot a shut-down sim (default 120_000 ms). */
|
|
647
|
+
bootTimeoutMs?: number | undefined;
|
|
648
|
+
};
|
|
649
|
+
/**
|
|
650
|
+
* Create and boot a DEDICATED simulator via simctl — no more "whatever is
|
|
651
|
+
* booted" — and return a {@link Device} pinned to its udid. Created sims are
|
|
652
|
+
* named `phone-use-<hex>` deliberately: if the process is kill -9'd the
|
|
653
|
+
* in-process reaper can't run, and the name prefix is how orphans are found.
|
|
654
|
+
* `close()` shuts the sim down AND deletes it (we created it); a failed boot
|
|
655
|
+
* best-effort-deletes before rethrowing.
|
|
656
|
+
*/
|
|
657
|
+
declare function launch(options?: IosLaunchOptions): Promise<Device>;
|
|
658
|
+
/**
|
|
659
|
+
* Reattach to an existing simulator by udid (booting it if shut down). The
|
|
660
|
+
* no-arg form is the sole survivor of the old booted-sim auto-detect: it
|
|
661
|
+
* attaches to the first booted, available sim. `close()` on a connected
|
|
662
|
+
* device shuts it down but never deletes it.
|
|
663
|
+
*/
|
|
664
|
+
declare function connect(udid?: string, options?: IosConnectOptions): Promise<Device>;
|
|
665
|
+
/**
|
|
666
|
+
* The iOS engine object (Playwright-style): `ios.launch()` for a dedicated
|
|
667
|
+
* simulator, `ios.connect()` to reattach. Both return the same Device type.
|
|
668
|
+
*/
|
|
669
|
+
declare const ios: {
|
|
670
|
+
/** Create + boot a dedicated simulator and return a Device pinned to it. */
|
|
671
|
+
readonly launch: typeof launch;
|
|
672
|
+
/** Reattach to an existing simulator (no-arg: first booted sim). */
|
|
673
|
+
readonly connect: typeof connect;
|
|
674
|
+
};
|
|
675
|
+
//#endregion
|
|
676
|
+
//#region src/index.d.ts
|
|
677
|
+
/**
|
|
678
|
+
* @phone-use/sdk — the device runtime SDK: engine-as-object lifecycle
|
|
679
|
+
* (ios.launch/connect → Device), Device backends, config, errors, capabilities
|
|
680
|
+
* (docs/20-runtime-sdk-v1-plan.md items 2-3; action verbs land in item 4).
|
|
681
|
+
*
|
|
682
|
+
* The test double (FakeBackend) lives on the "@phone-use/sdk/testing" subpath,
|
|
683
|
+
* deliberately not re-exported here.
|
|
684
|
+
*/
|
|
685
|
+
/** The published package version (kept in sync with package.json by the release flow). */
|
|
686
|
+
declare const VERSION = "0.1.0";
|
|
687
|
+
//#endregion
|
|
688
|
+
export { ALL_CAPABILITIES, AbortedError, type ActOptions, type Action, type ActionEvidence, ActionFailedError, type ActionResult, type ActionVerb, type AlertAction, type AlertOutcome, type AndroidDeviceConfig, type BackendAlertResult, type BackendFactory, BaseDeviceBackend, type Capability, type CommonDeviceConfig, type CompiledSkill, type CreateDeviceHandleOptions, type Device, type DeviceBackend, type DeviceConfig, DeviceCore, DeviceInUseError, DeviceNotFoundError, type DevicePlatform, type DeviceStatus, type ElementQuery, type IosConnectOptions, type IosDeviceConfig, type IosLaunchOptions, type Observation, type ObserveResult, type ObservedElement, type OpenAppResult, PhoneUseError, type PhoneUseErrorCode, type PhoneUseErrorDetails, type PressTarget, type Rect, type RenderState, type Resolution, type ResolveOpts, type ScrollDirection, SecretStore, SessionNotFoundError, type Snapshot, type SnapshotNode, TimeoutError, type UiElement, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createDeviceHandle, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
|
|
689
|
+
//# sourceMappingURL=index.d.mts.map
|