@phone-use/sdk 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +355 -136
- package/dist/index.mjs +1112 -362
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
- package/src/backends/android-hierarchy.ts +153 -0
- package/src/backends/android.ts +890 -0
- package/src/backends/cloud-sandbox.ts +43 -4
- package/src/index.ts +17 -2
- package/src/lifecycle.ts +3 -3
- package/src/observe.ts +43 -9
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { a as registerBackend, c as DeviceInUseError, d as SessionNotFoundError, f as TimeoutError, i as listBackends, l as DeviceNotFoundError, m as toPhoneUseError, n as BaseDeviceBackend, o as AbortedError, p as UnsupportedCapabilityError, r as getBackendFactory, s as ActionFailedError, t as ALL_CAPABILITIES, u as PhoneUseError } from "./device-Xsy_LPUF.mjs";
|
|
2
2
|
import { createAgentDeviceClient } from "agent-device";
|
|
3
|
+
import { execFile, spawn } from "node:child_process";
|
|
3
4
|
import { writeFile } from "node:fs/promises";
|
|
4
|
-
import { execFile } from "node:child_process";
|
|
5
5
|
import { promisify } from "node:util";
|
|
6
6
|
//#region src/observe.ts
|
|
7
7
|
const NOISE_LABELS = /* @__PURE__ */ new Set([
|
|
@@ -50,11 +50,24 @@ function keptViewportNodes(nodes) {
|
|
|
50
50
|
const kept = [];
|
|
51
51
|
let above = 0;
|
|
52
52
|
let below = 0;
|
|
53
|
+
const aboveNames = [];
|
|
54
|
+
const belowNames = [];
|
|
55
|
+
const controlName = (n) => {
|
|
56
|
+
const role = n.role ?? n.type ?? "";
|
|
57
|
+
if (!TAPPABLE.has(role) || role === "StaticText" || role === "Cell") return void 0;
|
|
58
|
+
return (n.label ?? n.identifier ?? "").trim() || void 0;
|
|
59
|
+
};
|
|
53
60
|
for (const n of nodes) {
|
|
54
61
|
if (isNoise(n)) continue;
|
|
55
62
|
if (!intersectsViewport(n.rect, vw, vh)) {
|
|
56
|
-
|
|
57
|
-
|
|
63
|
+
const name = controlName(n);
|
|
64
|
+
if (n.rect && n.rect.y >= vh) {
|
|
65
|
+
below += 1;
|
|
66
|
+
if (name && belowNames.length < 5 && !belowNames.includes(name)) belowNames.push(name);
|
|
67
|
+
} else {
|
|
68
|
+
above += 1;
|
|
69
|
+
if (name && aboveNames.length < 5 && !aboveNames.includes(name)) aboveNames.push(name);
|
|
70
|
+
}
|
|
58
71
|
continue;
|
|
59
72
|
}
|
|
60
73
|
kept.push(n);
|
|
@@ -63,18 +76,21 @@ function keptViewportNodes(nodes) {
|
|
|
63
76
|
kept,
|
|
64
77
|
above,
|
|
65
78
|
below,
|
|
79
|
+
aboveNames,
|
|
80
|
+
belowNames,
|
|
66
81
|
suppressFocused,
|
|
67
82
|
vw
|
|
68
83
|
};
|
|
69
84
|
}
|
|
70
85
|
function compressNodes(nodes) {
|
|
71
|
-
const { kept, above, below, suppressFocused, vw } = keptViewportNodes(nodes);
|
|
86
|
+
const { kept, above, below, aboveNames, belowNames, suppressFocused, vw } = keptViewportNodes(nodes);
|
|
72
87
|
const lines = kept.map((n) => formatNode(n, {
|
|
73
88
|
suppressFocused,
|
|
74
89
|
vw
|
|
75
90
|
}));
|
|
76
|
-
|
|
77
|
-
if (
|
|
91
|
+
const controls = (names) => names.length ? ` — controls there: ${names.map((n) => `"${n}"`).join(", ")}` : "";
|
|
92
|
+
if (above > 0) lines.unshift(`[${above} elements above the viewport — scroll up to reach them${controls(aboveNames)}]`);
|
|
93
|
+
if (below > 0) lines.push(`[${below} elements below the viewport — scroll down to reach them${controls(belowNames)}]`);
|
|
78
94
|
return lines.join("\n");
|
|
79
95
|
}
|
|
80
96
|
function elementKey(n) {
|
|
@@ -281,7 +297,7 @@ var DeviceCore = class {
|
|
|
281
297
|
*/
|
|
282
298
|
renderObservation(full = false) {
|
|
283
299
|
const app = this.lastApp.app;
|
|
284
|
-
const { kept, above, below, suppressFocused, vw } = keptViewportNodes(this.cachedNodes);
|
|
300
|
+
const { kept, above, below, aboveNames, belowNames, suppressFocused, vw } = keptViewportNodes(this.cachedNodes);
|
|
285
301
|
const keys = kept.map(elementKey);
|
|
286
302
|
const lineByKey = /* @__PURE__ */ new Map();
|
|
287
303
|
for (const n of kept) lineByKey.set(elementKey(n), formatNode(n, {
|
|
@@ -311,8 +327,9 @@ var DeviceCore = class {
|
|
|
311
327
|
suppressFocused,
|
|
312
328
|
vw
|
|
313
329
|
}));
|
|
314
|
-
|
|
315
|
-
if (
|
|
330
|
+
const controls = (names) => names.length ? ` — controls there: ${names.map((n) => `"${n}"`).join(", ")}` : "";
|
|
331
|
+
if (above > 0) lines.unshift(`[${above} elements above the viewport — scroll up to reach them${controls(aboveNames)}]`);
|
|
332
|
+
if (below > 0) lines.push(`[${below} elements below the viewport — scroll down to reach them${controls(belowNames)}]`);
|
|
316
333
|
return lines.join("\n");
|
|
317
334
|
}
|
|
318
335
|
cacheSnapshot(nodes) {
|
|
@@ -654,7 +671,7 @@ var DeviceCore = class {
|
|
|
654
671
|
await this.press(el.ref);
|
|
655
672
|
return true;
|
|
656
673
|
} catch (error) {
|
|
657
|
-
if (!/off-?screen/i.test(describeError(error))) throw error;
|
|
674
|
+
if (!/off-?screen|covered/i.test(describeError(error))) throw error;
|
|
658
675
|
}
|
|
659
676
|
const dir = el?.rect ? el.rect.y < 0 ? "up" : "down" : i < 5 ? "up" : "down";
|
|
660
677
|
await this.scroll(dir);
|
|
@@ -1454,363 +1471,75 @@ function createAgentDeviceBackend(config) {
|
|
|
1454
1471
|
return new AgentDeviceBackend(config);
|
|
1455
1472
|
}
|
|
1456
1473
|
//#endregion
|
|
1457
|
-
//#region src/
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
});
|
|
1485
|
-
if (!body.ok) throw new PhoneUseError(body.error.message, {
|
|
1486
|
-
code: normalizeErrorCode(body.error.code),
|
|
1487
|
-
retryable: body.error.retryable ?? false
|
|
1488
|
-
});
|
|
1489
|
-
return body.result;
|
|
1490
|
-
}
|
|
1491
|
-
snapshot(opts) {
|
|
1492
|
-
return this.rpc("snapshot", opts);
|
|
1493
|
-
}
|
|
1494
|
-
async screenshot(opts) {
|
|
1495
|
-
const result = await this.rpc("screenshot", { overlayRefs: opts.overlayRefs });
|
|
1496
|
-
await writeFile(opts.path, Buffer.from(result.base64, "base64"));
|
|
1497
|
-
return { path: opts.path };
|
|
1498
|
-
}
|
|
1499
|
-
press(target) {
|
|
1500
|
-
return this.rpc("press", target);
|
|
1501
|
-
}
|
|
1502
|
-
longPress(ref, durationMs) {
|
|
1503
|
-
return this.rpc("longPress", ref, durationMs);
|
|
1504
|
-
}
|
|
1505
|
-
fill(ref, text) {
|
|
1506
|
-
return this.rpc("fill", ref, text);
|
|
1507
|
-
}
|
|
1508
|
-
typeText(text) {
|
|
1509
|
-
return this.rpc("typeText", text);
|
|
1510
|
-
}
|
|
1511
|
-
pressKey(key) {
|
|
1512
|
-
return this.rpc("pressKey", key);
|
|
1513
|
-
}
|
|
1514
|
-
scroll(direction) {
|
|
1515
|
-
return this.rpc("scroll", direction);
|
|
1516
|
-
}
|
|
1517
|
-
pan(x, y, dx, dy, durationMs) {
|
|
1518
|
-
return this.rpc("pan", x, y, dx, dy, durationMs);
|
|
1519
|
-
}
|
|
1520
|
-
waitForText(text, timeoutMs) {
|
|
1521
|
-
return this.rpc("waitForText", text, timeoutMs);
|
|
1522
|
-
}
|
|
1523
|
-
systemAlert(action) {
|
|
1524
|
-
return this.rpc("systemAlert", action);
|
|
1525
|
-
}
|
|
1526
|
-
home() {
|
|
1527
|
-
return this.rpc("home");
|
|
1528
|
-
}
|
|
1529
|
-
back() {
|
|
1530
|
-
return this.rpc("back");
|
|
1474
|
+
//#region src/exec.ts
|
|
1475
|
+
const pExecFile$1 = promisify(execFile);
|
|
1476
|
+
const defaultExecRunner = async (file, args, opts) => {
|
|
1477
|
+
const { stdout, stderr } = await pExecFile$1(file, args, {
|
|
1478
|
+
encoding: "utf8",
|
|
1479
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
1480
|
+
...opts?.timeoutMs === void 0 ? {} : { timeout: opts.timeoutMs },
|
|
1481
|
+
...opts?.env === void 0 ? {} : { env: opts.env }
|
|
1482
|
+
});
|
|
1483
|
+
return {
|
|
1484
|
+
stdout,
|
|
1485
|
+
stderr
|
|
1486
|
+
};
|
|
1487
|
+
};
|
|
1488
|
+
function isExecError(err) {
|
|
1489
|
+
return err instanceof Error && ("code" in err || "killed" in err || "stderr" in err);
|
|
1490
|
+
}
|
|
1491
|
+
//#endregion
|
|
1492
|
+
//#region src/lifecycle.ts
|
|
1493
|
+
var IdleLease = class {
|
|
1494
|
+
timer = null;
|
|
1495
|
+
windowMs;
|
|
1496
|
+
onExpire;
|
|
1497
|
+
constructor(windowMs, onExpire) {
|
|
1498
|
+
this.windowMs = windowMs;
|
|
1499
|
+
this.onExpire = onExpire;
|
|
1500
|
+
this.touch();
|
|
1531
1501
|
}
|
|
1532
|
-
|
|
1533
|
-
|
|
1502
|
+
/** Re-arm with the configured window (no-op when disabled). */
|
|
1503
|
+
touch() {
|
|
1504
|
+
this.arm(this.windowMs);
|
|
1534
1505
|
}
|
|
1535
|
-
|
|
1536
|
-
|
|
1506
|
+
/** Re-arm with a one-shot override window. */
|
|
1507
|
+
extend(ms) {
|
|
1508
|
+
this.arm(ms ?? this.windowMs);
|
|
1537
1509
|
}
|
|
1538
|
-
|
|
1539
|
-
|
|
1510
|
+
arm(ms) {
|
|
1511
|
+
if (this.timer) clearTimeout(this.timer);
|
|
1512
|
+
this.timer = null;
|
|
1513
|
+
if (ms === false) return;
|
|
1514
|
+
const t = setTimeout(this.onExpire, ms);
|
|
1515
|
+
t.unref?.();
|
|
1516
|
+
this.timer = t;
|
|
1540
1517
|
}
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1518
|
+
dispose() {
|
|
1519
|
+
if (this.timer) clearTimeout(this.timer);
|
|
1520
|
+
this.timer = null;
|
|
1544
1521
|
}
|
|
1545
1522
|
};
|
|
1546
1523
|
/**
|
|
1547
|
-
*
|
|
1548
|
-
*
|
|
1549
|
-
*
|
|
1524
|
+
* Assemble a Device handle over a backend: lease/reaper, verb surface,
|
|
1525
|
+
* close/dispose semantics. Engine authors (ios and android here,
|
|
1526
|
+
* phone-backend-* third parties) build on this; tests fabricate devices with
|
|
1527
|
+
* it over a FakeBackend.
|
|
1550
1528
|
*/
|
|
1551
|
-
function
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
if ((/* @__PURE__ */ new Set([
|
|
1566
|
-
"DEVICE_NOT_FOUND",
|
|
1567
|
-
"DEVICE_IN_USE",
|
|
1568
|
-
"SESSION_NOT_FOUND",
|
|
1569
|
-
"TIMEOUT",
|
|
1570
|
-
"ACTION_FAILED",
|
|
1571
|
-
"UNSUPPORTED_CAPABILITY",
|
|
1572
|
-
"BACKEND_NOT_FOUND",
|
|
1573
|
-
"ABORTED",
|
|
1574
|
-
"UNKNOWN"
|
|
1575
|
-
])).has(code ?? "")) return code;
|
|
1576
|
-
return {
|
|
1577
|
-
device_gone: "DEVICE_NOT_FOUND",
|
|
1578
|
-
sandbox_not_found: "DEVICE_NOT_FOUND",
|
|
1579
|
-
timeout: "TIMEOUT",
|
|
1580
|
-
session_conflict: "DEVICE_IN_USE",
|
|
1581
|
-
target_not_found: "ACTION_FAILED",
|
|
1582
|
-
unauthorized: "ABORTED",
|
|
1583
|
-
payload_too_large: "ACTION_FAILED",
|
|
1584
|
-
bad_request: "ACTION_FAILED",
|
|
1585
|
-
internal: "UNKNOWN"
|
|
1586
|
-
}[code ?? ""] ?? "UNKNOWN";
|
|
1587
|
-
}
|
|
1588
|
-
//#endregion
|
|
1589
|
-
//#region src/backends/device-runner.ts
|
|
1590
|
-
const RUNNER_CAPABILITIES = [
|
|
1591
|
-
"snapshot",
|
|
1592
|
-
"screenshot",
|
|
1593
|
-
"press",
|
|
1594
|
-
"fill",
|
|
1595
|
-
"type",
|
|
1596
|
-
"scroll",
|
|
1597
|
-
"pan",
|
|
1598
|
-
"openApp",
|
|
1599
|
-
"home"
|
|
1600
|
-
];
|
|
1601
|
-
/**
|
|
1602
|
-
* Backend that speaks to an on-device runner: an XCTest-hosted JSON-RPC server
|
|
1603
|
-
* running ON the iPhone itself, which holds the automation privileges iOS
|
|
1604
|
-
* denies to ordinary apps.
|
|
1605
|
-
*
|
|
1606
|
-
* The endpoint is just a URL, so the same backend serves every topology:
|
|
1607
|
-
* - `http://127.0.0.1:45678` — port-forwarded from a paired host
|
|
1608
|
-
* - `http://<phone-ip>:45678` — straight over the LAN / tailnet
|
|
1609
|
-
* - `https://relay.example/d/<id>` — the runner dials out to a cloud relay,
|
|
1610
|
-
* which is what lets an agent anywhere drive the phone with no inbound
|
|
1611
|
-
* ports and no Mac in the loop.
|
|
1612
|
-
*
|
|
1613
|
-
* The wire format matches the shape proven by rounak/PhoneAgent: newline-free
|
|
1614
|
-
* JSON request/response over HTTP POST, one method per call.
|
|
1615
|
-
*/
|
|
1616
|
-
var DeviceRunnerBackend = class extends BaseDeviceBackend {
|
|
1617
|
-
#endpoint;
|
|
1618
|
-
#token;
|
|
1619
|
-
#timeoutMs;
|
|
1620
|
-
constructor(config) {
|
|
1621
|
-
super("device-runner", RUNNER_CAPABILITIES);
|
|
1622
|
-
const endpoint = config?.endpoint ?? process.env.PHONE_USE_RUNNER_URL ?? "http://127.0.0.1:45678";
|
|
1623
|
-
this.#endpoint = endpoint.replace(/\/+$/, "");
|
|
1624
|
-
this.#token = config?.token ?? process.env.PHONE_USE_RUNNER_TOKEN;
|
|
1625
|
-
this.#timeoutMs = config?.timeoutMs ?? 3e4;
|
|
1626
|
-
}
|
|
1627
|
-
async #rpc(method, params = {}) {
|
|
1628
|
-
const controller = new AbortController();
|
|
1629
|
-
const timer = setTimeout(() => controller.abort(), this.#timeoutMs);
|
|
1630
|
-
let res;
|
|
1631
|
-
try {
|
|
1632
|
-
res = await fetch(this.#endpoint, {
|
|
1633
|
-
method: "POST",
|
|
1634
|
-
headers: {
|
|
1635
|
-
"content-type": "application/json",
|
|
1636
|
-
...this.#token ? { authorization: `Bearer ${this.#token}` } : {}
|
|
1637
|
-
},
|
|
1638
|
-
body: JSON.stringify({
|
|
1639
|
-
jsonrpc: "2.0",
|
|
1640
|
-
id: Date.now(),
|
|
1641
|
-
method,
|
|
1642
|
-
params
|
|
1643
|
-
}),
|
|
1644
|
-
signal: controller.signal
|
|
1645
|
-
});
|
|
1646
|
-
} catch (cause) {
|
|
1647
|
-
if (controller.signal.aborted) throw new TimeoutError(`runner did not answer ${method} within ${this.#timeoutMs}ms`);
|
|
1648
|
-
throw new DeviceNotFoundError(`cannot reach the on-device runner at ${this.#endpoint} — is it activated on the phone?`, { cause });
|
|
1649
|
-
} finally {
|
|
1650
|
-
clearTimeout(timer);
|
|
1651
|
-
}
|
|
1652
|
-
if (!res.ok) throw new ActionFailedError(`runner returned HTTP ${res.status} for ${method}`);
|
|
1653
|
-
const body = await res.json();
|
|
1654
|
-
if (body.error) throw new ActionFailedError(body.error.message ?? `runner rejected ${method}`);
|
|
1655
|
-
return body.result;
|
|
1656
|
-
}
|
|
1657
|
-
async snapshot(opts) {
|
|
1658
|
-
const wire = await this.#rpc("get_tree", {
|
|
1659
|
-
interactiveOnly: opts?.interactiveOnly ?? false,
|
|
1660
|
-
depth: opts?.depth
|
|
1661
|
-
});
|
|
1662
|
-
return {
|
|
1663
|
-
appBundleId: wire.app,
|
|
1664
|
-
appName: wire.app,
|
|
1665
|
-
nodes: (wire.elements ?? []).map((e) => ({
|
|
1666
|
-
ref: e.ref,
|
|
1667
|
-
role: e.role,
|
|
1668
|
-
type: e.role,
|
|
1669
|
-
label: e.label,
|
|
1670
|
-
value: e.value,
|
|
1671
|
-
enabled: e.enabled,
|
|
1672
|
-
rect: e.rect ? {
|
|
1673
|
-
x: e.rect.x,
|
|
1674
|
-
y: e.rect.y,
|
|
1675
|
-
width: e.rect.w,
|
|
1676
|
-
height: e.rect.h
|
|
1677
|
-
} : void 0
|
|
1678
|
-
}))
|
|
1679
|
-
};
|
|
1680
|
-
}
|
|
1681
|
-
async screenshot(opts) {
|
|
1682
|
-
const { base64 } = await this.#rpc("get_screen_image", { overlayRefs: opts.overlayRefs ?? false });
|
|
1683
|
-
const { writeFile } = await import("node:fs/promises");
|
|
1684
|
-
await writeFile(opts.path, Buffer.from(base64, "base64"));
|
|
1685
|
-
return { path: opts.path };
|
|
1686
|
-
}
|
|
1687
|
-
async press(target) {
|
|
1688
|
-
if ("ref" in target) {
|
|
1689
|
-
await this.#rpc("tap_element", { ref: target.ref });
|
|
1690
|
-
return;
|
|
1691
|
-
}
|
|
1692
|
-
await this.#rpc("tap", {
|
|
1693
|
-
x: target.x,
|
|
1694
|
-
y: target.y
|
|
1695
|
-
});
|
|
1696
|
-
}
|
|
1697
|
-
async fill(ref, text) {
|
|
1698
|
-
await this.#rpc("enter_text", {
|
|
1699
|
-
ref,
|
|
1700
|
-
text,
|
|
1701
|
-
replace: true
|
|
1702
|
-
});
|
|
1703
|
-
}
|
|
1704
|
-
async typeText(text) {
|
|
1705
|
-
await this.#rpc("enter_text", {
|
|
1706
|
-
text,
|
|
1707
|
-
replace: false
|
|
1708
|
-
});
|
|
1709
|
-
}
|
|
1710
|
-
async scroll(direction) {
|
|
1711
|
-
await this.#rpc("scroll", { direction });
|
|
1712
|
-
}
|
|
1713
|
-
async pan(x, y, dx, dy, durationMs = 300) {
|
|
1714
|
-
await this.#rpc("swipe", {
|
|
1715
|
-
x,
|
|
1716
|
-
y,
|
|
1717
|
-
dx,
|
|
1718
|
-
dy,
|
|
1719
|
-
durationMs
|
|
1720
|
-
});
|
|
1721
|
-
}
|
|
1722
|
-
async home() {
|
|
1723
|
-
await this.#rpc("home");
|
|
1724
|
-
}
|
|
1725
|
-
async openApp(opts) {
|
|
1726
|
-
return this.#rpc("open_app", {
|
|
1727
|
-
app: opts.app,
|
|
1728
|
-
url: opts.url,
|
|
1729
|
-
relaunch: opts.relaunch ?? false
|
|
1730
|
-
});
|
|
1731
|
-
}
|
|
1732
|
-
async systemAlert(action) {
|
|
1733
|
-
return this.#rpc("alert", { action });
|
|
1734
|
-
}
|
|
1735
|
-
async closeSession() {}
|
|
1736
|
-
/** Liveness probe used by `phone-use doctor` and the relay health check. */
|
|
1737
|
-
async ping() {
|
|
1738
|
-
return this.#rpc("get_context");
|
|
1739
|
-
}
|
|
1740
|
-
};
|
|
1741
|
-
/** Backend for a phone-runner reached by URL (defaults come from
|
|
1742
|
-
* `PHONE_USE_RUNNER_URL` / `PHONE_USE_RUNNER_TOKEN` when `config` is omitted). */
|
|
1743
|
-
const createDeviceRunnerBackend = (config) => new DeviceRunnerBackend(config);
|
|
1744
|
-
//#endregion
|
|
1745
|
-
//#region src/exec.ts
|
|
1746
|
-
const pExecFile = promisify(execFile);
|
|
1747
|
-
const defaultExecRunner = async (file, args, opts) => {
|
|
1748
|
-
const { stdout, stderr } = await pExecFile(file, args, {
|
|
1749
|
-
encoding: "utf8",
|
|
1750
|
-
maxBuffer: 16 * 1024 * 1024,
|
|
1751
|
-
...opts?.timeoutMs === void 0 ? {} : { timeout: opts.timeoutMs },
|
|
1752
|
-
...opts?.env === void 0 ? {} : { env: opts.env }
|
|
1753
|
-
});
|
|
1754
|
-
return {
|
|
1755
|
-
stdout,
|
|
1756
|
-
stderr
|
|
1757
|
-
};
|
|
1758
|
-
};
|
|
1759
|
-
function isExecError(err) {
|
|
1760
|
-
return err instanceof Error && ("code" in err || "killed" in err || "stderr" in err);
|
|
1761
|
-
}
|
|
1762
|
-
//#endregion
|
|
1763
|
-
//#region src/lifecycle.ts
|
|
1764
|
-
var IdleLease = class {
|
|
1765
|
-
timer = null;
|
|
1766
|
-
windowMs;
|
|
1767
|
-
onExpire;
|
|
1768
|
-
constructor(windowMs, onExpire) {
|
|
1769
|
-
this.windowMs = windowMs;
|
|
1770
|
-
this.onExpire = onExpire;
|
|
1771
|
-
this.touch();
|
|
1772
|
-
}
|
|
1773
|
-
/** Re-arm with the configured window (no-op when disabled). */
|
|
1774
|
-
touch() {
|
|
1775
|
-
this.arm(this.windowMs);
|
|
1776
|
-
}
|
|
1777
|
-
/** Re-arm with a one-shot override window. */
|
|
1778
|
-
extend(ms) {
|
|
1779
|
-
this.arm(ms ?? this.windowMs);
|
|
1780
|
-
}
|
|
1781
|
-
arm(ms) {
|
|
1782
|
-
if (this.timer) clearTimeout(this.timer);
|
|
1783
|
-
this.timer = null;
|
|
1784
|
-
if (ms === false) return;
|
|
1785
|
-
const t = setTimeout(this.onExpire, ms);
|
|
1786
|
-
t.unref?.();
|
|
1787
|
-
this.timer = t;
|
|
1788
|
-
}
|
|
1789
|
-
dispose() {
|
|
1790
|
-
if (this.timer) clearTimeout(this.timer);
|
|
1791
|
-
this.timer = null;
|
|
1792
|
-
}
|
|
1793
|
-
};
|
|
1794
|
-
/**
|
|
1795
|
-
* Assemble a Device handle over a backend: lease/reaper, verb surface,
|
|
1796
|
-
* close/dispose semantics. Engine authors (ios here, android in item 7b,
|
|
1797
|
-
* phone-backend-* third parties) build on this; tests fabricate devices with
|
|
1798
|
-
* it over a FakeBackend.
|
|
1799
|
-
*/
|
|
1800
|
-
function createDeviceHandle(opts) {
|
|
1801
|
-
let status = "running";
|
|
1802
|
-
let closePromise = null;
|
|
1803
|
-
const close = () => {
|
|
1804
|
-
closePromise ??= (async () => {
|
|
1805
|
-
status = "closed";
|
|
1806
|
-
lease.dispose();
|
|
1807
|
-
await opts.backend.closeSession().catch(() => void 0);
|
|
1808
|
-
await opts.doClose();
|
|
1809
|
-
})();
|
|
1810
|
-
return closePromise;
|
|
1811
|
-
};
|
|
1812
|
-
const lease = new IdleLease(opts.idleTimeoutMs ?? 18e4, () => {
|
|
1813
|
-
close().catch(() => void 0).then(() => opts.onIdleClose?.(device));
|
|
1529
|
+
function createDeviceHandle(opts) {
|
|
1530
|
+
let status = "running";
|
|
1531
|
+
let closePromise = null;
|
|
1532
|
+
const close = () => {
|
|
1533
|
+
closePromise ??= (async () => {
|
|
1534
|
+
status = "closed";
|
|
1535
|
+
lease.dispose();
|
|
1536
|
+
await opts.backend.closeSession().catch(() => void 0);
|
|
1537
|
+
await opts.doClose();
|
|
1538
|
+
})();
|
|
1539
|
+
return closePromise;
|
|
1540
|
+
};
|
|
1541
|
+
const lease = new IdleLease(opts.idleTimeoutMs ?? 18e4, () => {
|
|
1542
|
+
close().catch(() => void 0).then(() => opts.onIdleClose?.(device));
|
|
1814
1543
|
});
|
|
1815
1544
|
const touchingBackend = new Proxy(opts.backend, { get(target, prop, receiver) {
|
|
1816
1545
|
const value = Reflect.get(target, prop, receiver);
|
|
@@ -1970,6 +1699,1026 @@ function createDeviceHandle(opts) {
|
|
|
1970
1699
|
return device;
|
|
1971
1700
|
}
|
|
1972
1701
|
//#endregion
|
|
1702
|
+
//#region src/backends/android-hierarchy.ts
|
|
1703
|
+
/** `[x1,y1][x2,y2]` → a top-left Rect. Returns undefined if unparseable. */
|
|
1704
|
+
function parseBounds(bounds) {
|
|
1705
|
+
const m = bounds?.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
1706
|
+
if (!m) return void 0;
|
|
1707
|
+
const x1 = Number(m[1]);
|
|
1708
|
+
const y1 = Number(m[2]);
|
|
1709
|
+
const x2 = Number(m[3]);
|
|
1710
|
+
const y2 = Number(m[4]);
|
|
1711
|
+
return {
|
|
1712
|
+
x: x1,
|
|
1713
|
+
y: y1,
|
|
1714
|
+
width: x2 - x1,
|
|
1715
|
+
height: y2 - y1
|
|
1716
|
+
};
|
|
1717
|
+
}
|
|
1718
|
+
/** `android.widget.Button` → `Button`; keeps a bare token as-is. */
|
|
1719
|
+
function shortType(className) {
|
|
1720
|
+
if (!className) return void 0;
|
|
1721
|
+
const tail = className.split(".").pop();
|
|
1722
|
+
return tail && tail.length > 0 ? tail : className;
|
|
1723
|
+
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Normalise Android widget classes to the harness's canonical (iOS-named)
|
|
1726
|
+
* element types, so every layer above — inputFields(), setField(), the
|
|
1727
|
+
* interactive/editable sets in the SDK — works unchanged on Android.
|
|
1728
|
+
*
|
|
1729
|
+
* This is load-bearing: without it `type --field` reports "no editable text
|
|
1730
|
+
* field" on a real Android because `AutoCompleteTextView` never matches the
|
|
1731
|
+
* SDK's EDITABLE set ({SearchField, TextField, SecureTextField}). Measured on
|
|
1732
|
+
* a OnePlus Nord: the Settings search box is an AutoCompleteTextView.
|
|
1733
|
+
*/
|
|
1734
|
+
function canonicalType(shortName, attrs) {
|
|
1735
|
+
if (!shortName) return void 0;
|
|
1736
|
+
const isPassword = attrs.password === "true";
|
|
1737
|
+
switch (shortName) {
|
|
1738
|
+
case "EditText":
|
|
1739
|
+
case "AutoCompleteTextView":
|
|
1740
|
+
case "MultiAutoCompleteTextView":
|
|
1741
|
+
case "SearchView":
|
|
1742
|
+
case "TextInputEditText": return isPassword ? "SecureTextField" : "TextField";
|
|
1743
|
+
case "Switch":
|
|
1744
|
+
case "SwitchCompat":
|
|
1745
|
+
case "CheckBox":
|
|
1746
|
+
case "ToggleButton":
|
|
1747
|
+
case "RadioButton": return "Switch";
|
|
1748
|
+
case "SeekBar": return "Slider";
|
|
1749
|
+
case "ImageButton": return "Button";
|
|
1750
|
+
default:
|
|
1751
|
+
if (attrs.clickable === "true") return "Button";
|
|
1752
|
+
return shortName;
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
const ATTR = /([\w:-]+)="([^"]*)"/g;
|
|
1756
|
+
function attrs(fragment) {
|
|
1757
|
+
const out = {};
|
|
1758
|
+
for (const m of fragment.matchAll(ATTR)) {
|
|
1759
|
+
const key = m[1];
|
|
1760
|
+
const val = m[2];
|
|
1761
|
+
if (key !== void 0 && val !== void 0) out[key] = decodeEntities(val);
|
|
1762
|
+
}
|
|
1763
|
+
return out;
|
|
1764
|
+
}
|
|
1765
|
+
function decodeEntities(s) {
|
|
1766
|
+
return s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll(""", "\"").replaceAll("'", "'");
|
|
1767
|
+
}
|
|
1768
|
+
const isTrue = (v) => v === "true";
|
|
1769
|
+
/**
|
|
1770
|
+
* Parse a uiautomator XML dump. Each <node> becomes a SnapshotNode with a
|
|
1771
|
+
* fresh `@aN` ref. When `interactiveOnly`, keep only nodes that are actually
|
|
1772
|
+
* actionable (clickable / long-clickable / a focusable field) or that carry a
|
|
1773
|
+
* label — the rest is layout scaffolding an agent should not try to tap.
|
|
1774
|
+
*/
|
|
1775
|
+
function parseHierarchy(xml, opts = {}) {
|
|
1776
|
+
const nodes = [];
|
|
1777
|
+
const rects = /* @__PURE__ */ new Map();
|
|
1778
|
+
let packageName;
|
|
1779
|
+
let index = 0;
|
|
1780
|
+
for (const m of xml.matchAll(/<node\b([^>]*?)\/?>/g)) {
|
|
1781
|
+
const fragment = m[1];
|
|
1782
|
+
if (fragment === void 0) continue;
|
|
1783
|
+
const a = attrs(fragment);
|
|
1784
|
+
if (!packageName && a.package) packageName = a.package;
|
|
1785
|
+
const label = a.text || a["content-desc"] || void 0;
|
|
1786
|
+
const clickable = isTrue(a.clickable) || isTrue(a["long-clickable"]);
|
|
1787
|
+
const focusableField = isTrue(a.focusable) && (label !== void 0 || isTrue(a.editable));
|
|
1788
|
+
if (opts.interactiveOnly && !clickable && !focusableField && !label) continue;
|
|
1789
|
+
index += 1;
|
|
1790
|
+
const ref = `@a${index}`;
|
|
1791
|
+
const rect = parseBounds(a.bounds);
|
|
1792
|
+
if (rect) rects.set(ref, rect);
|
|
1793
|
+
const node = {
|
|
1794
|
+
ref,
|
|
1795
|
+
type: canonicalType(shortType(a.class), a),
|
|
1796
|
+
role: canonicalType(shortType(a.class), a),
|
|
1797
|
+
...label !== void 0 ? { label } : {},
|
|
1798
|
+
...a.text && a.text !== label ? { value: a.text } : {},
|
|
1799
|
+
...a["resource-id"] ? { identifier: a["resource-id"] } : {},
|
|
1800
|
+
enabled: a.enabled ? isTrue(a.enabled) : true,
|
|
1801
|
+
selected: isTrue(a.selected),
|
|
1802
|
+
focused: isTrue(a.focused),
|
|
1803
|
+
...rect ? { rect } : {},
|
|
1804
|
+
...a.enabled && !isTrue(a.enabled) ? { interactionBlocked: "disabled" } : {}
|
|
1805
|
+
};
|
|
1806
|
+
nodes.push(node);
|
|
1807
|
+
}
|
|
1808
|
+
return {
|
|
1809
|
+
nodes,
|
|
1810
|
+
rects,
|
|
1811
|
+
...packageName ? { packageName } : {}
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
//#endregion
|
|
1815
|
+
//#region src/backends/android.ts
|
|
1816
|
+
/** Android keycodes used below (KeyEvent constants). */
|
|
1817
|
+
const KEYCODE = {
|
|
1818
|
+
home: 3,
|
|
1819
|
+
back: 4,
|
|
1820
|
+
enter: 66,
|
|
1821
|
+
del: 67,
|
|
1822
|
+
moveEnd: 123
|
|
1823
|
+
};
|
|
1824
|
+
/** Where uiautomator writes its dump: /data/local/tmp round-trips on scoped-storage builds, /sdcard does not. */
|
|
1825
|
+
const DUMP_PATH = "/data/local/tmp/phone-use-dump.xml";
|
|
1826
|
+
/** A package id as Android accepts it: dotted Java-style segments. */
|
|
1827
|
+
const PACKAGE_RE = /^[A-Za-z][\w]*(\.[A-Za-z][\w]*)+$/;
|
|
1828
|
+
/** `com.pkg/.Activity` as `cmd package resolve-activity --brief` prints it. */
|
|
1829
|
+
const ACTIVITY_RE = /^[\w.]+\/[\w.$]+$/;
|
|
1830
|
+
/** Characters that need no quoting for the device's /bin/sh. */
|
|
1831
|
+
const SHELL_SAFE = /^[A-Za-z0-9_@%+=:,./-]+$/;
|
|
1832
|
+
/**
|
|
1833
|
+
* Quote one argument for the device shell. `adb shell` does not escape its
|
|
1834
|
+
* arguments, so this is the injection boundary for every device command.
|
|
1835
|
+
*/
|
|
1836
|
+
function shellQuote(arg) {
|
|
1837
|
+
if (arg.length > 0 && SHELL_SAFE.test(arg)) return arg;
|
|
1838
|
+
return `'${arg.replace(/'/g, `'\\''`)}'`;
|
|
1839
|
+
}
|
|
1840
|
+
/** True when every char is printable ASCII — the range `input text` can deliver. */
|
|
1841
|
+
function isPrintableAscii(text) {
|
|
1842
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
1843
|
+
const c = text.charCodeAt(i);
|
|
1844
|
+
if (c < 32 || c > 126) return false;
|
|
1845
|
+
}
|
|
1846
|
+
return true;
|
|
1847
|
+
}
|
|
1848
|
+
const pExecFile = promisify(execFile);
|
|
1849
|
+
const defaultBinaryExecRunner = async (file, args, opts) => {
|
|
1850
|
+
const { stdout } = await pExecFile(file, args, {
|
|
1851
|
+
encoding: "buffer",
|
|
1852
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
1853
|
+
...opts?.timeoutMs === void 0 ? {} : { timeout: opts.timeoutMs }
|
|
1854
|
+
});
|
|
1855
|
+
return new Uint8Array(stdout);
|
|
1856
|
+
};
|
|
1857
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1858
|
+
/**
|
|
1859
|
+
* Drive an Android device or emulator over adb. Implements the
|
|
1860
|
+
* {@link DeviceBackend} contract with uiautomator for the tree, screencap for
|
|
1861
|
+
* pixels, and `input` for gestures. Stateless across calls except the ref → rect
|
|
1862
|
+
* map from the last snapshot (what makes press-by-ref work).
|
|
1863
|
+
*/
|
|
1864
|
+
var AndroidBackend = class AndroidBackend extends BaseDeviceBackend {
|
|
1865
|
+
serial;
|
|
1866
|
+
adbBin;
|
|
1867
|
+
timeoutMs;
|
|
1868
|
+
exec;
|
|
1869
|
+
execBinary;
|
|
1870
|
+
sleep;
|
|
1871
|
+
rects = /* @__PURE__ */ new Map();
|
|
1872
|
+
sizeCache = null;
|
|
1873
|
+
launchablesCache = null;
|
|
1874
|
+
constructor(opts = {}) {
|
|
1875
|
+
super("android-adb", [
|
|
1876
|
+
"snapshot",
|
|
1877
|
+
"screenshot",
|
|
1878
|
+
"press",
|
|
1879
|
+
"longPress",
|
|
1880
|
+
"fill",
|
|
1881
|
+
"type",
|
|
1882
|
+
"key",
|
|
1883
|
+
"scroll",
|
|
1884
|
+
"pan",
|
|
1885
|
+
"waitForText",
|
|
1886
|
+
"home",
|
|
1887
|
+
"back",
|
|
1888
|
+
"openApp",
|
|
1889
|
+
"openUrl",
|
|
1890
|
+
"listApps",
|
|
1891
|
+
"closeSession"
|
|
1892
|
+
]);
|
|
1893
|
+
this.serial = opts.serial;
|
|
1894
|
+
this.adbBin = opts.adbBin ?? "adb";
|
|
1895
|
+
this.timeoutMs = opts.commandTimeoutMs ?? 3e4;
|
|
1896
|
+
this.exec = opts.exec ?? defaultExecRunner;
|
|
1897
|
+
this.execBinary = opts.execBinary ?? defaultBinaryExecRunner;
|
|
1898
|
+
this.sleep = opts.sleep ?? defaultSleep;
|
|
1899
|
+
}
|
|
1900
|
+
withSerial(args) {
|
|
1901
|
+
return this.serial ? [
|
|
1902
|
+
"-s",
|
|
1903
|
+
this.serial,
|
|
1904
|
+
...args
|
|
1905
|
+
] : args;
|
|
1906
|
+
}
|
|
1907
|
+
/** Normalize an exec failure into the PhoneUseError the contract promises. */
|
|
1908
|
+
fail(what, err) {
|
|
1909
|
+
if (err instanceof PhoneUseError) return err;
|
|
1910
|
+
if (isExecError(err)) {
|
|
1911
|
+
const stderr = (err.stderr ?? "").trim();
|
|
1912
|
+
if (err.killed || err.signal) return new TimeoutError(`adb ${what} timed out after ${this.timeoutMs}ms`, { cause: err });
|
|
1913
|
+
if (err.code === "ENOENT") return new ActionFailedError(`adb not found at "${this.adbBin}" — install the Android platform-tools or pass adbBin`, { cause: err });
|
|
1914
|
+
if (/not found|no devices\/emulators|device offline|more than one device|unauthorized/i.test(stderr)) return new DeviceNotFoundError(`adb ${what}: ${stderr}`, { cause: err });
|
|
1915
|
+
return new ActionFailedError(`adb ${what} failed (exit ${String(err.code ?? "?")}): ${stderr || err.message}`, {
|
|
1916
|
+
details: { backendCode: String(err.code ?? "EXEC") },
|
|
1917
|
+
cause: err
|
|
1918
|
+
});
|
|
1919
|
+
}
|
|
1920
|
+
return new ActionFailedError(`adb ${what} failed: ${String(err)}`, { cause: err });
|
|
1921
|
+
}
|
|
1922
|
+
/** Run an adb command, returning stdout text. */
|
|
1923
|
+
async adb(args) {
|
|
1924
|
+
try {
|
|
1925
|
+
return (await this.exec(this.adbBin, this.withSerial(args), { timeoutMs: this.timeoutMs })).stdout;
|
|
1926
|
+
} catch (err) {
|
|
1927
|
+
throw this.fail(args.slice(0, 2).join(" "), err);
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
/** `adb shell <argv>` — each argument quoted for the device shell. */
|
|
1931
|
+
shell(...argv) {
|
|
1932
|
+
return this.adb(["shell", argv.map(shellQuote).join(" ")]);
|
|
1933
|
+
}
|
|
1934
|
+
/** `adb shell <literal>` for the few CONSTANT commands that need a pipe. Never pass input. */
|
|
1935
|
+
shellRaw(literal) {
|
|
1936
|
+
return this.adb(["shell", literal]);
|
|
1937
|
+
}
|
|
1938
|
+
center(ref) {
|
|
1939
|
+
const rect = this.rects.get(ref);
|
|
1940
|
+
if (!rect) throw new ActionFailedError(`unknown element ${ref} — take a fresh snapshot first`);
|
|
1941
|
+
return {
|
|
1942
|
+
x: Math.round(rect.x + rect.width / 2),
|
|
1943
|
+
y: Math.round(rect.y + rect.height / 2)
|
|
1944
|
+
};
|
|
1945
|
+
}
|
|
1946
|
+
/** The screen size input coordinates map to (an `Override size` wins over `Physical size`). */
|
|
1947
|
+
async screenSize() {
|
|
1948
|
+
if (this.sizeCache) return this.sizeCache;
|
|
1949
|
+
const out = await this.shell("wm", "size");
|
|
1950
|
+
const m = out.match(/Override size:\s*(\d+)x(\d+)/) ?? out.match(/Physical size:\s*(\d+)x(\d+)/);
|
|
1951
|
+
if (!m) throw new ActionFailedError(`could not read screen size from: ${out.trim()}`);
|
|
1952
|
+
this.sizeCache = {
|
|
1953
|
+
width: Number(m[1]),
|
|
1954
|
+
height: Number(m[2])
|
|
1955
|
+
};
|
|
1956
|
+
return this.sizeCache;
|
|
1957
|
+
}
|
|
1958
|
+
async snapshot(opts) {
|
|
1959
|
+
const size = await this.screenSize();
|
|
1960
|
+
let xml = "";
|
|
1961
|
+
let lastError;
|
|
1962
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
1963
|
+
try {
|
|
1964
|
+
await this.shell("uiautomator", "dump", DUMP_PATH);
|
|
1965
|
+
xml = await this.adb([
|
|
1966
|
+
"exec-out",
|
|
1967
|
+
"cat",
|
|
1968
|
+
DUMP_PATH
|
|
1969
|
+
]);
|
|
1970
|
+
} catch (err) {
|
|
1971
|
+
lastError = err;
|
|
1972
|
+
}
|
|
1973
|
+
if (xml.includes("<node")) break;
|
|
1974
|
+
await this.sleep(400);
|
|
1975
|
+
}
|
|
1976
|
+
if (!xml.includes("<node")) throw new ActionFailedError("uiautomator dump produced no UI nodes after 3 attempts — the UI may be mid-transition or uiautomator may be wedged (try `adb shell uiautomator dump` by hand)", { cause: lastError });
|
|
1977
|
+
const parsed = parseHierarchy(xml, { interactiveOnly: opts?.interactiveOnly ?? false });
|
|
1978
|
+
this.rects = parsed.rects;
|
|
1979
|
+
const appName = await this.foregroundPackage().catch(() => void 0) ?? parsed.packageName;
|
|
1980
|
+
return {
|
|
1981
|
+
nodes: [{
|
|
1982
|
+
ref: "@a0",
|
|
1983
|
+
type: "Application",
|
|
1984
|
+
role: "Application",
|
|
1985
|
+
label: appName ?? "android",
|
|
1986
|
+
rect: {
|
|
1987
|
+
x: 0,
|
|
1988
|
+
y: 0,
|
|
1989
|
+
width: size.width,
|
|
1990
|
+
height: size.height
|
|
1991
|
+
}
|
|
1992
|
+
}, ...parsed.nodes],
|
|
1993
|
+
...appName ? {
|
|
1994
|
+
appName,
|
|
1995
|
+
appBundleId: appName
|
|
1996
|
+
} : {}
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1999
|
+
/** The package of the resumed (frontmost) activity, for the snapshot header. */
|
|
2000
|
+
async foregroundPackage() {
|
|
2001
|
+
return (await this.shellRaw("dumpsys activity activities | grep -m1 ResumedActivity")).match(/\su0\s+([\w.]+)\//)?.[1];
|
|
2002
|
+
}
|
|
2003
|
+
async screenshot(opts) {
|
|
2004
|
+
let png;
|
|
2005
|
+
try {
|
|
2006
|
+
png = await this.execBinary(this.adbBin, this.withSerial([
|
|
2007
|
+
"exec-out",
|
|
2008
|
+
"screencap",
|
|
2009
|
+
"-p"
|
|
2010
|
+
]), { timeoutMs: this.timeoutMs });
|
|
2011
|
+
} catch (err) {
|
|
2012
|
+
throw this.fail("screencap", err);
|
|
2013
|
+
}
|
|
2014
|
+
await writeFile(opts.path, png);
|
|
2015
|
+
return { path: opts.path };
|
|
2016
|
+
}
|
|
2017
|
+
async press(target) {
|
|
2018
|
+
const { x, y } = "ref" in target ? this.center(target.ref) : target;
|
|
2019
|
+
await this.shell("input", "tap", String(Math.round(x)), String(Math.round(y)));
|
|
2020
|
+
}
|
|
2021
|
+
async longPress(ref, durationMs = 700) {
|
|
2022
|
+
const { x, y } = this.center(ref);
|
|
2023
|
+
await this.shell("input", "swipe", String(x), String(y), String(x), String(y), String(Math.max(400, durationMs)));
|
|
2024
|
+
}
|
|
2025
|
+
async fill(ref, text) {
|
|
2026
|
+
const { x, y } = this.center(ref);
|
|
2027
|
+
await this.shell("input", "tap", String(x), String(y));
|
|
2028
|
+
await this.sleep(120);
|
|
2029
|
+
await this.clearFocusedField();
|
|
2030
|
+
await this.typeText(text);
|
|
2031
|
+
}
|
|
2032
|
+
/**
|
|
2033
|
+
* Empty the currently-focused text field. Ctrl+A then DEL (`input
|
|
2034
|
+
* keycombination`, Android 12+); a short MOVE_END + backspace sweep stays as
|
|
2035
|
+
* the fallback for the rare field that ignores the select combo. Replaces a
|
|
2036
|
+
* 60-backspace sweep that left residue on long fields and made repeated
|
|
2037
|
+
* `type`s ACCUMULATE, sending the agent into retry loops.
|
|
2038
|
+
*/
|
|
2039
|
+
async clearFocusedField() {
|
|
2040
|
+
await this.shell("input", "keycombination", "113", "29").catch(() => void 0);
|
|
2041
|
+
await this.shell("input", "keyevent", String(KEYCODE.del));
|
|
2042
|
+
await this.shell("input", "keyevent", String(KEYCODE.moveEnd));
|
|
2043
|
+
for (let i = 0; i < 8; i += 1) await this.shell("input", "keyevent", String(KEYCODE.del));
|
|
2044
|
+
}
|
|
2045
|
+
async typeText(text) {
|
|
2046
|
+
if (!text) return;
|
|
2047
|
+
for (const [i, line] of text.split("\n").entries()) {
|
|
2048
|
+
if (i > 0) await this.shell("input", "keyevent", String(KEYCODE.enter));
|
|
2049
|
+
for (const [j, chunk] of line.split("\b").entries()) {
|
|
2050
|
+
if (j > 0) await this.shell("input", "keyevent", String(KEYCODE.del));
|
|
2051
|
+
if (!chunk) continue;
|
|
2052
|
+
if (isPrintableAscii(chunk)) for (let at = 0; at < chunk.length; at += 400) await this.shell("input", "text", chunk.slice(at, at + 400));
|
|
2053
|
+
else await this.typeUnicode(chunk);
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
/** The IME id of senzhk/ADBKeyBoard, the de-facto Unicode input path for adb. */
|
|
2058
|
+
static ADB_IME = "com.android.adbkeyboard/.AdbIME";
|
|
2059
|
+
/**
|
|
2060
|
+
* Type non-ASCII via the ADB Keyboard broadcast. The IME must be the ACTIVE
|
|
2061
|
+
* keyboard; some OEM builds deny the shell WRITE_SECURE_SETTINGS so `ime set`
|
|
2062
|
+
* cannot switch it. We try (works on stock builds) and restore the previous
|
|
2063
|
+
* keyboard afterwards when we did the switching. Never a silent drop.
|
|
2064
|
+
*/
|
|
2065
|
+
async typeUnicode(text) {
|
|
2066
|
+
const ime = AndroidBackend.ADB_IME;
|
|
2067
|
+
if (!(await this.shell("pm", "list", "packages", "com.android.adbkeyboard").catch(() => "")).includes("com.android.adbkeyboard")) throw new ActionFailedError(`Android cannot type non-ASCII text ("${text.slice(0, 20)}…") over adb: the platform's own \`input text\` rejects it and no built-in alternative exists. ASCII works. For accented or non-Latin text, tap the field and use the on-screen keyboard (tap the keys), or paste text the user already has on the clipboard. (Dev/test devices only: the ADB Keyboard IME — github.com/senzhk/ADBKeyBoard — enables Unicode over adb.)`);
|
|
2068
|
+
const previous = (await this.shell("settings", "get", "secure", "default_input_method").catch(() => "")).trim();
|
|
2069
|
+
let switched = false;
|
|
2070
|
+
if (previous !== ime) {
|
|
2071
|
+
await this.shell("ime", "enable", ime).catch(() => void 0);
|
|
2072
|
+
await this.shell("ime", "set", ime).catch(() => void 0);
|
|
2073
|
+
switched = (await this.shell("settings", "get", "secure", "default_input_method").catch(() => "")).trim() === ime;
|
|
2074
|
+
if (!switched) throw new ActionFailedError("Android cannot type non-ASCII text over adb on this device: the ADB Keyboard IME is installed but the OS blocks adb from activating it (WRITE_SECURE_SETTINGS denied). ASCII works. For accented or non-Latin text, tap the field and use the on-screen keyboard, or paste from the clipboard. (Dev/test: enable ADB Keyboard once in Settings → Languages & input → Manage keyboards.)");
|
|
2075
|
+
await this.sleep(300);
|
|
2076
|
+
}
|
|
2077
|
+
try {
|
|
2078
|
+
const b64 = Buffer.from(text, "utf8").toString("base64");
|
|
2079
|
+
await this.shell("am", "broadcast", "-a", "ADB_INPUT_B64", "--es", "msg", b64);
|
|
2080
|
+
await this.sleep(150);
|
|
2081
|
+
} finally {
|
|
2082
|
+
if (switched && previous) await this.shell("ime", "set", previous).catch(() => void 0);
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
async pressKey(_key) {
|
|
2086
|
+
await this.shell("input", "keyevent", String(KEYCODE.enter));
|
|
2087
|
+
}
|
|
2088
|
+
async scroll(direction) {
|
|
2089
|
+
const { width, height } = await this.screenSize();
|
|
2090
|
+
const imeShown = (await this.shell("dumpsys", "input_method").catch(() => "")).includes("mInputShown=true");
|
|
2091
|
+
const cx = Math.round(width / 2);
|
|
2092
|
+
const cy = imeShown ? Math.round(height * .3) : Math.round(height / 2);
|
|
2093
|
+
const dx = Math.round(width * .35);
|
|
2094
|
+
const dy = imeShown ? Math.round(height * .15) : Math.round(height * .35);
|
|
2095
|
+
const from = {
|
|
2096
|
+
up: [cx, cy - dy],
|
|
2097
|
+
down: [cx, cy + dy],
|
|
2098
|
+
left: [cx - dx, cy],
|
|
2099
|
+
right: [cx + dx, cy]
|
|
2100
|
+
};
|
|
2101
|
+
const to = {
|
|
2102
|
+
up: [cx, cy + dy],
|
|
2103
|
+
down: [cx, cy - dy],
|
|
2104
|
+
left: [cx + dx, cy],
|
|
2105
|
+
right: [cx - dx, cy]
|
|
2106
|
+
};
|
|
2107
|
+
const [fx, fy] = from[direction];
|
|
2108
|
+
const [tx, ty] = to[direction];
|
|
2109
|
+
await this.shell("input", "swipe", String(fx), String(fy), String(tx), String(ty), "300");
|
|
2110
|
+
}
|
|
2111
|
+
async pan(x, y, dx, dy, durationMs = 300) {
|
|
2112
|
+
await this.shell("input", "swipe", String(Math.round(x)), String(Math.round(y)), String(Math.round(x + dx)), String(Math.round(y + dy)), String(Math.max(50, durationMs)));
|
|
2113
|
+
}
|
|
2114
|
+
async waitForText(text, timeoutMs = 8e3) {
|
|
2115
|
+
const deadline = Date.now() + timeoutMs;
|
|
2116
|
+
const needle = text.toLowerCase();
|
|
2117
|
+
for (;;) {
|
|
2118
|
+
if ((await this.snapshot()).nodes.some((n) => (n.label ?? "").toLowerCase().includes(needle))) return;
|
|
2119
|
+
if (Date.now() >= deadline) throw new TimeoutError(`"${text}" did not appear within ${timeoutMs}ms`);
|
|
2120
|
+
await this.sleep(500);
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
async home() {
|
|
2124
|
+
await this.shell("input", "keyevent", String(KEYCODE.home));
|
|
2125
|
+
}
|
|
2126
|
+
async back() {
|
|
2127
|
+
await this.shell("input", "keyevent", String(KEYCODE.back));
|
|
2128
|
+
}
|
|
2129
|
+
/** Packages that expose a launcher icon — the set `open <name>` can resolve to. */
|
|
2130
|
+
async launchables() {
|
|
2131
|
+
if (this.launchablesCache) return this.launchablesCache;
|
|
2132
|
+
const out = await this.shell("cmd", "package", "query-activities", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER").catch(() => "");
|
|
2133
|
+
const pkgs = [...new Set([...out.matchAll(/packageName=([\w.]+)/g)].map((m) => m[1]))];
|
|
2134
|
+
if (pkgs.length) this.launchablesCache = pkgs;
|
|
2135
|
+
return pkgs;
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2138
|
+
* Resolve a human app name to an installed package: agents say "Markor" or
|
|
2139
|
+
* "Simple Calendar", `am` needs `net.gsantner.markor`. Score each launchable
|
|
2140
|
+
* package by how many of the query's words appear in its id; best (shortest
|
|
2141
|
+
* on a tie) wins. An exact package id passes straight through. Returns
|
|
2142
|
+
* undefined when nothing matches and the query is not a package id at all.
|
|
2143
|
+
*/
|
|
2144
|
+
async resolvePackage(query) {
|
|
2145
|
+
const launch = await this.launchables();
|
|
2146
|
+
if (launch.includes(query)) return query;
|
|
2147
|
+
if (PACKAGE_RE.test(query)) return query;
|
|
2148
|
+
const norm = (v) => v.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
2149
|
+
const words = query.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
2150
|
+
let best = null;
|
|
2151
|
+
for (const pkg of launch) {
|
|
2152
|
+
const np = norm(pkg);
|
|
2153
|
+
const score = words.filter((w) => np.includes(w)).length;
|
|
2154
|
+
if (score === 0) continue;
|
|
2155
|
+
if (!best || score > best.score || score === best.score && pkg.length < best.pkg.length) best = {
|
|
2156
|
+
pkg,
|
|
2157
|
+
score
|
|
2158
|
+
};
|
|
2159
|
+
}
|
|
2160
|
+
return best?.pkg;
|
|
2161
|
+
}
|
|
2162
|
+
async openApp(opts) {
|
|
2163
|
+
if (opts.url) {
|
|
2164
|
+
await this.shell("am", "start", "-a", "android.intent.action.VIEW", "-d", opts.url);
|
|
2165
|
+
return {};
|
|
2166
|
+
}
|
|
2167
|
+
if (!opts.app) throw new ActionFailedError("openApp needs an app name/package id or a url");
|
|
2168
|
+
const pkg = await this.resolvePackage(opts.app);
|
|
2169
|
+
if (!pkg) throw new DeviceNotFoundError(`no installed app matches "${opts.app}" — pass a launcher name or a package id (see listApps)`);
|
|
2170
|
+
if (opts.relaunch) await this.shell("am", "force-stop", pkg).catch(() => void 0);
|
|
2171
|
+
const activity = (await this.shell("cmd", "package", "resolve-activity", "--brief", "-c", "android.intent.category.LAUNCHER", pkg).catch(() => "")).trim().split("\n").pop()?.trim();
|
|
2172
|
+
if (activity && ACTIVITY_RE.test(activity)) await this.shell("am", "start", "-n", activity);
|
|
2173
|
+
else await this.shell("monkey", "-p", pkg, "-c", "android.intent.category.LAUNCHER", "1").catch(() => void 0);
|
|
2174
|
+
return { appBundleId: pkg };
|
|
2175
|
+
}
|
|
2176
|
+
async listApps() {
|
|
2177
|
+
return (await this.shell("pm", "list", "packages")).split("\n").map((l) => l.replace(/^package:/, "").trim()).filter(Boolean).sort();
|
|
2178
|
+
}
|
|
2179
|
+
closeSession() {
|
|
2180
|
+
return Promise.resolve();
|
|
2181
|
+
}
|
|
2182
|
+
};
|
|
2183
|
+
/**
|
|
2184
|
+
* Backend factory for the `android-adb` registry entry: an
|
|
2185
|
+
* {@link AndroidDeviceConfig} pins the serial; anything else targets the single
|
|
2186
|
+
* attached device.
|
|
2187
|
+
*/
|
|
2188
|
+
function createAndroidBackend(config) {
|
|
2189
|
+
if (config && "platform" in config) return new AndroidBackend(config.platform === "android" ? { serial: config.serial } : {});
|
|
2190
|
+
return new AndroidBackend(config);
|
|
2191
|
+
}
|
|
2192
|
+
/** Parse `adb devices -l` (header dropped). */
|
|
2193
|
+
function parseDevices(stdout) {
|
|
2194
|
+
return stdout.split("\n").slice(1).map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("*")).map((line) => {
|
|
2195
|
+
const [serial = "", state = "unknown", ...rest] = line.split(/\s+/);
|
|
2196
|
+
const model = rest.find((f) => f.startsWith("model:"))?.slice(6);
|
|
2197
|
+
return {
|
|
2198
|
+
serial,
|
|
2199
|
+
state,
|
|
2200
|
+
...model ? { model } : {}
|
|
2201
|
+
};
|
|
2202
|
+
}).filter((d) => d.serial.length > 0);
|
|
2203
|
+
}
|
|
2204
|
+
async function runAdb(exec, adbBin, args, timeoutMs) {
|
|
2205
|
+
try {
|
|
2206
|
+
return (await exec(adbBin, args, { timeoutMs })).stdout;
|
|
2207
|
+
} catch (err) {
|
|
2208
|
+
if (isExecError(err)) {
|
|
2209
|
+
if (err.killed || err.signal) throw new TimeoutError(`adb ${args[0]} timed out after ${timeoutMs}ms`, { cause: err });
|
|
2210
|
+
if (err.code === "ENOENT") throw new ActionFailedError(`adb not found at "${adbBin}" — install the Android platform-tools`, { cause: err });
|
|
2211
|
+
throw new ActionFailedError(`adb ${args[0]} failed: ${(err.stderr ?? err.message).trim()}`, { cause: err });
|
|
2212
|
+
}
|
|
2213
|
+
throw new ActionFailedError(`adb ${args[0]} failed: ${String(err)}`, { cause: err });
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
/** Devices adb currently sees, with their state. */
|
|
2217
|
+
async function list(options = {}) {
|
|
2218
|
+
return parseDevices(await runAdb(options.exec ?? defaultExecRunner, options.adbBin ?? "adb", ["devices", "-l"], 15e3));
|
|
2219
|
+
}
|
|
2220
|
+
function finishHandle$1(row, createdByUs, opts, doClose) {
|
|
2221
|
+
const backend = new AndroidBackend({
|
|
2222
|
+
serial: row.serial,
|
|
2223
|
+
adbBin: opts.adbBin,
|
|
2224
|
+
exec: opts.exec,
|
|
2225
|
+
execBinary: opts.execBinary,
|
|
2226
|
+
sleep: opts.sleep
|
|
2227
|
+
});
|
|
2228
|
+
return createDeviceHandle({
|
|
2229
|
+
id: row.serial,
|
|
2230
|
+
platform: "android",
|
|
2231
|
+
name: row.model,
|
|
2232
|
+
backend,
|
|
2233
|
+
createdByUs,
|
|
2234
|
+
idleTimeoutMs: opts.idleTimeoutMs,
|
|
2235
|
+
onIdleClose: opts.onIdleClose,
|
|
2236
|
+
secrets: opts.secrets,
|
|
2237
|
+
coreFactory: opts.coreFactory,
|
|
2238
|
+
doClose
|
|
2239
|
+
});
|
|
2240
|
+
}
|
|
2241
|
+
const FIX_USB_DEBUGGING = "enable Developer options → USB debugging on the phone, connect over USB (or `adb connect <ip:port>`), and confirm with `adb devices`";
|
|
2242
|
+
/**
|
|
2243
|
+
* Attach to a device adb already sees. With a serial, that device; without,
|
|
2244
|
+
* the single ready device (several attached → an error naming them, so verbs
|
|
2245
|
+
* never land on the wrong phone). `close()` releases the handle and never
|
|
2246
|
+
* shuts the device down — it was yours before we connected.
|
|
2247
|
+
*/
|
|
2248
|
+
async function connect$1(serial, options = {}) {
|
|
2249
|
+
const exec = options.exec ?? defaultExecRunner;
|
|
2250
|
+
const devices = await list({
|
|
2251
|
+
adbBin: options.adbBin,
|
|
2252
|
+
exec
|
|
2253
|
+
});
|
|
2254
|
+
let row;
|
|
2255
|
+
if (serial !== void 0) {
|
|
2256
|
+
row = devices.find((d) => d.serial === serial);
|
|
2257
|
+
if (!row) throw new DeviceNotFoundError(`no adb device with serial ${serial} — attached: ${devices.map((d) => `${d.serial} (${d.state})`).join(", ") || "none"}`);
|
|
2258
|
+
} else {
|
|
2259
|
+
const ready = devices.filter((d) => d.state === "device");
|
|
2260
|
+
if (ready.length === 0) {
|
|
2261
|
+
const unauthorized = devices.find((d) => d.state === "unauthorized");
|
|
2262
|
+
throw new DeviceNotFoundError(unauthorized ? `device ${unauthorized.serial} is attached but unauthorized — unlock it and accept the "Allow USB debugging" prompt` : `no Android device attached — ${FIX_USB_DEBUGGING}`);
|
|
2263
|
+
}
|
|
2264
|
+
if (ready.length > 1) throw new DeviceNotFoundError(`several Android devices attached (${ready.map((d) => d.serial).join(", ")}) — pass a serial`);
|
|
2265
|
+
row = ready[0];
|
|
2266
|
+
}
|
|
2267
|
+
if (row.state !== "device") throw new DeviceNotFoundError(row.state === "unauthorized" ? `device ${row.serial} is unauthorized — unlock it and accept the "Allow USB debugging" prompt` : `device ${row.serial} is ${row.state} — ${FIX_USB_DEBUGGING}`);
|
|
2268
|
+
return finishHandle$1(row, false, options, () => Promise.resolve());
|
|
2269
|
+
}
|
|
2270
|
+
function defaultEmulatorBin() {
|
|
2271
|
+
const root = process.env.ANDROID_HOME ?? process.env.ANDROID_SDK_ROOT;
|
|
2272
|
+
return root ? `${root}/emulator/emulator` : "emulator";
|
|
2273
|
+
}
|
|
2274
|
+
function defaultSpawn(file, args) {
|
|
2275
|
+
const child = spawn(file, args, { stdio: "ignore" });
|
|
2276
|
+
let exited = false;
|
|
2277
|
+
child.once("exit", () => {
|
|
2278
|
+
exited = true;
|
|
2279
|
+
});
|
|
2280
|
+
child.once("error", () => {
|
|
2281
|
+
exited = true;
|
|
2282
|
+
child.emit("exit");
|
|
2283
|
+
});
|
|
2284
|
+
return {
|
|
2285
|
+
kill: (signal) => {
|
|
2286
|
+
child.kill(signal);
|
|
2287
|
+
},
|
|
2288
|
+
once: (event, listener) => child.once(event, listener),
|
|
2289
|
+
get exited() {
|
|
2290
|
+
return exited;
|
|
2291
|
+
}
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
/**
|
|
2295
|
+
* Boot a DEDICATED emulator instance from an AVD and return a {@link Device}
|
|
2296
|
+
* pinned to its serial (`emulator-<port>`). `close()` kills the instance.
|
|
2297
|
+
* `readOnly: true` lets several instances of one AVD run side by side, which
|
|
2298
|
+
* is how a cloud worker turns one golden image into N phones.
|
|
2299
|
+
*/
|
|
2300
|
+
async function launch$1(options) {
|
|
2301
|
+
const exec = options.exec ?? defaultExecRunner;
|
|
2302
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
2303
|
+
const adbBin = options.adbBin ?? "adb";
|
|
2304
|
+
const port = options.port ?? 5554;
|
|
2305
|
+
if (port % 2 !== 0 || port < 5554 || port > 5682) throw new ActionFailedError(`emulator port must be an even number in 5554..5682, got ${port}`);
|
|
2306
|
+
const serial = `emulator-${port}`;
|
|
2307
|
+
const bootTimeoutMs = options.bootTimeoutMs ?? 18e4;
|
|
2308
|
+
const pollMs = options.pollIntervalMs ?? 1e3;
|
|
2309
|
+
const spawnEmulator = options.spawn ?? defaultSpawn;
|
|
2310
|
+
const args = [
|
|
2311
|
+
"-avd",
|
|
2312
|
+
options.avd,
|
|
2313
|
+
"-port",
|
|
2314
|
+
String(port),
|
|
2315
|
+
"-no-boot-anim",
|
|
2316
|
+
"-no-audio",
|
|
2317
|
+
...options.headless === false ? [] : ["-no-window"],
|
|
2318
|
+
...options.readOnly ? ["-read-only"] : [],
|
|
2319
|
+
...options.extraArgs ?? []
|
|
2320
|
+
];
|
|
2321
|
+
const child = spawnEmulator(options.emulatorBin ?? defaultEmulatorBin(), args);
|
|
2322
|
+
const stop = async () => {
|
|
2323
|
+
await runAdb(exec, adbBin, [
|
|
2324
|
+
"-s",
|
|
2325
|
+
serial,
|
|
2326
|
+
"emu",
|
|
2327
|
+
"kill"
|
|
2328
|
+
], 1e4).catch(() => void 0);
|
|
2329
|
+
if (child.exited) return;
|
|
2330
|
+
await new Promise((resolve) => {
|
|
2331
|
+
const timer = setTimeout(() => {
|
|
2332
|
+
child.kill("SIGKILL");
|
|
2333
|
+
resolve();
|
|
2334
|
+
}, 1e4);
|
|
2335
|
+
child.once("exit", () => {
|
|
2336
|
+
clearTimeout(timer);
|
|
2337
|
+
resolve();
|
|
2338
|
+
});
|
|
2339
|
+
});
|
|
2340
|
+
};
|
|
2341
|
+
try {
|
|
2342
|
+
const deadline = Date.now() + bootTimeoutMs;
|
|
2343
|
+
await runAdb(exec, adbBin, [
|
|
2344
|
+
"-s",
|
|
2345
|
+
serial,
|
|
2346
|
+
"wait-for-device"
|
|
2347
|
+
], bootTimeoutMs);
|
|
2348
|
+
for (;;) {
|
|
2349
|
+
if (child.exited) throw new ActionFailedError(`emulator ${serial} exited during boot`);
|
|
2350
|
+
if ((await runAdb(exec, adbBin, [
|
|
2351
|
+
"-s",
|
|
2352
|
+
serial,
|
|
2353
|
+
"shell",
|
|
2354
|
+
"getprop",
|
|
2355
|
+
"sys.boot_completed"
|
|
2356
|
+
], 15e3).catch(() => "")).trim() === "1") break;
|
|
2357
|
+
if (Date.now() >= deadline) throw new TimeoutError(`emulator ${serial} did not finish booting within ${bootTimeoutMs}ms`);
|
|
2358
|
+
await sleep(pollMs);
|
|
2359
|
+
}
|
|
2360
|
+
} catch (err) {
|
|
2361
|
+
await stop().catch(() => void 0);
|
|
2362
|
+
throw err;
|
|
2363
|
+
}
|
|
2364
|
+
return finishHandle$1({
|
|
2365
|
+
serial,
|
|
2366
|
+
state: "device",
|
|
2367
|
+
model: options.avd
|
|
2368
|
+
}, true, options, stop);
|
|
2369
|
+
}
|
|
2370
|
+
/**
|
|
2371
|
+
* The Android engine object (the `ios` twin): `android.connect()` for a device
|
|
2372
|
+
* adb already sees, `android.launch()` for a dedicated emulator instance,
|
|
2373
|
+
* `android.list()` to see what is attached. All return/describe the same
|
|
2374
|
+
* Device type the iOS engine does.
|
|
2375
|
+
*/
|
|
2376
|
+
const android = {
|
|
2377
|
+
/** Attach to an attached device or running emulator by serial (no-arg: the single ready device). */
|
|
2378
|
+
connect: connect$1,
|
|
2379
|
+
/** Boot a dedicated emulator instance from an AVD and return a Device pinned to it. */
|
|
2380
|
+
launch: launch$1,
|
|
2381
|
+
/** Devices adb currently sees, with their state. */
|
|
2382
|
+
list
|
|
2383
|
+
};
|
|
2384
|
+
//#endregion
|
|
2385
|
+
//#region src/backends/cloud-sandbox.ts
|
|
2386
|
+
var CloudSandboxBackend = class extends BaseDeviceBackend {
|
|
2387
|
+
endpoint;
|
|
2388
|
+
token;
|
|
2389
|
+
fetchImpl;
|
|
2390
|
+
constructor(opts) {
|
|
2391
|
+
super("phone-use-cloud", opts.capabilities ?? ALL_CAPABILITIES);
|
|
2392
|
+
this.endpoint = opts.endpoint.replace(/\/+$/, "");
|
|
2393
|
+
this.token = opts.token;
|
|
2394
|
+
this.fetchImpl = opts.fetch ?? globalThis.fetch;
|
|
2395
|
+
}
|
|
2396
|
+
async rpc(method, ...args) {
|
|
2397
|
+
const response = await this.fetchImpl(`${this.endpoint}/rpc`, {
|
|
2398
|
+
method: "POST",
|
|
2399
|
+
headers: {
|
|
2400
|
+
authorization: `Bearer ${this.token}`,
|
|
2401
|
+
"content-type": "application/json"
|
|
2402
|
+
},
|
|
2403
|
+
body: JSON.stringify({
|
|
2404
|
+
method,
|
|
2405
|
+
args
|
|
2406
|
+
})
|
|
2407
|
+
});
|
|
2408
|
+
const body = await response.json().catch(() => ({}));
|
|
2409
|
+
if (!response.ok || !("ok" in body)) throw new PhoneUseError(("error" in body && typeof body.error === "string" ? body.error : void 0) ?? ("message" in body ? body.message : void 0) ?? `HTTP ${response.status}`, {
|
|
2410
|
+
code: "UNKNOWN",
|
|
2411
|
+
retryable: false
|
|
2412
|
+
});
|
|
2413
|
+
if (!body.ok) throw new PhoneUseError(body.error.message, {
|
|
2414
|
+
code: normalizeErrorCode(body.error.code),
|
|
2415
|
+
retryable: body.error.retryable ?? false
|
|
2416
|
+
});
|
|
2417
|
+
return body.result;
|
|
2418
|
+
}
|
|
2419
|
+
snapshot(opts) {
|
|
2420
|
+
return this.rpc("snapshot", opts);
|
|
2421
|
+
}
|
|
2422
|
+
async screenshot(opts) {
|
|
2423
|
+
const result = await this.rpc("screenshot", { overlayRefs: opts.overlayRefs });
|
|
2424
|
+
await writeFile(opts.path, Buffer.from(result.base64, "base64"));
|
|
2425
|
+
return { path: opts.path };
|
|
2426
|
+
}
|
|
2427
|
+
press(target) {
|
|
2428
|
+
return this.rpc("press", target);
|
|
2429
|
+
}
|
|
2430
|
+
longPress(ref, durationMs) {
|
|
2431
|
+
return this.rpc("longPress", ref, durationMs);
|
|
2432
|
+
}
|
|
2433
|
+
fill(ref, text) {
|
|
2434
|
+
return this.rpc("fill", ref, text);
|
|
2435
|
+
}
|
|
2436
|
+
typeText(text) {
|
|
2437
|
+
return this.rpc("typeText", text);
|
|
2438
|
+
}
|
|
2439
|
+
pressKey(key) {
|
|
2440
|
+
return this.rpc("pressKey", key);
|
|
2441
|
+
}
|
|
2442
|
+
scroll(direction) {
|
|
2443
|
+
return this.rpc("scroll", direction);
|
|
2444
|
+
}
|
|
2445
|
+
pan(x, y, dx, dy, durationMs) {
|
|
2446
|
+
return this.rpc("pan", x, y, dx, dy, durationMs);
|
|
2447
|
+
}
|
|
2448
|
+
waitForText(text, timeoutMs) {
|
|
2449
|
+
return this.rpc("waitForText", text, timeoutMs);
|
|
2450
|
+
}
|
|
2451
|
+
systemAlert(action) {
|
|
2452
|
+
return this.rpc("systemAlert", action);
|
|
2453
|
+
}
|
|
2454
|
+
home() {
|
|
2455
|
+
return this.rpc("home");
|
|
2456
|
+
}
|
|
2457
|
+
back() {
|
|
2458
|
+
return this.rpc("back");
|
|
2459
|
+
}
|
|
2460
|
+
openApp(opts) {
|
|
2461
|
+
return this.rpc("openApp", opts);
|
|
2462
|
+
}
|
|
2463
|
+
listApps() {
|
|
2464
|
+
return this.rpc("listApps");
|
|
2465
|
+
}
|
|
2466
|
+
closeSession() {
|
|
2467
|
+
return this.rpc("closeSession");
|
|
2468
|
+
}
|
|
2469
|
+
/** Save the device's current state. Returns the checkpoint's id. */
|
|
2470
|
+
checkpoint(opts) {
|
|
2471
|
+
return this.rpc("checkpoint", opts);
|
|
2472
|
+
}
|
|
2473
|
+
/** Restore a checkpoint by id or label. The checkpoint survives — restore as
|
|
2474
|
+
* often as needed. Device identity may change server-side; the sandbox
|
|
2475
|
+
* endpoint/token stay valid. */
|
|
2476
|
+
restoreCheckpoint(ref) {
|
|
2477
|
+
return this.rpc("restoreCheckpoint", ref);
|
|
2478
|
+
}
|
|
2479
|
+
/** List this sandbox's checkpoints. */
|
|
2480
|
+
listCheckpoints() {
|
|
2481
|
+
return this.rpc("listCheckpoints");
|
|
2482
|
+
}
|
|
2483
|
+
/** Delete a checkpoint by id or label. */
|
|
2484
|
+
deleteCheckpoint(ref) {
|
|
2485
|
+
return this.rpc("deleteCheckpoint", ref);
|
|
2486
|
+
}
|
|
2487
|
+
/** Upload a zipped .app bundle (base64) and install it on the sandbox device. */
|
|
2488
|
+
/**
|
|
2489
|
+
* Upload a zipped .app as a raw stream. Prefer this over {@link installApp}:
|
|
2490
|
+
* base64 inside a JSON body inflates the payload by a third, which caps real
|
|
2491
|
+
* apps near 144MB. Pass a `Bun.file(...)` (or any Blob) and the bytes go to the
|
|
2492
|
+
* wire without being buffered in memory.
|
|
2493
|
+
*/
|
|
2494
|
+
async installAppStream(zip) {
|
|
2495
|
+
const response = await this.fetchImpl(`${this.endpoint}/app`, {
|
|
2496
|
+
method: "POST",
|
|
2497
|
+
headers: {
|
|
2498
|
+
authorization: `Bearer ${this.token}`,
|
|
2499
|
+
"content-type": "application/zip"
|
|
2500
|
+
},
|
|
2501
|
+
body: zip
|
|
2502
|
+
});
|
|
2503
|
+
if (response.status === 404) {
|
|
2504
|
+
const bytes = await zip.arrayBuffer();
|
|
2505
|
+
return this.installApp(Buffer.from(bytes).toString("base64"));
|
|
2506
|
+
}
|
|
2507
|
+
const body = await response.json().catch(() => ({}));
|
|
2508
|
+
if (!response.ok || !("ok" in body)) throw new PhoneUseError(("error" in body && typeof body.error === "string" ? body.error : void 0) ?? `HTTP ${response.status}`, {
|
|
2509
|
+
code: "UNKNOWN",
|
|
2510
|
+
retryable: false
|
|
2511
|
+
});
|
|
2512
|
+
if (!body.ok) throw new PhoneUseError(body.error.message, {
|
|
2513
|
+
code: normalizeErrorCode(body.error.code),
|
|
2514
|
+
retryable: body.error.retryable ?? false
|
|
2515
|
+
});
|
|
2516
|
+
return body.result;
|
|
2517
|
+
}
|
|
2518
|
+
/** @deprecated Base64-in-JSON; use {@link installAppStream}. */
|
|
2519
|
+
installApp(base64Zip) {
|
|
2520
|
+
return this.rpc("installApp", base64Zip);
|
|
2521
|
+
}
|
|
2522
|
+
};
|
|
2523
|
+
/**
|
|
2524
|
+
* Build a CloudSandboxBackend from explicit options or the environment:
|
|
2525
|
+
* PHONE_USE_SANDBOX_URL + PHONE_USE_SANDBOX_TOKEN (printed by
|
|
2526
|
+
* `phone-use sandbox env <id>`).
|
|
2527
|
+
*/
|
|
2528
|
+
function createCloudSandboxBackend(config) {
|
|
2529
|
+
const endpoint = config?.endpoint ?? process.env.PHONE_USE_SANDBOX_URL;
|
|
2530
|
+
const token = config?.token ?? process.env.PHONE_USE_SANDBOX_TOKEN;
|
|
2531
|
+
if (!endpoint || !token) throw new PhoneUseError("cloud sandbox backend needs PHONE_USE_SANDBOX_URL and PHONE_USE_SANDBOX_TOKEN (see `phone-use sandbox env <id>`)", {
|
|
2532
|
+
code: "BACKEND_NOT_FOUND",
|
|
2533
|
+
retryable: false
|
|
2534
|
+
});
|
|
2535
|
+
return new CloudSandboxBackend({
|
|
2536
|
+
...config,
|
|
2537
|
+
endpoint,
|
|
2538
|
+
token
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
function normalizeErrorCode(code) {
|
|
2542
|
+
if ((/* @__PURE__ */ new Set([
|
|
2543
|
+
"DEVICE_NOT_FOUND",
|
|
2544
|
+
"DEVICE_IN_USE",
|
|
2545
|
+
"SESSION_NOT_FOUND",
|
|
2546
|
+
"TIMEOUT",
|
|
2547
|
+
"ACTION_FAILED",
|
|
2548
|
+
"UNSUPPORTED_CAPABILITY",
|
|
2549
|
+
"BACKEND_NOT_FOUND",
|
|
2550
|
+
"ABORTED",
|
|
2551
|
+
"UNKNOWN"
|
|
2552
|
+
])).has(code ?? "")) return code;
|
|
2553
|
+
return {
|
|
2554
|
+
device_gone: "DEVICE_NOT_FOUND",
|
|
2555
|
+
sandbox_not_found: "DEVICE_NOT_FOUND",
|
|
2556
|
+
timeout: "TIMEOUT",
|
|
2557
|
+
session_conflict: "DEVICE_IN_USE",
|
|
2558
|
+
target_not_found: "ACTION_FAILED",
|
|
2559
|
+
unauthorized: "ABORTED",
|
|
2560
|
+
payload_too_large: "ACTION_FAILED",
|
|
2561
|
+
bad_request: "ACTION_FAILED",
|
|
2562
|
+
internal: "UNKNOWN"
|
|
2563
|
+
}[code ?? ""] ?? "UNKNOWN";
|
|
2564
|
+
}
|
|
2565
|
+
//#endregion
|
|
2566
|
+
//#region src/backends/device-runner.ts
|
|
2567
|
+
const RUNNER_CAPABILITIES = [
|
|
2568
|
+
"snapshot",
|
|
2569
|
+
"screenshot",
|
|
2570
|
+
"press",
|
|
2571
|
+
"fill",
|
|
2572
|
+
"type",
|
|
2573
|
+
"scroll",
|
|
2574
|
+
"pan",
|
|
2575
|
+
"openApp",
|
|
2576
|
+
"home"
|
|
2577
|
+
];
|
|
2578
|
+
/**
|
|
2579
|
+
* Backend that speaks to an on-device runner: an XCTest-hosted JSON-RPC server
|
|
2580
|
+
* running ON the iPhone itself, which holds the automation privileges iOS
|
|
2581
|
+
* denies to ordinary apps.
|
|
2582
|
+
*
|
|
2583
|
+
* The endpoint is just a URL, so the same backend serves every topology:
|
|
2584
|
+
* - `http://127.0.0.1:45678` — port-forwarded from a paired host
|
|
2585
|
+
* - `http://<phone-ip>:45678` — straight over the LAN / tailnet
|
|
2586
|
+
* - `https://relay.example/d/<id>` — the runner dials out to a cloud relay,
|
|
2587
|
+
* which is what lets an agent anywhere drive the phone with no inbound
|
|
2588
|
+
* ports and no Mac in the loop.
|
|
2589
|
+
*
|
|
2590
|
+
* The wire format matches the shape proven by rounak/PhoneAgent: newline-free
|
|
2591
|
+
* JSON request/response over HTTP POST, one method per call.
|
|
2592
|
+
*/
|
|
2593
|
+
var DeviceRunnerBackend = class extends BaseDeviceBackend {
|
|
2594
|
+
#endpoint;
|
|
2595
|
+
#token;
|
|
2596
|
+
#timeoutMs;
|
|
2597
|
+
constructor(config) {
|
|
2598
|
+
super("device-runner", RUNNER_CAPABILITIES);
|
|
2599
|
+
const endpoint = config?.endpoint ?? process.env.PHONE_USE_RUNNER_URL ?? "http://127.0.0.1:45678";
|
|
2600
|
+
this.#endpoint = endpoint.replace(/\/+$/, "");
|
|
2601
|
+
this.#token = config?.token ?? process.env.PHONE_USE_RUNNER_TOKEN;
|
|
2602
|
+
this.#timeoutMs = config?.timeoutMs ?? 3e4;
|
|
2603
|
+
}
|
|
2604
|
+
async #rpc(method, params = {}) {
|
|
2605
|
+
const controller = new AbortController();
|
|
2606
|
+
const timer = setTimeout(() => controller.abort(), this.#timeoutMs);
|
|
2607
|
+
let res;
|
|
2608
|
+
try {
|
|
2609
|
+
res = await fetch(this.#endpoint, {
|
|
2610
|
+
method: "POST",
|
|
2611
|
+
headers: {
|
|
2612
|
+
"content-type": "application/json",
|
|
2613
|
+
...this.#token ? { authorization: `Bearer ${this.#token}` } : {}
|
|
2614
|
+
},
|
|
2615
|
+
body: JSON.stringify({
|
|
2616
|
+
jsonrpc: "2.0",
|
|
2617
|
+
id: Date.now(),
|
|
2618
|
+
method,
|
|
2619
|
+
params
|
|
2620
|
+
}),
|
|
2621
|
+
signal: controller.signal
|
|
2622
|
+
});
|
|
2623
|
+
} catch (cause) {
|
|
2624
|
+
if (controller.signal.aborted) throw new TimeoutError(`runner did not answer ${method} within ${this.#timeoutMs}ms`);
|
|
2625
|
+
throw new DeviceNotFoundError(`cannot reach the on-device runner at ${this.#endpoint} — is it activated on the phone?`, { cause });
|
|
2626
|
+
} finally {
|
|
2627
|
+
clearTimeout(timer);
|
|
2628
|
+
}
|
|
2629
|
+
if (!res.ok) throw new ActionFailedError(`runner returned HTTP ${res.status} for ${method}`);
|
|
2630
|
+
const body = await res.json();
|
|
2631
|
+
if (body.error) throw new ActionFailedError(body.error.message ?? `runner rejected ${method}`);
|
|
2632
|
+
return body.result;
|
|
2633
|
+
}
|
|
2634
|
+
async snapshot(opts) {
|
|
2635
|
+
const wire = await this.#rpc("get_tree", {
|
|
2636
|
+
interactiveOnly: opts?.interactiveOnly ?? false,
|
|
2637
|
+
depth: opts?.depth
|
|
2638
|
+
});
|
|
2639
|
+
return {
|
|
2640
|
+
appBundleId: wire.app,
|
|
2641
|
+
appName: wire.app,
|
|
2642
|
+
nodes: (wire.elements ?? []).map((e) => ({
|
|
2643
|
+
ref: e.ref,
|
|
2644
|
+
role: e.role,
|
|
2645
|
+
type: e.role,
|
|
2646
|
+
label: e.label,
|
|
2647
|
+
value: e.value,
|
|
2648
|
+
enabled: e.enabled,
|
|
2649
|
+
rect: e.rect ? {
|
|
2650
|
+
x: e.rect.x,
|
|
2651
|
+
y: e.rect.y,
|
|
2652
|
+
width: e.rect.w,
|
|
2653
|
+
height: e.rect.h
|
|
2654
|
+
} : void 0
|
|
2655
|
+
}))
|
|
2656
|
+
};
|
|
2657
|
+
}
|
|
2658
|
+
async screenshot(opts) {
|
|
2659
|
+
const { base64 } = await this.#rpc("get_screen_image", { overlayRefs: opts.overlayRefs ?? false });
|
|
2660
|
+
const { writeFile } = await import("node:fs/promises");
|
|
2661
|
+
await writeFile(opts.path, Buffer.from(base64, "base64"));
|
|
2662
|
+
return { path: opts.path };
|
|
2663
|
+
}
|
|
2664
|
+
async press(target) {
|
|
2665
|
+
if ("ref" in target) {
|
|
2666
|
+
await this.#rpc("tap_element", { ref: target.ref });
|
|
2667
|
+
return;
|
|
2668
|
+
}
|
|
2669
|
+
await this.#rpc("tap", {
|
|
2670
|
+
x: target.x,
|
|
2671
|
+
y: target.y
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
async fill(ref, text) {
|
|
2675
|
+
await this.#rpc("enter_text", {
|
|
2676
|
+
ref,
|
|
2677
|
+
text,
|
|
2678
|
+
replace: true
|
|
2679
|
+
});
|
|
2680
|
+
}
|
|
2681
|
+
async typeText(text) {
|
|
2682
|
+
await this.#rpc("enter_text", {
|
|
2683
|
+
text,
|
|
2684
|
+
replace: false
|
|
2685
|
+
});
|
|
2686
|
+
}
|
|
2687
|
+
async scroll(direction) {
|
|
2688
|
+
await this.#rpc("scroll", { direction });
|
|
2689
|
+
}
|
|
2690
|
+
async pan(x, y, dx, dy, durationMs = 300) {
|
|
2691
|
+
await this.#rpc("swipe", {
|
|
2692
|
+
x,
|
|
2693
|
+
y,
|
|
2694
|
+
dx,
|
|
2695
|
+
dy,
|
|
2696
|
+
durationMs
|
|
2697
|
+
});
|
|
2698
|
+
}
|
|
2699
|
+
async home() {
|
|
2700
|
+
await this.#rpc("home");
|
|
2701
|
+
}
|
|
2702
|
+
async openApp(opts) {
|
|
2703
|
+
return this.#rpc("open_app", {
|
|
2704
|
+
app: opts.app,
|
|
2705
|
+
url: opts.url,
|
|
2706
|
+
relaunch: opts.relaunch ?? false
|
|
2707
|
+
});
|
|
2708
|
+
}
|
|
2709
|
+
async systemAlert(action) {
|
|
2710
|
+
return this.#rpc("alert", { action });
|
|
2711
|
+
}
|
|
2712
|
+
async closeSession() {}
|
|
2713
|
+
/** Liveness probe used by `phone-use doctor` and the relay health check. */
|
|
2714
|
+
async ping() {
|
|
2715
|
+
return this.#rpc("get_context");
|
|
2716
|
+
}
|
|
2717
|
+
};
|
|
2718
|
+
/** Backend for a phone-runner reached by URL (defaults come from
|
|
2719
|
+
* `PHONE_USE_RUNNER_URL` / `PHONE_USE_RUNNER_TOKEN` when `config` is omitted). */
|
|
2720
|
+
const createDeviceRunnerBackend = (config) => new DeviceRunnerBackend(config);
|
|
2721
|
+
//#endregion
|
|
1973
2722
|
//#region src/backends/ios.ts
|
|
1974
2723
|
const UDID_RE = /^[0-9A-F]{8}(-[0-9A-F]{4}){3}-[0-9A-F]{12}$/i;
|
|
1975
2724
|
function simctlArgs(setPath, args) {
|
|
@@ -2122,16 +2871,17 @@ const ios = {
|
|
|
2122
2871
|
//#region src/index.ts
|
|
2123
2872
|
/**
|
|
2124
2873
|
* @phone-use/sdk — the device runtime SDK: engine-as-object lifecycle
|
|
2125
|
-
* (ios.launch/connect → Device), Device backends, config, errors, capabilities,
|
|
2874
|
+
* (ios.launch/connect and android.launch/connect → Device), Device backends, config, errors, capabilities,
|
|
2126
2875
|
* and the action verb surface.
|
|
2127
2876
|
*
|
|
2128
2877
|
* The test double (FakeBackend) lives on the "@phone-use/sdk/testing" subpath,
|
|
2129
2878
|
* deliberately not re-exported here.
|
|
2130
2879
|
*/
|
|
2131
2880
|
/** The published package version (kept in sync with package.json by the release flow). */
|
|
2132
|
-
const VERSION = "0.
|
|
2881
|
+
const VERSION = "0.5.0";
|
|
2133
2882
|
registerBackend("agent-device", createAgentDeviceBackend);
|
|
2883
|
+
registerBackend("android-adb", createAndroidBackend);
|
|
2134
2884
|
//#endregion
|
|
2135
|
-
export { ALL_CAPABILITIES, AbortedError, ActionFailedError, BaseDeviceBackend, CloudSandboxBackend, DeviceCore, DeviceInUseError, DeviceNotFoundError, DeviceRunnerBackend, PhoneUseError, SecretStore, SessionNotFoundError, TimeoutError, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createCloudSandboxBackend, createDeviceHandle, createDeviceRunnerBackend, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
|
|
2885
|
+
export { ALL_CAPABILITIES, AbortedError, ActionFailedError, AndroidBackend, BaseDeviceBackend, CloudSandboxBackend, DeviceCore, DeviceInUseError, DeviceNotFoundError, DeviceRunnerBackend, PhoneUseError, SecretStore, SessionNotFoundError, TimeoutError, UnsupportedCapabilityError, VERSION, android, buildObserveResult, createAgentDeviceBackend, createAndroidBackend, createCloudSandboxBackend, createDeviceHandle, createDeviceRunnerBackend, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
|
|
2136
2886
|
|
|
2137
2887
|
//# sourceMappingURL=index.mjs.map
|