@ishlabs/cli 0.25.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor.d.ts +42 -0
- package/dist/commands/doctor.js +359 -0
- package/dist/commands/iteration.js +23 -5
- package/dist/commands/study-participant.js +1 -1
- package/dist/commands/study-run.js +26 -1
- package/dist/commands/study-screenshots.js +38 -5
- package/dist/index.js +2 -0
- package/dist/lib/api-client.d.ts +3 -0
- package/dist/lib/api-client.js +6 -1
- package/dist/lib/docs.js +15 -3
- package/dist/lib/local-sim/actions.d.ts +18 -0
- package/dist/lib/local-sim/actions.js +32 -0
- package/dist/lib/local-sim/adb.d.ts +33 -0
- package/dist/lib/local-sim/adb.js +121 -17
- package/dist/lib/local-sim/android.d.ts +7 -1
- package/dist/lib/local-sim/android.js +21 -1
- package/dist/lib/local-sim/coordinates.d.ts +4 -4
- package/dist/lib/local-sim/coordinates.js +4 -4
- package/dist/lib/local-sim/device.d.ts +21 -2
- package/dist/lib/local-sim/device.js +1 -1
- package/dist/lib/local-sim/ios.d.ts +33 -10
- package/dist/lib/local-sim/ios.js +88 -20
- package/dist/lib/local-sim/loop.js +134 -25
- package/dist/lib/local-sim/native-a11y.d.ts +21 -7
- package/dist/lib/local-sim/native-a11y.js +82 -47
- package/dist/lib/local-sim/simctl.d.ts +28 -43
- package/dist/lib/local-sim/simctl.js +53 -142
- package/dist/lib/local-sim/types.d.ts +13 -2
- package/dist/lib/local-sim/xcuitest.d.ts +60 -0
- package/dist/lib/local-sim/xcuitest.js +303 -0
- package/dist/lib/paths.d.ts +14 -0
- package/dist/lib/paths.js +21 -0
- package/dist/lib/report-readiness.d.ts +44 -0
- package/dist/lib/report-readiness.js +74 -0
- package/dist/lib/skill-content.js +2 -0
- package/package.json +1 -1
package/dist/lib/docs.js
CHANGED
|
@@ -380,6 +380,14 @@ ish iteration create --platform figma --url https://figma.com/proto \\
|
|
|
380
380
|
--screen-format mobile_portrait --file-key abc123 --start-node-id 0:1 \\
|
|
381
381
|
--flow-name "Onboarding A"
|
|
382
382
|
|
|
383
|
+
# Native app (ios / android): --app names the target, stored as app_artifact (no URL).
|
|
384
|
+
ish iteration create --platform ios --app com.example.app
|
|
385
|
+
ish iteration create --platform ios # --app optional; "chosen at run time"
|
|
386
|
+
# drive it locally against a booted simulator / emulator — the iteration
|
|
387
|
+
# remembers the app, so no --app needed on reruns:
|
|
388
|
+
ish study run --local
|
|
389
|
+
ish study run --local --app ./Build.app # override with a fresh local build
|
|
390
|
+
|
|
383
391
|
# Text/email content from a file:
|
|
384
392
|
ish iteration create --content-text @./email.html --title "Newsletter"
|
|
385
393
|
|
|
@@ -1992,10 +2000,14 @@ Interactive study runs produce per-frame screenshots server-side. They
|
|
|
1992
2000
|
let you (or an agent) see what participants actually saw alongside the
|
|
1993
2001
|
sentiment summary.
|
|
1994
2002
|
|
|
1995
|
-
## Screenshots — interactive studies only
|
|
2003
|
+
## Screenshots — remote interactive studies only
|
|
1996
2004
|
|
|
1997
|
-
Screenshots are produced by interactive runs only — chat / video /
|
|
1998
|
-
studies don't have them.
|
|
2005
|
+
Screenshots are produced by remote interactive runs only — chat / video /
|
|
2006
|
+
text studies don't have them. **Local runs** (\`ish study run --local\`,
|
|
2007
|
+
including ios/android) don't push screenshots to the server either; they
|
|
2008
|
+
write a per-step HTML debug report to \`~/.ish/debug/sim-*.html\` (the path is
|
|
2009
|
+
printed at the end of the run). \`ish study screenshots list\` on a local-only
|
|
2010
|
+
study therefore returns nothing useful — open the debug report instead.
|
|
1999
2011
|
|
|
2000
2012
|
### CLI
|
|
2001
2013
|
|
|
@@ -25,6 +25,24 @@ export declare function resolveTextValue(action: LocalStepAction, contextValues:
|
|
|
25
25
|
* Compare two base64 screenshots to detect visible change.
|
|
26
26
|
*/
|
|
27
27
|
export declare function detectNoVisibleChange(before: string, after: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Logical classification of what a completed step's batch of actions did to the
|
|
30
|
+
* screen, used to tell the backend's frame matcher whether the NEXT observation
|
|
31
|
+
* is the same logical screen (so it reuses the previous frame instead of minting
|
|
32
|
+
* a new one off shifted pixels).
|
|
33
|
+
*
|
|
34
|
+
* - "scroll": a pure vertical scroll (all actions succeeded) — same screen.
|
|
35
|
+
* - "keyboard": raised the keyboard without submitting/navigating — same screen.
|
|
36
|
+
* - "ui_change": anything that (likely) changed the logical screen — NOT same.
|
|
37
|
+
* - "none": nothing actionable happened (empty batch / think-only).
|
|
38
|
+
*/
|
|
39
|
+
export type StepKind = "scroll" | "keyboard" | "ui_change" | "none";
|
|
40
|
+
/**
|
|
41
|
+
* Classify a step's batch of actions into a StepKind. Pure — `successByIndex[i]`
|
|
42
|
+
* is the executor's `result.success` for `actions[i]`. `think` actions are
|
|
43
|
+
* ignored entirely (they neither move the screen nor count toward a verdict).
|
|
44
|
+
*/
|
|
45
|
+
export declare function classifyStepKind(actions: LocalStepAction[], successByIndex: boolean[]): StepKind;
|
|
28
46
|
/**
|
|
29
47
|
* Build a human-readable action description matching backend's format_action_detail().
|
|
30
48
|
*/
|
|
@@ -403,6 +403,36 @@ function isRecoverableError(err) {
|
|
|
403
403
|
export function detectNoVisibleChange(before, after) {
|
|
404
404
|
return before === after;
|
|
405
405
|
}
|
|
406
|
+
// Scroll directions that keep us on the same logical screen (vertical pan).
|
|
407
|
+
// undefined/null defaults to a downward scroll in executeScroll(), so it counts.
|
|
408
|
+
const VERTICAL_SCROLL_DIRECTIONS = new Set([
|
|
409
|
+
"up", "down", "to_top", "to_bottom", "to_element",
|
|
410
|
+
]);
|
|
411
|
+
/**
|
|
412
|
+
* Classify a step's batch of actions into a StepKind. Pure — `successByIndex[i]`
|
|
413
|
+
* is the executor's `result.success` for `actions[i]`. `think` actions are
|
|
414
|
+
* ignored entirely (they neither move the screen nor count toward a verdict).
|
|
415
|
+
*/
|
|
416
|
+
export function classifyStepKind(actions, successByIndex) {
|
|
417
|
+
// Keep the original index so success lines up after dropping `think`s.
|
|
418
|
+
const considered = actions
|
|
419
|
+
.map((action, i) => ({ action, success: successByIndex[i] }))
|
|
420
|
+
.filter(({ action }) => action.type !== "think");
|
|
421
|
+
if (considered.length === 0)
|
|
422
|
+
return "none";
|
|
423
|
+
// Pure vertical scroll: every action a successful vertical scroll.
|
|
424
|
+
const allScroll = considered.every(({ action, success }) => action.type === "scroll" &&
|
|
425
|
+
success === true &&
|
|
426
|
+
VERTICAL_SCROLL_DIRECTIONS.has(action.direction ?? "down"));
|
|
427
|
+
if (allScroll)
|
|
428
|
+
return "scroll";
|
|
429
|
+
// Keyboard-only: every action a text_input that raises the keyboard WITHOUT
|
|
430
|
+
// submitting/navigating. A submit makes it a ui_change (it can navigate away).
|
|
431
|
+
const allKeyboard = considered.every(({ action }) => action.type === "text_input" && !action.submit);
|
|
432
|
+
if (allKeyboard)
|
|
433
|
+
return "keyboard";
|
|
434
|
+
return "ui_change";
|
|
435
|
+
}
|
|
406
436
|
/**
|
|
407
437
|
* Build a human-readable action description matching backend's format_action_detail().
|
|
408
438
|
*/
|
|
@@ -429,6 +459,8 @@ export function describeAction(action) {
|
|
|
429
459
|
return `wait ${action.duration_ms ?? 1000}ms`;
|
|
430
460
|
case "navigate_back":
|
|
431
461
|
return "navigate back";
|
|
462
|
+
case "open_system_panel":
|
|
463
|
+
return `open_system_panel (${action.panel ?? "notifications"})`;
|
|
432
464
|
case "long_press":
|
|
433
465
|
return `long_press on '${element}'${modSuffix}`;
|
|
434
466
|
case "double_tap":
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
* backend's 0-1000 coordinates against the screencap pixel size and taps
|
|
11
11
|
* directly. (Verified by the Layer-1 driver smoke; see scripts/mobile-e2e.)
|
|
12
12
|
*/
|
|
13
|
+
/** Resolve adb, downloading Google's platform-tools on first use if not found. */
|
|
14
|
+
export declare function ensureAdb(): Promise<string>;
|
|
13
15
|
export declare class AdbError extends Error {
|
|
14
16
|
constructor(message: string);
|
|
15
17
|
}
|
|
@@ -17,6 +19,27 @@ export declare class AdbError extends Error {
|
|
|
17
19
|
export declare function adb(args: string[], timeoutMs?: number): Promise<string>;
|
|
18
20
|
/** Run `adb shell <args>` and return trimmed stdout. */
|
|
19
21
|
export declare function adbShell(args: string[], timeoutMs?: number): Promise<string>;
|
|
22
|
+
/**
|
|
23
|
+
* Pull versionName / versionCode out of `dumpsys package <pkg>` text. The
|
|
24
|
+
* relevant lines read `versionCode=42 minSdk=24 targetSdk=34` and
|
|
25
|
+
* `versionName=1.2.3`; `\d+` stops the build before the trailing tokens and
|
|
26
|
+
* `\S+` takes the version up to the next space. Returns null when neither is
|
|
27
|
+
* present (wrong/empty package).
|
|
28
|
+
*/
|
|
29
|
+
export declare function parseDumpsysAppBuild(out: string): {
|
|
30
|
+
version: string | null;
|
|
31
|
+
build: string | null;
|
|
32
|
+
} | null;
|
|
33
|
+
/**
|
|
34
|
+
* Read an installed package's versionName / versionCode from
|
|
35
|
+
* `dumpsys package <pkg>`. Best-effort: returns null on any failure (the run
|
|
36
|
+
* never depends on it). Covers both freshly-installed apks and pre-installed
|
|
37
|
+
* packages — by call time the package name is already resolved.
|
|
38
|
+
*/
|
|
39
|
+
export declare function appBuildFromDevice(pkg: string): Promise<{
|
|
40
|
+
version: string | null;
|
|
41
|
+
build: string | null;
|
|
42
|
+
} | null>;
|
|
20
43
|
/**
|
|
21
44
|
* Capture the current screen as raw PNG bytes via `adb exec-out screencap -p`.
|
|
22
45
|
* `exec-out` (not `shell`) avoids the CRLF translation that corrupts binary
|
|
@@ -44,6 +67,16 @@ export declare function inputDrag(x1: number, y1: number, x2: number, y2: number
|
|
|
44
67
|
/** A long-press is a zero-distance swipe held for `durationMs`. */
|
|
45
68
|
export declare function inputLongPress(x: number, y: number, durationMs?: number): Promise<void>;
|
|
46
69
|
export declare function pressKeyEvent(keyevent: string): Promise<void>;
|
|
70
|
+
/**
|
|
71
|
+
* Open a system panel via `adb shell cmd statusbar expand-settings|expand-notifications`
|
|
72
|
+
* — a deterministic OS-driven open, chosen over a top-edge `input swipe` because the
|
|
73
|
+
* generic swipe is center-anchored and a synthetic top-edge swipe never registers the
|
|
74
|
+
* system pull-down. `expand-notifications` opens the notification shade;
|
|
75
|
+
* `expand-settings` pulls all the way to quick settings.
|
|
76
|
+
*/
|
|
77
|
+
export declare function statusbarExpand(panel: "expand-settings" | "expand-notifications"): Promise<void>;
|
|
78
|
+
/** Collapse the open system panel (`cmd statusbar collapse`). */
|
|
79
|
+
export declare function statusbarCollapse(): Promise<void>;
|
|
47
80
|
/**
|
|
48
81
|
* Force a device orientation. We first disable auto-rotation
|
|
49
82
|
* (`accelerometer_rotation 0`) — otherwise the sensor immediately overrides
|
|
@@ -10,30 +10,90 @@
|
|
|
10
10
|
* backend's 0-1000 coordinates against the screencap pixel size and taps
|
|
11
11
|
* directly. (Verified by the Layer-1 driver smoke; see scripts/mobile-e2e.)
|
|
12
12
|
*/
|
|
13
|
-
import { execFile } from "node:child_process";
|
|
14
|
-
import { existsSync } from "node:fs";
|
|
13
|
+
import { execFile, execFileSync } from "node:child_process";
|
|
14
|
+
import { existsSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
15
16
|
import { promisify } from "node:util";
|
|
17
|
+
import { binDir, adbBin } from "../paths.js";
|
|
16
18
|
const execFileAsync = promisify(execFile);
|
|
17
|
-
// adb
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
|
|
19
|
+
// Resolve adb without depending on the caller's PATH: ISH_ADB/ADB override → the
|
|
20
|
+
// Android SDK → Homebrew → our own download cache → PATH. If none is found,
|
|
21
|
+
// ensureAdb() fetches Google's standalone platform-tools (a small zip) into
|
|
22
|
+
// ~/.ish/bin, mirroring how cloudflared / the iOS WebDriverAgent runner are
|
|
23
|
+
// fetched. Override the binary with ISH_ADB / ADB.
|
|
24
|
+
function findAdb() {
|
|
21
25
|
const fromEnv = process.env.ISH_ADB || process.env.ADB;
|
|
22
26
|
if (fromEnv && existsSync(fromEnv))
|
|
23
27
|
return fromEnv;
|
|
24
|
-
const homebrew = "/opt/homebrew/bin/adb";
|
|
25
|
-
if (existsSync(homebrew))
|
|
26
|
-
return homebrew;
|
|
27
28
|
const sdkHome = process.env.ANDROID_HOME || process.env.ANDROID_SDK_ROOT;
|
|
28
29
|
if (sdkHome) {
|
|
29
|
-
const sdkAdb =
|
|
30
|
+
const sdkAdb = join(sdkHome, "platform-tools", "adb");
|
|
30
31
|
if (existsSync(sdkAdb))
|
|
31
32
|
return sdkAdb;
|
|
32
33
|
}
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
const homebrew = "/opt/homebrew/bin/adb";
|
|
35
|
+
if (existsSync(homebrew))
|
|
36
|
+
return homebrew;
|
|
37
|
+
if (existsSync(adbBin()))
|
|
38
|
+
return adbBin(); // our downloaded cache
|
|
39
|
+
// PATH fallback — only if `adb` actually resolves there.
|
|
40
|
+
try {
|
|
41
|
+
execFileSync(process.platform === "win32" ? "where" : "which", ["adb"], { stdio: "ignore" });
|
|
42
|
+
return "adb";
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
let cachedAdb = null;
|
|
49
|
+
/** Resolve adb, downloading Google's platform-tools on first use if not found. */
|
|
50
|
+
export async function ensureAdb() {
|
|
51
|
+
if (cachedAdb)
|
|
52
|
+
return cachedAdb;
|
|
53
|
+
cachedAdb = findAdb() ?? (await downloadAdb());
|
|
54
|
+
return cachedAdb;
|
|
55
|
+
}
|
|
56
|
+
const PLATFORM_TOOLS_OS = {
|
|
57
|
+
darwin: "darwin",
|
|
58
|
+
linux: "linux",
|
|
59
|
+
win32: "windows",
|
|
60
|
+
};
|
|
61
|
+
/** Fetch + unpack Google's standalone platform-tools into ~/.ish/bin. */
|
|
62
|
+
async function downloadAdb() {
|
|
63
|
+
const os = PLATFORM_TOOLS_OS[process.platform];
|
|
64
|
+
if (!os) {
|
|
65
|
+
throw new AdbError(`no prebuilt adb for ${process.platform}; install Android platform-tools and set ISH_ADB`);
|
|
66
|
+
}
|
|
67
|
+
const url = `https://dl.google.com/android/repository/platform-tools-latest-${os}.zip`;
|
|
68
|
+
const dir = binDir();
|
|
69
|
+
console.error("Fetching adb (Android platform-tools) from Google...");
|
|
70
|
+
mkdirSync(dir, { recursive: true });
|
|
71
|
+
const zipPath = join(dir, "platform-tools.zip");
|
|
72
|
+
let resp;
|
|
73
|
+
try {
|
|
74
|
+
resp = await fetch(url, { signal: AbortSignal.timeout(120_000) });
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
throw new AdbError(`failed to download platform-tools from ${url}: ${e instanceof Error ? e.message : String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
if (!resp.ok)
|
|
80
|
+
throw new AdbError(`failed to download platform-tools: HTTP ${resp.status} from ${url}`);
|
|
81
|
+
writeFileSync(zipPath, Buffer.from(await resp.arrayBuffer()));
|
|
82
|
+
try {
|
|
83
|
+
// The zip carries a top-level `platform-tools/` dir; extract into binDir().
|
|
84
|
+
const [cmd, args] = process.platform === "win32"
|
|
85
|
+
? ["tar", ["-xf", zipPath, "-C", dir]]
|
|
86
|
+
: ["unzip", ["-o", "-q", zipPath, "-d", dir]];
|
|
87
|
+
await execFileAsync(cmd, args, { timeout: 120_000 });
|
|
88
|
+
}
|
|
89
|
+
catch (e) {
|
|
90
|
+
throw new AdbError(`failed to unpack platform-tools: ${e instanceof Error ? e.message : String(e)}`);
|
|
91
|
+
}
|
|
92
|
+
rmSync(zipPath, { force: true });
|
|
93
|
+
if (!existsSync(adbBin()))
|
|
94
|
+
throw new AdbError(`platform-tools unpacked but adb is missing at ${adbBin()}`);
|
|
95
|
+
return adbBin();
|
|
35
96
|
}
|
|
36
|
-
const ADB = resolveAdb();
|
|
37
97
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
38
98
|
// screencap on a cold emulator frame can be slow; give it generous headroom.
|
|
39
99
|
const SCREENCAP_TIMEOUT_MS = 30_000;
|
|
@@ -47,8 +107,9 @@ export class AdbError extends Error {
|
|
|
47
107
|
}
|
|
48
108
|
/** Run `adb <args>` and return trimmed stdout. Throws AdbError on failure. */
|
|
49
109
|
export async function adb(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
110
|
+
const bin = await ensureAdb();
|
|
50
111
|
try {
|
|
51
|
-
const { stdout } = await execFileAsync(
|
|
112
|
+
const { stdout } = await execFileAsync(bin, args, {
|
|
52
113
|
timeout: timeoutMs,
|
|
53
114
|
maxBuffer: 4 * 1024 * 1024,
|
|
54
115
|
});
|
|
@@ -63,14 +124,43 @@ export async function adb(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
|
63
124
|
export async function adbShell(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
64
125
|
return adb(["shell", ...args], timeoutMs);
|
|
65
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Pull versionName / versionCode out of `dumpsys package <pkg>` text. The
|
|
129
|
+
* relevant lines read `versionCode=42 minSdk=24 targetSdk=34` and
|
|
130
|
+
* `versionName=1.2.3`; `\d+` stops the build before the trailing tokens and
|
|
131
|
+
* `\S+` takes the version up to the next space. Returns null when neither is
|
|
132
|
+
* present (wrong/empty package).
|
|
133
|
+
*/
|
|
134
|
+
export function parseDumpsysAppBuild(out) {
|
|
135
|
+
const version = out.match(/versionName=(\S+)/)?.[1] ?? null;
|
|
136
|
+
const build = out.match(/versionCode=(\d+)/)?.[1] ?? null;
|
|
137
|
+
if (!version && !build)
|
|
138
|
+
return null;
|
|
139
|
+
return { version, build };
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Read an installed package's versionName / versionCode from
|
|
143
|
+
* `dumpsys package <pkg>`. Best-effort: returns null on any failure (the run
|
|
144
|
+
* never depends on it). Covers both freshly-installed apks and pre-installed
|
|
145
|
+
* packages — by call time the package name is already resolved.
|
|
146
|
+
*/
|
|
147
|
+
export async function appBuildFromDevice(pkg) {
|
|
148
|
+
try {
|
|
149
|
+
return parseDumpsysAppBuild(await adbShell(["dumpsys", "package", pkg], 30_000));
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
66
155
|
/**
|
|
67
156
|
* Capture the current screen as raw PNG bytes via `adb exec-out screencap -p`.
|
|
68
157
|
* `exec-out` (not `shell`) avoids the CRLF translation that corrupts binary
|
|
69
158
|
* output. Returns the PNG buffer at full device resolution.
|
|
70
159
|
*/
|
|
71
160
|
export async function screencapPng() {
|
|
161
|
+
const bin = await ensureAdb();
|
|
72
162
|
try {
|
|
73
|
-
const { stdout } = await execFileAsync(
|
|
163
|
+
const { stdout } = await execFileAsync(bin, ["exec-out", "screencap", "-p"], {
|
|
74
164
|
timeout: SCREENCAP_TIMEOUT_MS,
|
|
75
165
|
maxBuffer: SCREENCAP_MAX_BUFFER,
|
|
76
166
|
encoding: "buffer",
|
|
@@ -90,7 +180,7 @@ export async function requireOneDevice() {
|
|
|
90
180
|
}
|
|
91
181
|
catch (err) {
|
|
92
182
|
const msg = err instanceof Error ? err.message : String(err);
|
|
93
|
-
throw new AdbError(`Could not run adb (looked for "${
|
|
183
|
+
throw new AdbError(`Could not run adb (looked for "${findAdb() ?? "adb"}"). Run \`ish check android\` to check your setup. ${msg}`);
|
|
94
184
|
}
|
|
95
185
|
// Output: "List of devices attached\n<serial>\tdevice\n..."
|
|
96
186
|
const online = out
|
|
@@ -99,7 +189,7 @@ export async function requireOneDevice() {
|
|
|
99
189
|
.map((l) => l.trim())
|
|
100
190
|
.filter((l) => l && l.endsWith("\tdevice"));
|
|
101
191
|
if (online.length === 0) {
|
|
102
|
-
throw new AdbError("No Android device/emulator online.
|
|
192
|
+
throw new AdbError("No Android device/emulator online. Run `ish check android` to check your setup and how to boot one.");
|
|
103
193
|
}
|
|
104
194
|
if (online.length > 1) {
|
|
105
195
|
throw new AdbError(`Expected exactly one Android device, found ${online.length}. Stop the extras (the sim drives a single device).`);
|
|
@@ -153,6 +243,20 @@ export async function inputLongPress(x, y, durationMs = 600) {
|
|
|
153
243
|
export async function pressKeyEvent(keyevent) {
|
|
154
244
|
await adbShell(["input", "keyevent", keyevent]);
|
|
155
245
|
}
|
|
246
|
+
/**
|
|
247
|
+
* Open a system panel via `adb shell cmd statusbar expand-settings|expand-notifications`
|
|
248
|
+
* — a deterministic OS-driven open, chosen over a top-edge `input swipe` because the
|
|
249
|
+
* generic swipe is center-anchored and a synthetic top-edge swipe never registers the
|
|
250
|
+
* system pull-down. `expand-notifications` opens the notification shade;
|
|
251
|
+
* `expand-settings` pulls all the way to quick settings.
|
|
252
|
+
*/
|
|
253
|
+
export async function statusbarExpand(panel) {
|
|
254
|
+
await adbShell(["cmd", "statusbar", panel]);
|
|
255
|
+
}
|
|
256
|
+
/** Collapse the open system panel (`cmd statusbar collapse`). */
|
|
257
|
+
export async function statusbarCollapse() {
|
|
258
|
+
await adbShell(["cmd", "statusbar", "collapse"]);
|
|
259
|
+
}
|
|
156
260
|
/**
|
|
157
261
|
* Force a device orientation. We first disable auto-rotation
|
|
158
262
|
* (`accelerometer_rotation 0`) — otherwise the sensor immediately overrides
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* - Vision path: px = round(x / 1000 * screencapWidth); same for y.
|
|
20
20
|
*/
|
|
21
21
|
import type { LocalStepAction, ContextValue } from "./types.js";
|
|
22
|
-
import type { SimulationDevice, DeviceObservation, DeviceActionResult } from "./device.js";
|
|
22
|
+
import type { SimulationDevice, DeviceObservation, DeviceActionResult, AppBuild } from "./device.js";
|
|
23
23
|
export interface AndroidDeviceOptions {
|
|
24
24
|
/** App package name to force-stop/relaunch between participants. May be derived from --app. */
|
|
25
25
|
appPackage?: string;
|
|
@@ -47,6 +47,12 @@ export declare class AndroidDevice implements SimulationDevice {
|
|
|
47
47
|
private adbKeyboardActive;
|
|
48
48
|
constructor(opts: AndroidDeviceOptions);
|
|
49
49
|
launchOrReset(target: string): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* The installed app's version/build, read off the device after
|
|
52
|
+
* launchOrReset has resolved the package. Best-effort — null until the
|
|
53
|
+
* package is known, or if dumpsys can't report it.
|
|
54
|
+
*/
|
|
55
|
+
appBuild(): Promise<AppBuild | null>;
|
|
50
56
|
/**
|
|
51
57
|
* Resolve which package to drive, returning a non-null package name or
|
|
52
58
|
* throwing. For a local .apk we read the package straight from its binary
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* - Vision path: px = round(x / 1000 * screencapWidth); same for y.
|
|
20
20
|
*/
|
|
21
21
|
import { resolveTextValue } from "./actions.js";
|
|
22
|
-
import { requireOneDevice, screencapPng, pngDimensions, dumpUiautomatorXml, inputTap, inputSwipe, inputDrag, inputLongPress, setUserRotation, forceStop, launchApp, installApk, isPackageInstalled, listPackages, isAdbKeyboardInstalled, enableAdbKeyboard, setIme, resetIme, currentIme, adbKeyboardType, adbKeyboardClear, pressKeyEvent, ADB_KEYBOARD_PKG, } from "./adb.js";
|
|
22
|
+
import { requireOneDevice, screencapPng, pngDimensions, dumpUiautomatorXml, inputTap, inputSwipe, inputDrag, inputLongPress, setUserRotation, forceStop, launchApp, installApk, isPackageInstalled, listPackages, isAdbKeyboardInstalled, enableAdbKeyboard, setIme, resetIme, currentIme, adbKeyboardType, adbKeyboardClear, pressKeyEvent, statusbarExpand, appBuildFromDevice, ADB_KEYBOARD_PKG, } from "./adb.js";
|
|
23
23
|
import { isLocalPath } from "../upload.js";
|
|
24
24
|
import { deNormalizePoint, deNormalizeDrag } from "./coordinates.js";
|
|
25
25
|
import { parseUiautomatorXml, serializeNativeTree, boundsCenter } from "./native-a11y.js";
|
|
@@ -74,6 +74,21 @@ export class AndroidDevice {
|
|
|
74
74
|
// Prime screencap dimensions for the first de-normalization.
|
|
75
75
|
await this.refreshDimensions();
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* The installed app's version/build, read off the device after
|
|
79
|
+
* launchOrReset has resolved the package. Best-effort — null until the
|
|
80
|
+
* package is known, or if dumpsys can't report it.
|
|
81
|
+
*/
|
|
82
|
+
async appBuild() {
|
|
83
|
+
if (!this.appPackage)
|
|
84
|
+
return null;
|
|
85
|
+
const meta = await appBuildFromDevice(this.appPackage);
|
|
86
|
+
return {
|
|
87
|
+
package: this.appPackage,
|
|
88
|
+
version: meta?.version ?? null,
|
|
89
|
+
build: meta?.build ?? null,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
77
92
|
/**
|
|
78
93
|
* Resolve which package to drive, returning a non-null package name or
|
|
79
94
|
* throwing. For a local .apk we read the package straight from its binary
|
|
@@ -276,6 +291,11 @@ export class AndroidDevice {
|
|
|
276
291
|
await pressKeyEvent("KEYCODE_BACK");
|
|
277
292
|
break;
|
|
278
293
|
}
|
|
294
|
+
case "open_system_panel": {
|
|
295
|
+
// Element-less, like navigate_back: no px, just an OS-driven panel open.
|
|
296
|
+
await statusbarExpand(action.panel === "quick_settings" ? "expand-settings" : "expand-notifications");
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
279
299
|
case "drag": {
|
|
280
300
|
// A drag GRABS an element and RELEASES it elsewhere ("click the
|
|
281
301
|
// element, move, let go") — distinct from a swipe (element-less
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The backend's vision locator returns NORMALIZED 0-1000 coordinates. A native
|
|
5
5
|
* device de-normalizes them against a concrete dimension (screencap pixels for
|
|
6
|
-
* Android;
|
|
6
|
+
* Android; WDA POINTS for the iOS tap, screenshot PIXELS for the iOS record),
|
|
7
7
|
* and the loop later re-normalizes the recorded coordinate back to 0-1000.
|
|
8
8
|
*
|
|
9
9
|
* Pure and side-effect-free so the round-trip can be unit-tested without a
|
|
@@ -32,7 +32,7 @@ export declare function deNormalizePoint(c: {
|
|
|
32
32
|
* De-normalize a drag's start AND end points (each normalized 0-1000) against
|
|
33
33
|
* one device dimension into the {start,end} pixel/point pair the drivers feed to
|
|
34
34
|
* a slow swipe. The drag path is a from→to gesture, so BOTH ends de-normalize
|
|
35
|
-
* against the SAME basis (Android: screencap pixels; iOS:
|
|
35
|
+
* against the SAME basis (Android: screencap pixels; iOS: WDA POINTS). Pure so
|
|
36
36
|
* the two-ended de-normalization is unit-testable without a device.
|
|
37
37
|
*/
|
|
38
38
|
export declare function deNormalizeDrag(drag: {
|
|
@@ -52,9 +52,9 @@ export declare function deNormalizeDrag(drag: {
|
|
|
52
52
|
};
|
|
53
53
|
/**
|
|
54
54
|
* Scale an iOS POINT coordinate into the PIXEL space, used by the element path:
|
|
55
|
-
*
|
|
55
|
+
* WebDriverAgent reports a11y frames (and so the tappable bounds-center) in POINTS, but
|
|
56
56
|
* the loop records — and re-normalizes against `dimensions()` — in PIXELS. So
|
|
57
|
-
* the element path taps the point-center directly (
|
|
57
|
+
* the element path taps the point-center directly (WDA consumes points) yet must
|
|
58
58
|
* RECORD a pixel-center; this converts the one to the other per-axis by the
|
|
59
59
|
* point→pixel ratio (the @Nx scale). Pure so the conversion is unit-testable
|
|
60
60
|
* without a simulator. Android needs no analog: its screencap and tap share one
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The backend's vision locator returns NORMALIZED 0-1000 coordinates. A native
|
|
5
5
|
* device de-normalizes them against a concrete dimension (screencap pixels for
|
|
6
|
-
* Android;
|
|
6
|
+
* Android; WDA POINTS for the iOS tap, screenshot PIXELS for the iOS record),
|
|
7
7
|
* and the loop later re-normalizes the recorded coordinate back to 0-1000.
|
|
8
8
|
*
|
|
9
9
|
* Pure and side-effect-free so the round-trip can be unit-tested without a
|
|
@@ -32,7 +32,7 @@ export function deNormalizePoint(c, width, height) {
|
|
|
32
32
|
* De-normalize a drag's start AND end points (each normalized 0-1000) against
|
|
33
33
|
* one device dimension into the {start,end} pixel/point pair the drivers feed to
|
|
34
34
|
* a slow swipe. The drag path is a from→to gesture, so BOTH ends de-normalize
|
|
35
|
-
* against the SAME basis (Android: screencap pixels; iOS:
|
|
35
|
+
* against the SAME basis (Android: screencap pixels; iOS: WDA POINTS). Pure so
|
|
36
36
|
* the two-ended de-normalization is unit-testable without a device.
|
|
37
37
|
*/
|
|
38
38
|
export function deNormalizeDrag(drag, width, height) {
|
|
@@ -43,9 +43,9 @@ export function deNormalizeDrag(drag, width, height) {
|
|
|
43
43
|
}
|
|
44
44
|
/**
|
|
45
45
|
* Scale an iOS POINT coordinate into the PIXEL space, used by the element path:
|
|
46
|
-
*
|
|
46
|
+
* WebDriverAgent reports a11y frames (and so the tappable bounds-center) in POINTS, but
|
|
47
47
|
* the loop records — and re-normalizes against `dimensions()` — in PIXELS. So
|
|
48
|
-
* the element path taps the point-center directly (
|
|
48
|
+
* the element path taps the point-center directly (WDA consumes points) yet must
|
|
49
49
|
* RECORD a pixel-center; this converts the one to the other per-axis by the
|
|
50
50
|
* point→pixel ratio (the @Nx scale). Pure so the conversion is unit-testable
|
|
51
51
|
* without a simulator. Android needs no analog: its screencap and tap share one
|
|
@@ -18,7 +18,7 @@ import type { BrowserSession } from "./browser.js";
|
|
|
18
18
|
* One observation of the target's current state.
|
|
19
19
|
*
|
|
20
20
|
* `accessibilityTree` is populated by the browser (CDP) and by native targets
|
|
21
|
-
* (uiautomator /
|
|
21
|
+
* (uiautomator / WDA /source), serialized to the same `[id] role "name"`
|
|
22
22
|
* format the backend DOMLocator reasons over; it's "" only when a native dump
|
|
23
23
|
* fails or yields a sparse tree, which makes the backend take its vision branch.
|
|
24
24
|
* `url` is browser-only ("" for native). `tabs` is browser-only and empty for
|
|
@@ -56,6 +56,19 @@ export interface DeviceActionResult {
|
|
|
56
56
|
} | null;
|
|
57
57
|
openedNewTab: boolean;
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* The version/build of the installed native app being driven, read off the
|
|
61
|
+
* device after `launchOrReset`. Lets the web app show which build an iteration
|
|
62
|
+
* last ran against. `package` is the resolved bundle id (iOS) / package name
|
|
63
|
+
* (Android); `version` is the marketing version (CFBundleShortVersionString /
|
|
64
|
+
* versionName) and `build` the build number (CFBundleVersion / versionCode),
|
|
65
|
+
* either of which may be null when the device doesn't report it.
|
|
66
|
+
*/
|
|
67
|
+
export interface AppBuild {
|
|
68
|
+
package: string;
|
|
69
|
+
version: string | null;
|
|
70
|
+
build: string | null;
|
|
71
|
+
}
|
|
59
72
|
/**
|
|
60
73
|
* A drivable simulation target. Implementations own their own lifecycle and
|
|
61
74
|
* (for the browser) tab bookkeeping.
|
|
@@ -90,6 +103,12 @@ export interface SimulationDevice {
|
|
|
90
103
|
executeAction(action: LocalStepAction): Promise<DeviceActionResult>;
|
|
91
104
|
/** Current location string for recording (URL for browser; "" for native). */
|
|
92
105
|
currentUrl(): string;
|
|
106
|
+
/**
|
|
107
|
+
* Native only: the version/build of the installed app being driven, read
|
|
108
|
+
* off the device after `launchOrReset`. Browser omits it. Best-effort — a
|
|
109
|
+
* failed read resolves to null and never disturbs the run.
|
|
110
|
+
*/
|
|
111
|
+
appBuild?(): Promise<AppBuild | null>;
|
|
93
112
|
/** Tear down. For shared-browser tabs this closes just the tab. */
|
|
94
113
|
close(): Promise<void>;
|
|
95
114
|
}
|
|
@@ -130,7 +149,7 @@ export declare class BrowserDevice implements SimulationDevice {
|
|
|
130
149
|
/**
|
|
131
150
|
* Build the device for a platform. `web`/`browser`/`""` → Playwright
|
|
132
151
|
* `BrowserDevice`; `android` → `AndroidDevice` (adb); `ios` → `IOSDevice`
|
|
133
|
-
* (simctl +
|
|
152
|
+
* (simctl + WebDriverAgent). The native cases are dynamically imported so the browser path
|
|
134
153
|
* never pulls in the adb/simctl modules.
|
|
135
154
|
*/
|
|
136
155
|
export declare function createDevice(platform: string, opts: {
|
|
@@ -116,7 +116,7 @@ export class BrowserDevice {
|
|
|
116
116
|
/**
|
|
117
117
|
* Build the device for a platform. `web`/`browser`/`""` → Playwright
|
|
118
118
|
* `BrowserDevice`; `android` → `AndroidDevice` (adb); `ios` → `IOSDevice`
|
|
119
|
-
* (simctl +
|
|
119
|
+
* (simctl + WebDriverAgent). The native cases are dynamically imported so the browser path
|
|
120
120
|
* never pulls in the adb/simctl modules.
|
|
121
121
|
*/
|
|
122
122
|
export async function createDevice(platform, opts) {
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* IOSDevice — drives a local iOS simulator via `xcrun simctl` +
|
|
2
|
+
* IOSDevice — drives a local iOS simulator via `xcrun simctl` (lifecycle +
|
|
3
|
+
* screenshot) and WebDriverAgent/XCUITest (UI + a11y; see xcuitest.ts),
|
|
3
4
|
* implementing the SimulationDevice surface the loop expects. Mirrors
|
|
4
5
|
* AndroidDevice; the one substantive difference is the coordinate space.
|
|
5
6
|
*
|
|
6
7
|
* Two resolution paths, mirroring the browser:
|
|
7
|
-
* - ELEMENT (preferred): observe() reads
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* - ELEMENT (preferred): observe() reads WDA's `/source` a11y tree, serializes
|
|
9
|
+
* it to the `[id] role "label"` string the backend DOMLocator reasons over,
|
|
10
|
+
* and keeps a local `shortId → bounds` map (bounds in POINTS). The backend
|
|
11
|
+
* returns a `node_id`; executeAction() looks the bounds up and taps the
|
|
12
|
+
* element's CENTER.
|
|
12
13
|
* - VISION (fallback): when the tree is empty/sparse, observe() returns an
|
|
13
14
|
* empty tree so the backend takes its vision branch and returns NORMALIZED
|
|
14
15
|
* 0-1000 coordinates. Also taken per-action whenever node_id is absent.
|
|
@@ -16,7 +17,7 @@
|
|
|
16
17
|
* COORDINATE SPACE — two spaces, the key difference from Android (where
|
|
17
18
|
* screencap and tap share one pixel space):
|
|
18
19
|
* `simctl io booted screenshot` is in PIXELS (e.g. 1179x2556 @3x), but
|
|
19
|
-
*
|
|
20
|
+
* WDA taps/swipes AND the `/source` a11y frames are POINTS (393x852).
|
|
20
21
|
* The invariant in BOTH paths: TAP in points, RECORD in pixels, because the
|
|
21
22
|
* loop re-normalizes the recorded coord against dimensions() (PIXELS).
|
|
22
23
|
* - VISION: tap pt = round(n/1000 * pointSize); record px = round(n/1000 * pixelSize).
|
|
@@ -30,7 +31,7 @@
|
|
|
30
31
|
* backend never converts coords with screen_width/height.
|
|
31
32
|
*/
|
|
32
33
|
import type { LocalStepAction, ContextValue } from "./types.js";
|
|
33
|
-
import type { SimulationDevice, DeviceObservation, DeviceActionResult } from "./device.js";
|
|
34
|
+
import type { SimulationDevice, DeviceObservation, DeviceActionResult, AppBuild } from "./device.js";
|
|
34
35
|
export interface IosDeviceOptions {
|
|
35
36
|
/** Bundle id to terminate/relaunch between participants. Derived from --app when a .app is given. */
|
|
36
37
|
bundleId?: string;
|
|
@@ -46,6 +47,8 @@ export declare class IOSDevice implements SimulationDevice {
|
|
|
46
47
|
private readonly appPath;
|
|
47
48
|
/** udid of the single booted simulator we drive. */
|
|
48
49
|
private udid;
|
|
50
|
+
/** Set once the WebDriverAgent runner is up, so the startup note logs once. */
|
|
51
|
+
private wdaStarted;
|
|
49
52
|
/** POINT size — what idb ui tap/swipe consume (de-normalization basis for TAPS). */
|
|
50
53
|
private pointWidth;
|
|
51
54
|
private pointHeight;
|
|
@@ -67,6 +70,12 @@ export declare class IOSDevice implements SimulationDevice {
|
|
|
67
70
|
private lastNodeMap;
|
|
68
71
|
constructor(opts: IosDeviceOptions);
|
|
69
72
|
launchOrReset(target: string): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* The installed app's version/build, read off the simulator after
|
|
75
|
+
* launchOrReset has resolved the bundle id. Best-effort — null until the
|
|
76
|
+
* bundle id is known, or if simctl/plutil can't report it.
|
|
77
|
+
*/
|
|
78
|
+
appBuild(): Promise<AppBuild | null>;
|
|
70
79
|
/**
|
|
71
80
|
* Resolve the bundle id to drive, returning a non-null id or throwing.
|
|
72
81
|
* Installs a local `.app` first and reads its CFBundleIdentifier from
|
|
@@ -77,7 +86,7 @@ export declare class IOSDevice implements SimulationDevice {
|
|
|
77
86
|
private refreshScreen;
|
|
78
87
|
observe(): Promise<DeviceObservation>;
|
|
79
88
|
/**
|
|
80
|
-
* Read + serialize
|
|
89
|
+
* Read + serialize WDA's /source a11y tree (bounds in POINTS). Any
|
|
81
90
|
* failure (retries exhausted on a trivial tree, parse error) degrades to an
|
|
82
91
|
* empty tree so the backend falls back to vision — a missing tree must never
|
|
83
92
|
* abort the observation.
|
|
@@ -89,7 +98,7 @@ export declare class IOSDevice implements SimulationDevice {
|
|
|
89
98
|
width: number;
|
|
90
99
|
height: number;
|
|
91
100
|
};
|
|
92
|
-
/** Normalized 0-1000 → POINT space (
|
|
101
|
+
/** Normalized 0-1000 → POINT space (WDA taps/swipes take points). */
|
|
93
102
|
private toPoints;
|
|
94
103
|
/** Normalized 0-1000 → PIXEL space (the recorded/reported coord). */
|
|
95
104
|
private toPixels;
|
|
@@ -140,6 +149,20 @@ export declare class IOSDevice implements SimulationDevice {
|
|
|
140
149
|
* do drive the system gesture) when no back button is visible.
|
|
141
150
|
*/
|
|
142
151
|
private navigateBack;
|
|
152
|
+
/**
|
|
153
|
+
* Best-effort open of an iOS system panel by swiping down from the top edge.
|
|
154
|
+
* iOS has no `cmd statusbar` equivalent, so on a Face-ID layout:
|
|
155
|
+
* - notifications → Notification Center: swipe down from the top-CENTER.
|
|
156
|
+
* - quick_settings → Control Center: swipe down from the top-RIGHT corner.
|
|
157
|
+
* Coordinates are POINTS (idb consumes points; see toPoints()/the swipe()
|
|
158
|
+
* helper). This is FLAKY on the simulator — idb's synthetic touch frequently
|
|
159
|
+
* doesn't trigger the system edge gesture (the same limitation navigateBack's
|
|
160
|
+
* edge-swipe hits). We compare a before/after screenshot and log LOUDLY when
|
|
161
|
+
* the screen didn't change, rather than silently reporting success, so a
|
|
162
|
+
* no-op is visible in the run. The executeAction caller still returns
|
|
163
|
+
* success:true (the gesture was attempted); the loud log is the signal.
|
|
164
|
+
*/
|
|
165
|
+
private openSystemPanel;
|
|
143
166
|
/**
|
|
144
167
|
* The nav-bar back button: the leading (leftmost) actionable button in the
|
|
145
168
|
* top nav-bar band. iOS HIG guarantees "back" is the leading nav item in a
|