@houwert/conductor 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/capture-ui.js +14 -1
- package/dist/commands/debug.js +33 -1
- package/dist/commands/logs.js +17 -2
- package/dist/commands/network.js +67 -2
- package/dist/commands/press-key.js +7 -0
- package/dist/commands/screenshot.js +17 -7
- package/dist/daemon/web-server.js +219 -0
- package/dist/drivers/element-resolver.js +16 -7
- package/dist/drivers/web.js +26 -0
- package/dist/index.js +1 -1
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.HELP = void 0;
|
|
7
7
|
exports.captureUI = captureUI;
|
|
8
|
-
exports.HELP = ` capture-ui [--output <path>]
|
|
8
|
+
exports.HELP = ` capture-ui [--output <path.json>] Capture screenshot + hierarchy + a11y snapshot as a JSON bundle (for Argus UI panel)`;
|
|
9
9
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
10
|
const path_1 = __importDefault(require("path"));
|
|
11
11
|
const runner_js_1 = require("../runner.js");
|
|
@@ -17,6 +17,19 @@ const a11y_js_1 = require("../drivers/a11y.js");
|
|
|
17
17
|
const snapshot_store_js_1 = require("../snapshot-store.js");
|
|
18
18
|
async function captureUI(outputPath, opts = {}, sessionName = 'default') {
|
|
19
19
|
try {
|
|
20
|
+
// capture-ui always emits a JSON bundle (screenshot is embedded as base64).
|
|
21
|
+
// Reject non-JSON output paths up front so a `.png`/`.jpg` mistake doesn't
|
|
22
|
+
// silently produce an image-named file full of JSON. Use take-screenshot
|
|
23
|
+
// for an actual image.
|
|
24
|
+
if (outputPath) {
|
|
25
|
+
const ext = path_1.default.extname(outputPath).toLowerCase();
|
|
26
|
+
if (ext && ext !== '.json') {
|
|
27
|
+
(0, output_js_1.printError)(`capture-ui — \`--output\` must be a .json path (got "${ext}"). ` +
|
|
28
|
+
`capture-ui writes a JSON bundle (screenshot + hierarchy + a11y snapshot), not an image. ` +
|
|
29
|
+
`Use \`take-screenshot\` to save an image file.`, opts);
|
|
30
|
+
return 1;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
20
33
|
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
21
34
|
const capturedAt = new Date().toISOString();
|
|
22
35
|
let platform;
|
package/dist/commands/debug.js
CHANGED
|
@@ -11,7 +11,7 @@ exports.debugInspectElement = debugInspectElement;
|
|
|
11
11
|
exports.debugLogRegistry = debugLogRegistry;
|
|
12
12
|
exports.debugReload = debugReload;
|
|
13
13
|
exports.HELP = ` debug status [--port N] Show RN debugger connection info
|
|
14
|
-
debug evaluate <expr> [--port N] Run JS in the app runtime (Hermes/Fusebox)
|
|
14
|
+
debug evaluate <expr> [--port N] Run JS in the app runtime (RN: Hermes/Fusebox; web: page context)
|
|
15
15
|
debug component-tree [--port N] Print the React component tree (on-screen)
|
|
16
16
|
debug inspect-element <x,y> Print the React component at a screen point
|
|
17
17
|
debug log-registry [--source metro] Summarize recent Metro/Hermes console logs`;
|
|
@@ -22,6 +22,18 @@ const bootstrap_js_1 = require("../drivers/bootstrap.js");
|
|
|
22
22
|
const metro_js_1 = require("../drivers/log-sources/metro.js");
|
|
23
23
|
const metro_scripts_js_1 = require("../drivers/metro-scripts.js");
|
|
24
24
|
const logs_js_1 = require("./logs.js");
|
|
25
|
+
const runner_js_1 = require("../runner.js");
|
|
26
|
+
const web_js_1 = require("../drivers/web.js");
|
|
27
|
+
/** Resolve the web driver when this session targets a web device, else null (→ Metro path). */
|
|
28
|
+
async function webDriverFor(sessionName) {
|
|
29
|
+
if (!sessionName || sessionName === 'default')
|
|
30
|
+
return null;
|
|
31
|
+
const platform = await (0, bootstrap_js_1.detectPlatform)(sessionName).catch(() => undefined);
|
|
32
|
+
if (platform !== 'web')
|
|
33
|
+
return null;
|
|
34
|
+
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
35
|
+
return driver instanceof web_js_1.WebDriver ? driver : null;
|
|
36
|
+
}
|
|
25
37
|
function newRequestId() {
|
|
26
38
|
return crypto_1.default.randomBytes(6).toString('hex');
|
|
27
39
|
}
|
|
@@ -88,6 +100,26 @@ async function debugEvaluate(expr, opts, sessionName, debugOpts) {
|
|
|
88
100
|
(0, output_js_1.printError)('debug evaluate requires a JS expression', opts);
|
|
89
101
|
return 1;
|
|
90
102
|
}
|
|
103
|
+
// Web: evaluate in the page runtime via Playwright (no Metro/Hermes).
|
|
104
|
+
const web = await webDriverFor(sessionName);
|
|
105
|
+
if (web) {
|
|
106
|
+
try {
|
|
107
|
+
const { result, error } = await web.evaluate(expr);
|
|
108
|
+
if (error) {
|
|
109
|
+
(0, output_js_1.printError)(`debug evaluate — ${error}`, opts);
|
|
110
|
+
return 1;
|
|
111
|
+
}
|
|
112
|
+
if (opts.json)
|
|
113
|
+
(0, output_js_1.printData)({ result }, opts);
|
|
114
|
+
else
|
|
115
|
+
console.log(typeof result === 'string' ? result : JSON.stringify(result, null, 2));
|
|
116
|
+
return 0;
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
(0, output_js_1.printError)(`debug evaluate — ${err instanceof Error ? err.message : String(err)}`, opts);
|
|
120
|
+
return 1;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
91
123
|
const port = debugOpts.port ?? 8081;
|
|
92
124
|
const { deviceId, platformPromise } = resolveSession(sessionName);
|
|
93
125
|
try {
|
package/dist/commands/logs.js
CHANGED
|
@@ -42,6 +42,21 @@ function formatEntry(entry, opts) {
|
|
|
42
42
|
}
|
|
43
43
|
return line;
|
|
44
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Ensure both the device driver and its daemon are up before reading logs.
|
|
47
|
+
*
|
|
48
|
+
* getDriver() starts the daemon when the driver *port* is closed, but skips it
|
|
49
|
+
* when the port is already open — which happens after the daemon idle-times-out
|
|
50
|
+
* while leaving the driver alive (e.g. tvOS deliberately keeps its runner up
|
|
51
|
+
* across daemon restarts). The log collector lives inside the daemon, so a live
|
|
52
|
+
* driver port is not enough: we must guarantee the daemon socket itself is up,
|
|
53
|
+
* otherwise the log source connects to a dead socket. startDaemon() is
|
|
54
|
+
* idempotent — it returns immediately when the daemon already answers /status.
|
|
55
|
+
*/
|
|
56
|
+
async function ensureLogDaemon(sessionName) {
|
|
57
|
+
await (0, runner_js_1.getDriver)(sessionName);
|
|
58
|
+
await (0, client_js_1.startDaemon)(sessionName);
|
|
59
|
+
}
|
|
45
60
|
async function resolvePlatformAndDevice(sessionName) {
|
|
46
61
|
try {
|
|
47
62
|
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
@@ -130,7 +145,7 @@ async function logs(opts = {}, sessionName = 'default', { source, level, list, r
|
|
|
130
145
|
try {
|
|
131
146
|
// ── Snapshot mode (--recent N) ──────────────────────────────────────────
|
|
132
147
|
if (recent !== undefined) {
|
|
133
|
-
await (
|
|
148
|
+
await ensureLogDaemon(sessionName);
|
|
134
149
|
const minSeverity = level ? (types_js_1.LEVEL_SEVERITY[level] ?? 0) : 0;
|
|
135
150
|
const entries = await (0, client_js_1.fetchDaemonLogs)(sessionName, { limit: recent, level });
|
|
136
151
|
for (const entry of entries) {
|
|
@@ -145,7 +160,7 @@ async function logs(opts = {}, sessionName = 'default', { source, level, list, r
|
|
|
145
160
|
}
|
|
146
161
|
// ── Streaming modes ─────────────────────────────────────────────────────
|
|
147
162
|
// Ensure daemon is running; its log collector auto-discovers Metro.
|
|
148
|
-
await (
|
|
163
|
+
await ensureLogDaemon(sessionName);
|
|
149
164
|
const minSeverity = level ? (types_js_1.LEVEL_SEVERITY[level] ?? 0) : 0;
|
|
150
165
|
const logSource = new daemon_js_1.DaemonLogSource(sessionName);
|
|
151
166
|
await logSource.connect();
|
package/dist/commands/network.js
CHANGED
|
@@ -3,12 +3,30 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.HELP = void 0;
|
|
4
4
|
exports.networkLogs = networkLogs;
|
|
5
5
|
exports.networkRequest = networkRequest;
|
|
6
|
-
exports.HELP = ` network logs [--port N] [--limit N] Read recent HTTP traffic (
|
|
6
|
+
exports.HELP = ` network logs [--port N] [--limit N] Read recent HTTP traffic (RN: fetch/XHR shim; web: all traffic via Playwright)
|
|
7
7
|
network request <url> [--method M] [--body STR] [--header K=V] [--port N]
|
|
8
8
|
Issue an HTTP request from the app's context`;
|
|
9
9
|
const output_js_1 = require("../output.js");
|
|
10
10
|
const metro_cdp_js_1 = require("../drivers/metro-cdp.js");
|
|
11
11
|
const bootstrap_js_1 = require("../drivers/bootstrap.js");
|
|
12
|
+
const runner_js_1 = require("../runner.js");
|
|
13
|
+
const web_js_1 = require("../drivers/web.js");
|
|
14
|
+
/** Resolve the web driver when this session targets a web device, else null (→ Metro path). */
|
|
15
|
+
async function webDriverFor(sessionName) {
|
|
16
|
+
if (!sessionName || sessionName === 'default')
|
|
17
|
+
return null;
|
|
18
|
+
const platform = await (0, bootstrap_js_1.detectPlatform)(sessionName).catch(() => undefined);
|
|
19
|
+
if (platform !== 'web')
|
|
20
|
+
return null;
|
|
21
|
+
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
22
|
+
return driver instanceof web_js_1.WebDriver ? driver : null;
|
|
23
|
+
}
|
|
24
|
+
function formatNetEntry(e) {
|
|
25
|
+
const ts = e.timestamp.slice(11, 23);
|
|
26
|
+
const status = e.error ? `ERR ${e.error}` : e.status !== null ? String(e.status) : '...';
|
|
27
|
+
const dur = e.durationMs !== null ? `${e.durationMs}ms` : '-';
|
|
28
|
+
return `${ts} ${status.padEnd(6)} ${e.method.padEnd(6)} ${e.url} (${dur}, ${e.resourceType ?? e.kind ?? ''})`;
|
|
29
|
+
}
|
|
12
30
|
const INSTALL_SHIM_SCRIPT = `
|
|
13
31
|
(() => {
|
|
14
32
|
if (globalThis.__CONDUCTOR_NET__ && globalThis.__CONDUCTOR_NET__.installed) {
|
|
@@ -99,8 +117,30 @@ function resolveSession(sessionName) {
|
|
|
99
117
|
};
|
|
100
118
|
}
|
|
101
119
|
async function networkLogs(opts, sessionName, netOpts) {
|
|
102
|
-
const port = netOpts.port ?? 8081;
|
|
103
120
|
const limit = netOpts.limit ?? 50;
|
|
121
|
+
// Web: Playwright captures all traffic natively — no Metro, no injected shim.
|
|
122
|
+
const web = await webDriverFor(sessionName);
|
|
123
|
+
if (web) {
|
|
124
|
+
try {
|
|
125
|
+
const { entries } = await web.networkLogs({ limit });
|
|
126
|
+
if (opts.json) {
|
|
127
|
+
(0, output_js_1.printData)({ installed: true, count: entries.length, entries }, opts);
|
|
128
|
+
}
|
|
129
|
+
else if (entries.length === 0) {
|
|
130
|
+
console.log('No network entries captured yet. Reload the app and try again.');
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
for (const e of entries)
|
|
134
|
+
console.log(formatNetEntry(e));
|
|
135
|
+
}
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
catch (err) {
|
|
139
|
+
(0, output_js_1.printError)(`network logs — ${err instanceof Error ? err.message : String(err)}`, opts);
|
|
140
|
+
return 1;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const port = netOpts.port ?? 8081;
|
|
104
144
|
const { deviceId, platformPromise } = resolveSession(sessionName);
|
|
105
145
|
try {
|
|
106
146
|
const platform = await platformPromise;
|
|
@@ -144,6 +184,31 @@ async function networkRequest(url, opts, sessionName, reqOpts) {
|
|
|
144
184
|
headers[h.slice(0, idx)] = h.slice(idx + 1);
|
|
145
185
|
}
|
|
146
186
|
const body = reqOpts.body;
|
|
187
|
+
// Web: issue via the browser context (shares the page's cookies/session).
|
|
188
|
+
const web = await webDriverFor(sessionName);
|
|
189
|
+
if (web) {
|
|
190
|
+
try {
|
|
191
|
+
const result = await web.networkRequest(url, { method, headers, body });
|
|
192
|
+
if (opts.json)
|
|
193
|
+
(0, output_js_1.printData)(result, opts);
|
|
194
|
+
else {
|
|
195
|
+
if (result.error)
|
|
196
|
+
console.error(`error: ${result.error}`);
|
|
197
|
+
console.log(`status: ${result.status ?? 'n/a'}`);
|
|
198
|
+
for (const [k, v] of Object.entries(result.headers ?? {}))
|
|
199
|
+
console.log(`${k}: ${v}`);
|
|
200
|
+
if (result.body !== undefined) {
|
|
201
|
+
console.log('');
|
|
202
|
+
console.log(result.body);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return result.ok ? 0 : 1;
|
|
206
|
+
}
|
|
207
|
+
catch (err) {
|
|
208
|
+
(0, output_js_1.printError)(`network request — ${err instanceof Error ? err.message : String(err)}`, opts);
|
|
209
|
+
return 1;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
147
212
|
const init = JSON.stringify({
|
|
148
213
|
method,
|
|
149
214
|
headers,
|
|
@@ -149,6 +149,13 @@ async function pressKey(key, opts = {}, sessionName = 'default') {
|
|
|
149
149
|
Escape: 'Escape',
|
|
150
150
|
Home: 'Home',
|
|
151
151
|
End: 'End',
|
|
152
|
+
// Canvas webtv apps (Lightning/WPE) navigate focus via the D-pad, which they listen
|
|
153
|
+
// for as arrow keys (and Enter for select). Maps the TV remote onto web keyboard.
|
|
154
|
+
'Remote Dpad Up': 'ArrowUp',
|
|
155
|
+
'Remote Dpad Down': 'ArrowDown',
|
|
156
|
+
'Remote Dpad Left': 'ArrowLeft',
|
|
157
|
+
'Remote Dpad Right': 'ArrowRight',
|
|
158
|
+
'Remote Dpad Center': 'Enter',
|
|
152
159
|
};
|
|
153
160
|
const webKey = WEB_KEY_MAP[matched];
|
|
154
161
|
if (webKey) {
|
|
@@ -68,9 +68,14 @@ async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPa
|
|
|
68
68
|
let hierarchyW;
|
|
69
69
|
let hierarchyH;
|
|
70
70
|
if (driver instanceof ios_js_1.IOSDriver) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
71
|
+
// The root axElement returned by the driver is a synthetic wrapper
|
|
72
|
+
// around the foreground app + status bars and has frame=.zero, so
|
|
73
|
+
// reading scale from it would always collapse to 1× and crop the
|
|
74
|
+
// wrong region on retina/4K screens. Use deviceInfo, which reports
|
|
75
|
+
// both points (AX space) and pixels (screenshot space).
|
|
76
|
+
const info = await driver.deviceInfo();
|
|
77
|
+
hierarchyW = info.widthPoints;
|
|
78
|
+
hierarchyH = info.heightPoints;
|
|
74
79
|
el = await (0, wait_js_1.waitForIOSElement)((o) => driver.viewHierarchy(false, [], { cache: o?.cached }).then((x) => x.axElement), sel, undefined, undefined, (0, direct_ios_selector_js_1.makeIOSDirectResolver)(driver, sel));
|
|
75
80
|
}
|
|
76
81
|
else if (driver instanceof web_js_1.WebDriver) {
|
|
@@ -93,10 +98,15 @@ async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPa
|
|
|
93
98
|
const { width: pngW, height: pngH } = (0, png_crop_js_1.readPngDimensions)(buf);
|
|
94
99
|
const scaleX = hierarchyW > 0 ? pngW / hierarchyW : 1;
|
|
95
100
|
const scaleY = hierarchyH > 0 ? pngH / hierarchyH : 1;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const
|
|
101
|
+
// Margin is in the same logical units as the bounds (points on iOS,
|
|
102
|
+
// pixels on Android/Web — same units the `inspect` command prints),
|
|
103
|
+
// so scale it into screenshot pixels alongside the bounds.
|
|
104
|
+
const marginX = margin * scaleX;
|
|
105
|
+
const marginY = margin * scaleY;
|
|
106
|
+
const rectX = Math.round(el.bounds.x * scaleX - marginX);
|
|
107
|
+
const rectY = Math.round(el.bounds.y * scaleY - marginY);
|
|
108
|
+
const rectW = Math.round(el.bounds.width * scaleX + marginX * 2);
|
|
109
|
+
const rectH = Math.round(el.bounds.height * scaleY + marginY * 2);
|
|
100
110
|
if (rectX + rectW <= 0 || rectY + rectH <= 0 || rectX >= pngW || rectY >= pngH) {
|
|
101
111
|
throw new Error(`element ${label} bounds [${rectX},${rectY} ${rectW}x${rectH}] are outside the screenshot (${pngW}x${pngH})`);
|
|
102
112
|
}
|
|
@@ -71,6 +71,52 @@ function attachConsoleListeners(page) {
|
|
|
71
71
|
source: 'console',
|
|
72
72
|
});
|
|
73
73
|
});
|
|
74
|
+
attachNetworkListeners(page);
|
|
75
|
+
}
|
|
76
|
+
const MAX_NETWORK_BUFFER = 500;
|
|
77
|
+
const _networkBuffer = [];
|
|
78
|
+
let _netSeq = 0;
|
|
79
|
+
function pushNetworkEntry(entry) {
|
|
80
|
+
_networkBuffer.push(entry);
|
|
81
|
+
if (_networkBuffer.length > MAX_NETWORK_BUFFER) {
|
|
82
|
+
_networkBuffer.splice(0, _networkBuffer.length - MAX_NETWORK_BUFFER);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Capture all page network traffic via Playwright events — covers fetch/XHR plus
|
|
87
|
+
* document/script/image/media loads (a canvas webtv app's API calls included), without
|
|
88
|
+
* injecting a shim into the page (unlike the React Native Metro path).
|
|
89
|
+
*/
|
|
90
|
+
function attachNetworkListeners(page) {
|
|
91
|
+
const inflight = new WeakMap();
|
|
92
|
+
page.on('request', (req) => {
|
|
93
|
+
const entry = {
|
|
94
|
+
id: _netSeq++,
|
|
95
|
+
timestamp: new Date().toISOString(),
|
|
96
|
+
method: req.method(),
|
|
97
|
+
url: req.url(),
|
|
98
|
+
resourceType: req.resourceType(),
|
|
99
|
+
status: null,
|
|
100
|
+
durationMs: null,
|
|
101
|
+
error: null,
|
|
102
|
+
};
|
|
103
|
+
inflight.set(req, { entry, start: Date.now() });
|
|
104
|
+
pushNetworkEntry(entry);
|
|
105
|
+
});
|
|
106
|
+
page.on('response', (res) => {
|
|
107
|
+
const rec = inflight.get(res.request());
|
|
108
|
+
if (rec) {
|
|
109
|
+
rec.entry.status = res.status();
|
|
110
|
+
rec.entry.durationMs = Date.now() - rec.start;
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
page.on('requestfailed', (req) => {
|
|
114
|
+
const rec = inflight.get(req);
|
|
115
|
+
if (rec) {
|
|
116
|
+
rec.entry.error = req.failure()?.errorText ?? 'request failed';
|
|
117
|
+
rec.entry.durationMs = Date.now() - rec.start;
|
|
118
|
+
}
|
|
119
|
+
});
|
|
74
120
|
}
|
|
75
121
|
// ── ARIA snapshot parser ─────────────────────────────────────────────────────
|
|
76
122
|
/**
|
|
@@ -423,6 +469,113 @@ async function stampFocusFromDocumentActiveElement(page, elements) {
|
|
|
423
469
|
rectPick.best.focused = true;
|
|
424
470
|
}
|
|
425
471
|
}
|
|
472
|
+
/** IoU + center-inside score of a candidate node's bounds against a mirror rect. */
|
|
473
|
+
function overlapScore(bounds, m) {
|
|
474
|
+
const x1 = Math.max(m.x, bounds.x);
|
|
475
|
+
const y1 = Math.max(m.y, bounds.y);
|
|
476
|
+
const x2 = Math.min(m.x + m.width, bounds.x + bounds.width);
|
|
477
|
+
const y2 = Math.min(m.y + m.height, bounds.y + bounds.height);
|
|
478
|
+
const inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
|
|
479
|
+
if (inter <= 0)
|
|
480
|
+
return 0;
|
|
481
|
+
const union = m.width * m.height + bounds.width * bounds.height - inter;
|
|
482
|
+
const iou = union > 0 ? inter / union : 0;
|
|
483
|
+
const mcx = m.x + m.width / 2;
|
|
484
|
+
const mcy = m.y + m.height / 2;
|
|
485
|
+
const centerInside = mcx >= bounds.x &&
|
|
486
|
+
mcx <= bounds.x + bounds.width &&
|
|
487
|
+
mcy >= bounds.y &&
|
|
488
|
+
mcy <= bounds.y + bounds.height;
|
|
489
|
+
return centerInside ? iou + 1 : iou;
|
|
490
|
+
}
|
|
491
|
+
/** Find the existing tree node whose bounds best overlap a mirror rect (>0.5 score). */
|
|
492
|
+
function bestOverlappingNode(elements, m) {
|
|
493
|
+
let best = null;
|
|
494
|
+
let bestScore = 0.5;
|
|
495
|
+
const walk = (els) => {
|
|
496
|
+
for (const el of els) {
|
|
497
|
+
if (el.bounds) {
|
|
498
|
+
const s = overlapScore(el.bounds, m);
|
|
499
|
+
if (s > bestScore) {
|
|
500
|
+
bestScore = s;
|
|
501
|
+
best = el;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
if (el.children)
|
|
505
|
+
walk(el.children);
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
walk(elements);
|
|
509
|
+
return best;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Harvest a canvas DOM-inspector mirror into the hierarchy. Canvas webtv frameworks
|
|
513
|
+
* (Lightning/WPE/RDK) render the whole UI into one `<canvas>` and expose the scene graph as
|
|
514
|
+
* off-screen `<div>`s carrying `data-testid` (real identity) and `data-focused="true"` (focus
|
|
515
|
+
* — the canvas owns `document.activeElement`, so the normal focus path can't see it).
|
|
516
|
+
*
|
|
517
|
+
* Each mirror node is matched to an existing ARIA node by bounds overlap and used to enrich it
|
|
518
|
+
* (testId + focus); unmatched mirror nodes are appended as new nodes. Mirror rects come from
|
|
519
|
+
* `getBoundingClientRect`, i.e. the same viewport-CSS-pixel space taps use — drive TV apps at
|
|
520
|
+
* the app's native resolution (e.g. `set-viewport 1920 1080`) so lower nodes aren't off-screen.
|
|
521
|
+
*/
|
|
522
|
+
async function harvestDomMirror(page, elements) {
|
|
523
|
+
let mirror = null;
|
|
524
|
+
try {
|
|
525
|
+
mirror = (await page.evaluate(`(() => {
|
|
526
|
+
const nodes = document.querySelectorAll('[data-testid],[data-focused]');
|
|
527
|
+
if (!nodes.length) return null;
|
|
528
|
+
const out = [];
|
|
529
|
+
nodes.forEach((el) => {
|
|
530
|
+
const r = el.getBoundingClientRect();
|
|
531
|
+
out.push({
|
|
532
|
+
testId: el.getAttribute('data-testid') || '',
|
|
533
|
+
focused: el.getAttribute('data-focused') === 'true',
|
|
534
|
+
role: el.getAttribute('role') || 'generic',
|
|
535
|
+
name: (el.getAttribute('aria-label') || el.textContent || '').trim().slice(0, 200),
|
|
536
|
+
disabled: el.getAttribute('aria-disabled') === 'true',
|
|
537
|
+
x: r.x, y: r.y, width: r.width, height: r.height,
|
|
538
|
+
});
|
|
539
|
+
});
|
|
540
|
+
return out;
|
|
541
|
+
})()`));
|
|
542
|
+
}
|
|
543
|
+
catch {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
if (!mirror || mirror.length === 0)
|
|
547
|
+
return;
|
|
548
|
+
// data-focused is authoritative for canvas apps (activeElement is the <canvas>, never the
|
|
549
|
+
// focused scene node), so clear any focus already inferred from the ARIA snapshot.
|
|
550
|
+
if (mirror.some((m) => m.focused))
|
|
551
|
+
clearFocusedFlags(elements);
|
|
552
|
+
for (const m of mirror) {
|
|
553
|
+
const bounds = m.width > 0 && m.height > 0
|
|
554
|
+
? { x: m.x, y: m.y, width: m.width, height: m.height }
|
|
555
|
+
: undefined;
|
|
556
|
+
const target = bounds ? bestOverlappingNode(elements, m) : null;
|
|
557
|
+
if (target) {
|
|
558
|
+
if (m.testId)
|
|
559
|
+
target.testId = m.testId;
|
|
560
|
+
if (m.focused)
|
|
561
|
+
target.focused = true;
|
|
562
|
+
if (!target.name && m.name)
|
|
563
|
+
target.name = m.name;
|
|
564
|
+
}
|
|
565
|
+
else {
|
|
566
|
+
elements.push({
|
|
567
|
+
role: m.role,
|
|
568
|
+
name: m.name,
|
|
569
|
+
ref: '',
|
|
570
|
+
testId: m.testId || undefined,
|
|
571
|
+
bounds,
|
|
572
|
+
enabled: !m.disabled,
|
|
573
|
+
focused: m.focused,
|
|
574
|
+
children: [],
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
}
|
|
426
579
|
// ── Web server ───────────────────────────────────────────────────────────────
|
|
427
580
|
const DEFAULT_VIEWPORT = { width: 1280, height: 720 };
|
|
428
581
|
let _browser = null;
|
|
@@ -832,6 +985,9 @@ async function handleRequest(req, res, dlog) {
|
|
|
832
985
|
await resolveBoundingBoxesByRole(p, elements);
|
|
833
986
|
clearFocusedFlags(elements);
|
|
834
987
|
applyFocusFromAriaSnapshotYaml(ariaSnapshot, elements);
|
|
988
|
+
// Canvas webtv: merge the data-testid/data-focused mirror before falling back to
|
|
989
|
+
// document.activeElement (which is the <canvas>, not the focused scene node).
|
|
990
|
+
await harvestDomMirror(p, elements);
|
|
835
991
|
await stampFocusFromDocumentActiveElement(p, elements);
|
|
836
992
|
jsonResponse(res, {
|
|
837
993
|
url: p.url(),
|
|
@@ -937,6 +1093,17 @@ async function handleRequest(req, res, dlog) {
|
|
|
937
1093
|
jsonResponse(res, { entries });
|
|
938
1094
|
return;
|
|
939
1095
|
}
|
|
1096
|
+
case '/networkLogs': {
|
|
1097
|
+
const since = parsedUrl.query['since'] ?? '';
|
|
1098
|
+
const limit = Number(parsedUrl.query['limit'] ?? 0);
|
|
1099
|
+
let entries = since
|
|
1100
|
+
? _networkBuffer.filter((e) => e.timestamp > since)
|
|
1101
|
+
: _networkBuffer.slice();
|
|
1102
|
+
if (limit > 0)
|
|
1103
|
+
entries = entries.slice(-limit);
|
|
1104
|
+
jsonResponse(res, { entries });
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
940
1107
|
default: {
|
|
941
1108
|
res.writeHead(404);
|
|
942
1109
|
res.end('Not found');
|
|
@@ -1005,6 +1172,58 @@ async function handleRequest(req, res, dlog) {
|
|
|
1005
1172
|
jsonResponse(res, { ok: true });
|
|
1006
1173
|
return;
|
|
1007
1174
|
}
|
|
1175
|
+
case '/evaluate': {
|
|
1176
|
+
const expr = body['expr'];
|
|
1177
|
+
if (!expr) {
|
|
1178
|
+
jsonResponse(res, { error: 'expr is required' }, 400);
|
|
1179
|
+
return;
|
|
1180
|
+
}
|
|
1181
|
+
const p = await getPage(dlog);
|
|
1182
|
+
try {
|
|
1183
|
+
// Wrap so a bare expression (e.g. `1+1`) and a statement block both work.
|
|
1184
|
+
const result = await p.evaluate(`(() => { return (${expr}); })()`).catch(
|
|
1185
|
+
// Fall back to evaluating as-is for IIFEs / `async () => …` expressions.
|
|
1186
|
+
() => p.evaluate(expr));
|
|
1187
|
+
jsonResponse(res, { result });
|
|
1188
|
+
}
|
|
1189
|
+
catch (e) {
|
|
1190
|
+
jsonResponse(res, { error: e instanceof Error ? e.message : String(e) });
|
|
1191
|
+
}
|
|
1192
|
+
return;
|
|
1193
|
+
}
|
|
1194
|
+
case '/networkRequest': {
|
|
1195
|
+
const url = body['url'];
|
|
1196
|
+
if (!url) {
|
|
1197
|
+
jsonResponse(res, { error: 'url is required' }, 400);
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1200
|
+
const reqMethod = (body['method'] ?? 'GET').toUpperCase();
|
|
1201
|
+
const headers = body['headers'] ?? {};
|
|
1202
|
+
const data = body['body'];
|
|
1203
|
+
if (!_context) {
|
|
1204
|
+
jsonResponse(res, { error: 'no browser context' }, 500);
|
|
1205
|
+
return;
|
|
1206
|
+
}
|
|
1207
|
+
try {
|
|
1208
|
+
const apiRes = await _context.request.fetch(url, {
|
|
1209
|
+
method: reqMethod,
|
|
1210
|
+
headers,
|
|
1211
|
+
...(data !== undefined ? { data } : {}),
|
|
1212
|
+
});
|
|
1213
|
+
const hdrs = apiRes.headers();
|
|
1214
|
+
const text = await apiRes.text();
|
|
1215
|
+
jsonResponse(res, {
|
|
1216
|
+
ok: apiRes.ok(),
|
|
1217
|
+
status: apiRes.status(),
|
|
1218
|
+
headers: hdrs,
|
|
1219
|
+
body: text,
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
catch (e) {
|
|
1223
|
+
jsonResponse(res, { ok: false, error: e instanceof Error ? e.message : String(e) });
|
|
1224
|
+
}
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1008
1227
|
case '/navigate':
|
|
1009
1228
|
case '/launchApp': {
|
|
1010
1229
|
const targetUrl = (body['url'] ?? body['bundleId'] ?? body['appId']);
|
|
@@ -439,7 +439,7 @@ function collectWebElements(nodes, results) {
|
|
|
439
439
|
for (const node of nodes) {
|
|
440
440
|
const visible = node.bounds && node.bounds.width > 0 && node.bounds.height > 0;
|
|
441
441
|
if (visible) {
|
|
442
|
-
const hasContent = !!(node.name || node.ref);
|
|
442
|
+
const hasContent = !!(node.name || node.ref || node.testId);
|
|
443
443
|
const isLeaf = !node.children || node.children.length === 0;
|
|
444
444
|
if (hasContent || isLeaf) {
|
|
445
445
|
results.push(node);
|
|
@@ -450,10 +450,14 @@ function collectWebElements(nodes, results) {
|
|
|
450
450
|
}
|
|
451
451
|
}
|
|
452
452
|
}
|
|
453
|
+
/** Web id field: prefer the canvas mirror's `data-testid`, else the ARIA `ref`. */
|
|
454
|
+
function webElementId(node) {
|
|
455
|
+
return node.testId || node.ref;
|
|
456
|
+
}
|
|
453
457
|
function matchesWebElement(node, sel) {
|
|
454
458
|
if (sel.query) {
|
|
455
459
|
const textOk = matchPatternAgainstAnyTextField(sel.query, [node.name]);
|
|
456
|
-
const idOk = matchPatternAgainstElementId(sel.query, node
|
|
460
|
+
const idOk = matchPatternAgainstElementId(sel.query, webElementId(node));
|
|
457
461
|
if (!textOk && !idOk)
|
|
458
462
|
return false;
|
|
459
463
|
}
|
|
@@ -462,7 +466,7 @@ function matchesWebElement(node, sel) {
|
|
|
462
466
|
return false;
|
|
463
467
|
}
|
|
464
468
|
if (sel.id) {
|
|
465
|
-
if (!matchPatternAgainstElementId(sel.id, node
|
|
469
|
+
if (!matchPatternAgainstElementId(sel.id, webElementId(node)))
|
|
466
470
|
return false;
|
|
467
471
|
}
|
|
468
472
|
if (sel.enabled !== undefined && node.enabled !== sel.enabled)
|
|
@@ -532,7 +536,7 @@ function findWebElement(hierarchy, sel) {
|
|
|
532
536
|
matches.forEach((n, i) => {
|
|
533
537
|
const b = n.bounds;
|
|
534
538
|
const interactive = WEB_INTERACTIVE_ROLES.has(n.role);
|
|
535
|
-
(0, verbose_js_1.log)(` [${i}] text="${n.name}"
|
|
539
|
+
(0, verbose_js_1.log)(` [${i}] text="${n.name}" id="${webElementId(n)}" role="${n.role}" ` +
|
|
536
540
|
`bounds=[${Math.round(b.x)},${Math.round(b.y)}][${Math.round(b.x + b.width)},${Math.round(b.y + b.height)}]` +
|
|
537
541
|
`${interactive ? ' (interactive)' : ''}`);
|
|
538
542
|
});
|
|
@@ -554,7 +558,7 @@ function findWebElement(hierarchy, sel) {
|
|
|
554
558
|
return null;
|
|
555
559
|
}
|
|
556
560
|
const b = node.bounds;
|
|
557
|
-
(0, verbose_js_1.log)(`[Web] chose [${idx}] text="${node.name}"
|
|
561
|
+
(0, verbose_js_1.log)(`[Web] chose [${idx}] text="${node.name}" id="${webElementId(node)}" role="${node.role}" ` +
|
|
558
562
|
`bounds=[${Math.round(b.x)},${Math.round(b.y)}][${Math.round(b.x + b.width)},${Math.round(b.y + b.height)}] ` +
|
|
559
563
|
`→ tap (${Math.round(b.x + b.width / 2)}, ${Math.round(b.y + b.height / 2)})`);
|
|
560
564
|
return {
|
|
@@ -562,7 +566,7 @@ function findWebElement(hierarchy, sel) {
|
|
|
562
566
|
centerY: b.y + b.height / 2,
|
|
563
567
|
bounds: { x: b.x, y: b.y, width: b.width, height: b.height },
|
|
564
568
|
text: node.name || undefined,
|
|
565
|
-
id: node
|
|
569
|
+
id: webElementId(node) || undefined,
|
|
566
570
|
};
|
|
567
571
|
}
|
|
568
572
|
/**
|
|
@@ -579,6 +583,8 @@ function visitWeb(nodes, lines, depth) {
|
|
|
579
583
|
parts.push(node.role);
|
|
580
584
|
if (node.name)
|
|
581
585
|
parts.push(`"${node.name}"`);
|
|
586
|
+
if (node.testId)
|
|
587
|
+
parts.push(`id=${node.testId}`);
|
|
582
588
|
if (node.ref)
|
|
583
589
|
parts.push(`ref=${node.ref}`);
|
|
584
590
|
if (node.bounds) {
|
|
@@ -588,7 +594,10 @@ function visitWeb(nodes, lines, depth) {
|
|
|
588
594
|
if (!node.enabled)
|
|
589
595
|
parts.push('disabled');
|
|
590
596
|
// Only output nodes that have content
|
|
591
|
-
if (node.name ||
|
|
597
|
+
if (node.name ||
|
|
598
|
+
node.ref ||
|
|
599
|
+
node.testId ||
|
|
600
|
+
(node.bounds && (!node.children || node.children.length === 0))) {
|
|
592
601
|
lines.push(`${' '.repeat(depth)}${parts.join(' ')}`);
|
|
593
602
|
}
|
|
594
603
|
if (node.children) {
|
package/dist/drivers/web.js
CHANGED
|
@@ -60,6 +60,14 @@ class WebDriver {
|
|
|
60
60
|
}
|
|
61
61
|
return JSON.parse(data.toString('utf-8'));
|
|
62
62
|
}
|
|
63
|
+
/** POST that returns the parsed JSON response body (unlike `post`, which discards it). */
|
|
64
|
+
async postJson(path, body) {
|
|
65
|
+
const { status, data } = await this.request('POST', `/${path}`, body);
|
|
66
|
+
if (status < 200 || status >= 300) {
|
|
67
|
+
throw new Error(`Web driver ${path} failed (HTTP ${status}): ${data.toString('utf-8').slice(0, 200)}`);
|
|
68
|
+
}
|
|
69
|
+
return JSON.parse(data.toString('utf-8'));
|
|
70
|
+
}
|
|
63
71
|
async isAlive() {
|
|
64
72
|
try {
|
|
65
73
|
const { status } = await this.request('GET', '/status');
|
|
@@ -154,6 +162,24 @@ class WebDriver {
|
|
|
154
162
|
}
|
|
155
163
|
return data.toString('utf-8');
|
|
156
164
|
}
|
|
165
|
+
/** Recent page network traffic captured via Playwright request/response events. */
|
|
166
|
+
async networkLogs(opts = {}) {
|
|
167
|
+
const qs = new URLSearchParams();
|
|
168
|
+
if (opts.limit)
|
|
169
|
+
qs.set('limit', String(opts.limit));
|
|
170
|
+
if (opts.since)
|
|
171
|
+
qs.set('since', opts.since);
|
|
172
|
+
const s = qs.toString();
|
|
173
|
+
return this.get(`networkLogs${s ? `?${s}` : ''}`);
|
|
174
|
+
}
|
|
175
|
+
/** Issue an HTTP request from the browser context (shares cookies/session with the page). */
|
|
176
|
+
async networkRequest(url, opts = {}) {
|
|
177
|
+
return this.postJson('networkRequest', { url, ...opts });
|
|
178
|
+
}
|
|
179
|
+
/** Evaluate a JS expression in the page runtime and return its (JSON-serializable) value. */
|
|
180
|
+
async evaluate(expr) {
|
|
181
|
+
return this.postJson('evaluate', { expr });
|
|
182
|
+
}
|
|
157
183
|
async eraseAllText(count = 50) {
|
|
158
184
|
await this.post('eraseText', { count });
|
|
159
185
|
}
|
package/dist/index.js
CHANGED
|
@@ -222,7 +222,7 @@ async function main() {
|
|
|
222
222
|
'user-agent',
|
|
223
223
|
'color-scheme',
|
|
224
224
|
],
|
|
225
|
-
alias: { h: 'help', v: 'verbose', V: 'version' },
|
|
225
|
+
alias: { h: 'help', v: 'verbose', V: 'version', o: 'output' },
|
|
226
226
|
});
|
|
227
227
|
if (argv['verbose'])
|
|
228
228
|
(0, verbose_js_1.setVerbose)(true);
|