@houwert/conductor 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/commands/capture-ui.js +118 -0
- package/dist/commands/cheat-sheet.js +2 -1
- package/dist/commands/inspect.js +11 -3
- package/dist/drivers/a11y.js +416 -0
- package/dist/index.js +7 -0
- package/package.json +1 -1
- package/skills/conductor/SKILL.md +47 -1
- package/skills/skills.yaml +1 -1
package/README.md
CHANGED
|
@@ -71,7 +71,7 @@ Claude learns every available command, how to coordinate across devices, and how
|
|
|
71
71
|
|---|---|
|
|
72
72
|
| App lifecycle | `launch-app`, `stop-app`, `clear-state`, `uninstall-app`, `install-app`, `foreground-app`, `copy-app` |
|
|
73
73
|
| Interaction | `tap-on`, `input-text`, `scroll`, `scroll-until-visible`, `swipe`, `press-key`, `erase-text`, `hide-keyboard` |
|
|
74
|
-
| Inspection | `inspect`, `focused`, `take-screenshot`, `list-apps` |
|
|
74
|
+
| Inspection | `inspect`, `focused`, `take-screenshot`, `capture-ui`, `list-apps` |
|
|
75
75
|
| Assertions | `assert-visible`, `assert-not-visible` |
|
|
76
76
|
| Navigation | `open-link`, `back` |
|
|
77
77
|
| Flows | `run-flow`, `run-flow-inline`, `run-parallel` |
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.HELP = void 0;
|
|
7
|
+
exports.captureUI = captureUI;
|
|
8
|
+
exports.HELP = ` capture-ui [--output <path>] Capture screenshot + hierarchy + a11y snapshot (for Argus UI panel)`;
|
|
9
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const runner_js_1 = require("../runner.js");
|
|
12
|
+
const output_js_1 = require("../output.js");
|
|
13
|
+
const ios_js_1 = require("../drivers/ios.js");
|
|
14
|
+
const android_js_1 = require("../drivers/android.js");
|
|
15
|
+
const web_js_1 = require("../drivers/web.js");
|
|
16
|
+
const a11y_js_1 = require("../drivers/a11y.js");
|
|
17
|
+
async function captureUI(outputPath, opts = {}, sessionName = 'default') {
|
|
18
|
+
try {
|
|
19
|
+
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
20
|
+
const capturedAt = new Date().toISOString();
|
|
21
|
+
let platform;
|
|
22
|
+
let width = 0;
|
|
23
|
+
let height = 0;
|
|
24
|
+
let hierarchy;
|
|
25
|
+
let a11ySnapshot;
|
|
26
|
+
let screenshotBuf;
|
|
27
|
+
if (driver instanceof ios_js_1.IOSDriver) {
|
|
28
|
+
platform = driver.platform; // 'ios' | 'tvos'
|
|
29
|
+
const [info, vh, shot] = await Promise.all([
|
|
30
|
+
driver.deviceInfo(),
|
|
31
|
+
driver.viewHierarchy(false),
|
|
32
|
+
driver.screenshot(),
|
|
33
|
+
]);
|
|
34
|
+
width = info.widthPoints;
|
|
35
|
+
height = info.heightPoints;
|
|
36
|
+
const built = (0, a11y_js_1.buildIOSA11y)(vh.axElement);
|
|
37
|
+
hierarchy = { axElement: built.hierarchy, depth: vh.depth };
|
|
38
|
+
a11ySnapshot = built.a11ySnapshot;
|
|
39
|
+
screenshotBuf = shot;
|
|
40
|
+
}
|
|
41
|
+
else if (driver instanceof web_js_1.WebDriver) {
|
|
42
|
+
platform = 'web';
|
|
43
|
+
const [info, vh, shot] = await Promise.all([
|
|
44
|
+
driver.deviceInfo(),
|
|
45
|
+
driver.viewHierarchy(),
|
|
46
|
+
driver.screenshot(),
|
|
47
|
+
]);
|
|
48
|
+
width = info.widthPixels;
|
|
49
|
+
height = info.heightPixels;
|
|
50
|
+
const built = (0, a11y_js_1.buildWebA11y)(vh);
|
|
51
|
+
hierarchy = { ...vh, elements: built.hierarchy };
|
|
52
|
+
a11ySnapshot = built.a11ySnapshot;
|
|
53
|
+
screenshotBuf = shot;
|
|
54
|
+
}
|
|
55
|
+
else if (driver instanceof android_js_1.AndroidDriver) {
|
|
56
|
+
platform = 'android';
|
|
57
|
+
const [info, xml, shot] = await Promise.all([
|
|
58
|
+
driver.deviceInfo(),
|
|
59
|
+
driver.viewHierarchy(),
|
|
60
|
+
driver.screenshot(),
|
|
61
|
+
]);
|
|
62
|
+
width = info.widthPixels;
|
|
63
|
+
height = info.heightPixels;
|
|
64
|
+
const built = (0, a11y_js_1.buildAndroidA11y)(xml);
|
|
65
|
+
hierarchy = { xml, elements: built.hierarchy };
|
|
66
|
+
a11ySnapshot = built.a11ySnapshot;
|
|
67
|
+
screenshotBuf = shot;
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
throw new Error('Unknown driver type');
|
|
71
|
+
}
|
|
72
|
+
const bundle = {
|
|
73
|
+
version: 1,
|
|
74
|
+
capturedAt,
|
|
75
|
+
device: {
|
|
76
|
+
platform,
|
|
77
|
+
deviceId: sessionName,
|
|
78
|
+
width,
|
|
79
|
+
height,
|
|
80
|
+
},
|
|
81
|
+
screenshot: {
|
|
82
|
+
kind: 'composite',
|
|
83
|
+
encoding: 'png',
|
|
84
|
+
data: screenshotBuf.toString('base64'),
|
|
85
|
+
},
|
|
86
|
+
hierarchy,
|
|
87
|
+
a11ySnapshot,
|
|
88
|
+
capabilities: { perViewPixels: false, depthData: false },
|
|
89
|
+
};
|
|
90
|
+
const json = JSON.stringify(bundle);
|
|
91
|
+
if (outputPath) {
|
|
92
|
+
const resolved = path_1.default.resolve(outputPath);
|
|
93
|
+
await promises_1.default.writeFile(resolved, json);
|
|
94
|
+
if (opts.json) {
|
|
95
|
+
console.log(JSON.stringify({
|
|
96
|
+
status: 'ok',
|
|
97
|
+
path: resolved,
|
|
98
|
+
bytes: Buffer.byteLength(json, 'utf-8'),
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
(0, output_js_1.printSuccess)(`capture-ui saved to ${resolved}`, opts);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
// Stdout: raw JSON bundle (no pretty-printing — screenshot is huge).
|
|
107
|
+
process.stdout.write(json);
|
|
108
|
+
if (!opts.json)
|
|
109
|
+
process.stdout.write('\n');
|
|
110
|
+
}
|
|
111
|
+
return 0;
|
|
112
|
+
}
|
|
113
|
+
catch (err) {
|
|
114
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
115
|
+
(0, output_js_1.printError)(`capture-ui — failed\n${msg}`, opts);
|
|
116
|
+
return 1;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -67,7 +67,8 @@ ASSERTIONS
|
|
|
67
67
|
|
|
68
68
|
SCREENSHOTS & INSPECTION
|
|
69
69
|
take-screenshot [--output <path>] Take screenshot (default: ./screenshot-<ts>.png)
|
|
70
|
-
inspect
|
|
70
|
+
inspect [--dump] Print UI hierarchy (--dump: a11y-enriched JSON)
|
|
71
|
+
capture-ui [--output <path>] Screenshot + hierarchy + a11y snapshot bundle (Argus)
|
|
71
72
|
|
|
72
73
|
FLOW EXECUTION
|
|
73
74
|
run-flow <file> [--device <id>] Run a Maestro YAML flow file
|
package/dist/commands/inspect.js
CHANGED
|
@@ -9,6 +9,7 @@ const ios_js_1 = require("../drivers/ios.js");
|
|
|
9
9
|
const android_js_1 = require("../drivers/android.js");
|
|
10
10
|
const web_js_1 = require("../drivers/web.js");
|
|
11
11
|
const element_resolver_js_1 = require("../drivers/element-resolver.js");
|
|
12
|
+
const a11y_js_1 = require("../drivers/a11y.js");
|
|
12
13
|
async function inspect(opts = {}, sessionName = 'default', inspectOpts = {}) {
|
|
13
14
|
try {
|
|
14
15
|
const driver = await (0, runner_js_1.getDriver)(sessionName);
|
|
@@ -16,13 +17,20 @@ async function inspect(opts = {}, sessionName = 'default', inspectOpts = {}) {
|
|
|
16
17
|
let raw;
|
|
17
18
|
if (driver instanceof ios_js_1.IOSDriver) {
|
|
18
19
|
const hierarchy = await driver.viewHierarchy(false);
|
|
19
|
-
|
|
20
|
+
// Augment each node with a11y fields (traits, accessibilityOrder,
|
|
21
|
+
// isAccessibilityElement, announcement). All existing fields are preserved.
|
|
22
|
+
const built = (0, a11y_js_1.buildIOSA11y)(hierarchy.axElement);
|
|
23
|
+
raw = JSON.stringify({ axElement: built.hierarchy, depth: hierarchy.depth }, null, 2);
|
|
20
24
|
}
|
|
21
25
|
else if (driver instanceof web_js_1.WebDriver) {
|
|
22
|
-
|
|
26
|
+
const vh = await driver.viewHierarchy();
|
|
27
|
+
const built = (0, a11y_js_1.buildWebA11y)(vh);
|
|
28
|
+
raw = JSON.stringify({ ...vh, elements: built.hierarchy }, null, 2);
|
|
23
29
|
}
|
|
24
30
|
else if (driver instanceof android_js_1.AndroidDriver) {
|
|
25
|
-
|
|
31
|
+
const xml = await driver.viewHierarchy();
|
|
32
|
+
const built = (0, a11y_js_1.buildAndroidA11y)(xml);
|
|
33
|
+
raw = JSON.stringify(built.hierarchy, null, 2);
|
|
26
34
|
}
|
|
27
35
|
else {
|
|
28
36
|
throw new Error('Unknown driver type');
|
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.composeIOSAnnouncement = composeIOSAnnouncement;
|
|
4
|
+
exports.buildIOSA11y = buildIOSA11y;
|
|
5
|
+
exports.composeAndroidAnnouncement = composeAndroidAnnouncement;
|
|
6
|
+
exports.buildAndroidA11y = buildAndroidA11y;
|
|
7
|
+
exports.composeWebAnnouncement = composeWebAnnouncement;
|
|
8
|
+
exports.buildWebA11y = buildWebA11y;
|
|
9
|
+
const element_resolver_js_1 = require("./element-resolver.js");
|
|
10
|
+
// ── iOS ──────────────────────────────────────────────────────────────────────
|
|
11
|
+
/** XCUIElementType raw values → trait/role strings. Incomplete by design: only
|
|
12
|
+
* the element types that map to user-facing roles are listed; others fall back
|
|
13
|
+
* to an empty traits array and an empty role string. */
|
|
14
|
+
const IOS_TYPE_TO_TRAIT = {
|
|
15
|
+
3: 'application',
|
|
16
|
+
4: 'window',
|
|
17
|
+
8: 'image',
|
|
18
|
+
9: 'button',
|
|
19
|
+
23: 'adjustable',
|
|
20
|
+
40: 'switch',
|
|
21
|
+
48: 'staticText',
|
|
22
|
+
49: 'textField',
|
|
23
|
+
50: 'secureTextField',
|
|
24
|
+
54: 'link',
|
|
25
|
+
70: 'table',
|
|
26
|
+
73: 'picker',
|
|
27
|
+
74: 'pickerWheel',
|
|
28
|
+
75: 'cell',
|
|
29
|
+
90: 'stepper',
|
|
30
|
+
93: 'searchField',
|
|
31
|
+
};
|
|
32
|
+
function iosTraitsFor(node) {
|
|
33
|
+
const out = [];
|
|
34
|
+
const base = IOS_TYPE_TO_TRAIT[node.elementType];
|
|
35
|
+
if (base)
|
|
36
|
+
out.push(base);
|
|
37
|
+
if (node.selected)
|
|
38
|
+
out.push('selected');
|
|
39
|
+
if (!node.enabled)
|
|
40
|
+
out.push('disabled');
|
|
41
|
+
if (node.hasFocus)
|
|
42
|
+
out.push('focused');
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
function isIOSA11yElement(node) {
|
|
46
|
+
// VoiceOver considers an element "accessible" if it has a label or a known interactive type.
|
|
47
|
+
// AXElement doesn't expose isAccessibilityElement directly; this is a documented approximation.
|
|
48
|
+
const hasText = !!(node.label || node.title || node.value || node.placeholderValue);
|
|
49
|
+
const hasTrait = !!IOS_TYPE_TO_TRAIT[node.elementType];
|
|
50
|
+
const hasFrame = node.frame.Width > 0 && node.frame.Height > 0;
|
|
51
|
+
return hasFrame && (hasText || hasTrait);
|
|
52
|
+
}
|
|
53
|
+
/** iOS announcement: `[label], [traits], [value], [hint]` — commas between parts. */
|
|
54
|
+
function composeIOSAnnouncement(label, traits, value, hint) {
|
|
55
|
+
const parts = [];
|
|
56
|
+
if (label)
|
|
57
|
+
parts.push(label);
|
|
58
|
+
const announceableTraits = traits.filter((t) => t !== 'staticText' && t !== 'window' && t !== 'application');
|
|
59
|
+
if (announceableTraits.length)
|
|
60
|
+
parts.push(announceableTraits.join(', '));
|
|
61
|
+
if (value && value !== label)
|
|
62
|
+
parts.push(value);
|
|
63
|
+
if (hint)
|
|
64
|
+
parts.push(hint);
|
|
65
|
+
return parts.join(', ');
|
|
66
|
+
}
|
|
67
|
+
function buildIOSA11y(root) {
|
|
68
|
+
const snapshot = [];
|
|
69
|
+
let order = 0;
|
|
70
|
+
function walk(node, path) {
|
|
71
|
+
const traits = iosTraitsFor(node);
|
|
72
|
+
const isA11y = isIOSA11yElement(node);
|
|
73
|
+
const label = node.label || node.title || '';
|
|
74
|
+
const value = node.value || node.placeholderValue || '';
|
|
75
|
+
const hint = node.hint || '';
|
|
76
|
+
const announcement = isA11y ? composeIOSAnnouncement(label, traits, value, hint) : '';
|
|
77
|
+
let accessibilityOrder = null;
|
|
78
|
+
if (isA11y) {
|
|
79
|
+
accessibilityOrder = order++;
|
|
80
|
+
snapshot.push({
|
|
81
|
+
nodeId: path,
|
|
82
|
+
order: accessibilityOrder,
|
|
83
|
+
frame: {
|
|
84
|
+
x: node.frame.X,
|
|
85
|
+
y: node.frame.Y,
|
|
86
|
+
w: node.frame.Width,
|
|
87
|
+
h: node.frame.Height,
|
|
88
|
+
},
|
|
89
|
+
label,
|
|
90
|
+
hint,
|
|
91
|
+
role: traits[0] ?? '',
|
|
92
|
+
traits,
|
|
93
|
+
announcement,
|
|
94
|
+
value,
|
|
95
|
+
state: {
|
|
96
|
+
enabled: node.enabled,
|
|
97
|
+
selected: node.selected,
|
|
98
|
+
focused: node.hasFocus,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
const children = (node.children ?? []).map((c, i) => walk(c, path === '' ? String(i) : `${path}.${i}`));
|
|
103
|
+
const enriched = {
|
|
104
|
+
...node,
|
|
105
|
+
nodeId: path,
|
|
106
|
+
accessibilityOrder,
|
|
107
|
+
traits,
|
|
108
|
+
isAccessibilityElement: isA11y,
|
|
109
|
+
accessibilityIdentifier: node.identifier,
|
|
110
|
+
accessibilityLabel: label || undefined,
|
|
111
|
+
accessibilityHint: hint || undefined,
|
|
112
|
+
accessibilityValue: value || undefined,
|
|
113
|
+
announcement: announcement || undefined,
|
|
114
|
+
children: children.length ? children : undefined,
|
|
115
|
+
};
|
|
116
|
+
return enriched;
|
|
117
|
+
}
|
|
118
|
+
const hierarchy = walk(root, '0');
|
|
119
|
+
return { hierarchy, a11ySnapshot: snapshot };
|
|
120
|
+
}
|
|
121
|
+
// Android className → semantic role. Matches AccessibilityNodeInfoCompat defaults.
|
|
122
|
+
const ANDROID_CLASS_ROLE = [
|
|
123
|
+
[/Button$/, 'button'],
|
|
124
|
+
[/ImageButton$/, 'button'],
|
|
125
|
+
[/EditText$/, 'textField'],
|
|
126
|
+
[/CheckBox$/, 'checkbox'],
|
|
127
|
+
[/Switch$/, 'switch'],
|
|
128
|
+
[/ToggleButton$/, 'switch'],
|
|
129
|
+
[/RadioButton$/, 'radio'],
|
|
130
|
+
[/Spinner$/, 'dropdown'],
|
|
131
|
+
[/SeekBar$/, 'adjustable'],
|
|
132
|
+
[/ProgressBar$/, 'progressIndicator'],
|
|
133
|
+
[/TextView$/, 'staticText'],
|
|
134
|
+
[/ImageView$/, 'image'],
|
|
135
|
+
[/WebView$/, 'webView'],
|
|
136
|
+
];
|
|
137
|
+
function androidRoleFor(className) {
|
|
138
|
+
for (const [re, role] of ANDROID_CLASS_ROLE) {
|
|
139
|
+
if (re.test(className))
|
|
140
|
+
return role;
|
|
141
|
+
}
|
|
142
|
+
return '';
|
|
143
|
+
}
|
|
144
|
+
/** Android announcement: `[contentDescription || text], [role], [state], [hint]`. */
|
|
145
|
+
function composeAndroidAnnouncement(text, contentDescription, role, stateParts, hint) {
|
|
146
|
+
const parts = [];
|
|
147
|
+
const spoken = contentDescription || text;
|
|
148
|
+
if (spoken)
|
|
149
|
+
parts.push(spoken);
|
|
150
|
+
if (role)
|
|
151
|
+
parts.push(role);
|
|
152
|
+
if (stateParts.length)
|
|
153
|
+
parts.push(stateParts.join(', '));
|
|
154
|
+
if (hint)
|
|
155
|
+
parts.push(hint);
|
|
156
|
+
return parts.join(', ');
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Parse Android XML while preserving nesting — parseAndroidHierarchy returns a flat list,
|
|
160
|
+
* but we need tree structure for stable nodeId paths. We re-walk the XML here with a small
|
|
161
|
+
* state machine (open-tag depth) to reconstruct parent/child relationships.
|
|
162
|
+
*/
|
|
163
|
+
function parseAndroidTree(xml) {
|
|
164
|
+
const flat = (0, element_resolver_js_1.parseAndroidHierarchy)(xml);
|
|
165
|
+
// Parent index per node, computed from ordered XML scan.
|
|
166
|
+
const openStack = [];
|
|
167
|
+
const parentOf = new Array(flat.length).fill(-1);
|
|
168
|
+
// Regex that walks open/close tags in order.
|
|
169
|
+
const tokenRe = /<node\b[^>]*?(\/?)>|<\/node>/g;
|
|
170
|
+
let idx = 0;
|
|
171
|
+
let m;
|
|
172
|
+
while ((m = tokenRe.exec(xml)) !== null) {
|
|
173
|
+
const tag = m[0];
|
|
174
|
+
if (tag === '</node>') {
|
|
175
|
+
openStack.pop();
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
const selfClosing = m[1] === '/';
|
|
179
|
+
// Does this node have a valid bounds? Only those appear in `flat`.
|
|
180
|
+
const boundsMatch = /\bbounds="(\[[^"]+\])"/.exec(tag);
|
|
181
|
+
const hasValidBounds = boundsMatch &&
|
|
182
|
+
/\[(\d+),(\d+)]\[(\d+),(\d+)]/.test(boundsMatch[1]) &&
|
|
183
|
+
(() => {
|
|
184
|
+
const [, x1, y1, x2, y2] = /\[(\d+),(\d+)]\[(\d+),(\d+)]/.exec(boundsMatch[1]);
|
|
185
|
+
return +x2 - +x1 > 0 && +y2 - +y1 > 0;
|
|
186
|
+
})();
|
|
187
|
+
if (hasValidBounds) {
|
|
188
|
+
parentOf[idx] = openStack.length ? openStack[openStack.length - 1] : -1;
|
|
189
|
+
if (!selfClosing)
|
|
190
|
+
openStack.push(idx);
|
|
191
|
+
idx++;
|
|
192
|
+
}
|
|
193
|
+
else if (!selfClosing) {
|
|
194
|
+
// Non-visible container: push a sentinel so close tags still pop correctly.
|
|
195
|
+
openStack.push(-1);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const childrenOf = new Map();
|
|
199
|
+
const roots = [];
|
|
200
|
+
parentOf.forEach((p, i) => {
|
|
201
|
+
if (p === -1)
|
|
202
|
+
roots.push(i);
|
|
203
|
+
else {
|
|
204
|
+
const arr = childrenOf.get(p) ?? [];
|
|
205
|
+
arr.push(i);
|
|
206
|
+
childrenOf.set(p, arr);
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
const result = flat.map((n) => ({
|
|
210
|
+
text: n.text,
|
|
211
|
+
resourceId: n.resourceId,
|
|
212
|
+
contentDesc: n.contentDesc,
|
|
213
|
+
className: n.className,
|
|
214
|
+
hintText: n.hintText,
|
|
215
|
+
bounds: n.bounds,
|
|
216
|
+
clickable: n.clickable,
|
|
217
|
+
focusable: n.focusable,
|
|
218
|
+
enabled: n.enabled,
|
|
219
|
+
checked: n.checked,
|
|
220
|
+
checkable: n.checkable,
|
|
221
|
+
focused: n.focused,
|
|
222
|
+
selected: n.selected,
|
|
223
|
+
visibleToUser: n.visibleToUser,
|
|
224
|
+
}));
|
|
225
|
+
result.childrenOf = childrenOf;
|
|
226
|
+
result.roots = roots;
|
|
227
|
+
return result;
|
|
228
|
+
}
|
|
229
|
+
function buildAndroidA11y(xml) {
|
|
230
|
+
const tree = parseAndroidTree(xml);
|
|
231
|
+
const snapshot = [];
|
|
232
|
+
const meta = new Array(tree.length);
|
|
233
|
+
const computeMeta = (rawIdx) => {
|
|
234
|
+
const n = tree[rawIdx];
|
|
235
|
+
const role = androidRoleFor(n.className);
|
|
236
|
+
const screenReaderFocusable = n.focusable && (!!n.text || !!n.contentDesc);
|
|
237
|
+
const importantForAccessibility = screenReaderFocusable ? 'yes' : 'auto';
|
|
238
|
+
const stateParts = [];
|
|
239
|
+
if (n.checkable)
|
|
240
|
+
stateParts.push(n.checked ? 'checked' : 'not checked');
|
|
241
|
+
if (n.selected)
|
|
242
|
+
stateParts.push('selected');
|
|
243
|
+
if (!n.enabled)
|
|
244
|
+
stateParts.push('disabled');
|
|
245
|
+
const announcement = composeAndroidAnnouncement(n.text, n.contentDesc, role, stateParts, n.hintText);
|
|
246
|
+
// importantForAccessibility is inferred as 'yes' or 'auto' here (uiautomator
|
|
247
|
+
// dumps omit 'no' nodes). The 'no' case is kept as a documented future extension.
|
|
248
|
+
const wouldAnnounce = n.visibleToUser && (!!n.text || !!n.contentDesc || screenReaderFocusable);
|
|
249
|
+
return {
|
|
250
|
+
role,
|
|
251
|
+
screenReaderFocusable,
|
|
252
|
+
importantForAccessibility,
|
|
253
|
+
announcement,
|
|
254
|
+
stateParts,
|
|
255
|
+
wouldAnnounce,
|
|
256
|
+
hasAnnouncedDescendant: false,
|
|
257
|
+
};
|
|
258
|
+
};
|
|
259
|
+
// Post-order pass to fill hasAnnouncedDescendant.
|
|
260
|
+
const postOrder = (rawIdx) => {
|
|
261
|
+
meta[rawIdx] = computeMeta(rawIdx);
|
|
262
|
+
let anyChild = false;
|
|
263
|
+
for (const c of tree.childrenOf.get(rawIdx) ?? []) {
|
|
264
|
+
const childHasAnnounced = postOrder(c);
|
|
265
|
+
if (childHasAnnounced || meta[c].wouldAnnounce)
|
|
266
|
+
anyChild = true;
|
|
267
|
+
}
|
|
268
|
+
meta[rawIdx].hasAnnouncedDescendant = anyChild;
|
|
269
|
+
return anyChild;
|
|
270
|
+
};
|
|
271
|
+
for (const r of tree.roots)
|
|
272
|
+
postOrder(r);
|
|
273
|
+
// Phase 2: pre-order walk to emit snapshot entries + build enriched tree.
|
|
274
|
+
let order = 0;
|
|
275
|
+
const walk = (rawIdx, path) => {
|
|
276
|
+
const n = tree[rawIdx];
|
|
277
|
+
const m = meta[rawIdx];
|
|
278
|
+
const inOrder = m.wouldAnnounce && !m.hasAnnouncedDescendant;
|
|
279
|
+
let accessibilityOrder = null;
|
|
280
|
+
if (inOrder) {
|
|
281
|
+
accessibilityOrder = order++;
|
|
282
|
+
snapshot.push({
|
|
283
|
+
nodeId: path,
|
|
284
|
+
order: accessibilityOrder,
|
|
285
|
+
frame: {
|
|
286
|
+
x: n.bounds.x1,
|
|
287
|
+
y: n.bounds.y1,
|
|
288
|
+
w: n.bounds.x2 - n.bounds.x1,
|
|
289
|
+
h: n.bounds.y2 - n.bounds.y1,
|
|
290
|
+
},
|
|
291
|
+
label: n.contentDesc || n.text,
|
|
292
|
+
hint: n.hintText,
|
|
293
|
+
role: m.role,
|
|
294
|
+
traits: m.role ? [m.role] : [],
|
|
295
|
+
announcement: m.announcement,
|
|
296
|
+
value: '',
|
|
297
|
+
state: {
|
|
298
|
+
enabled: n.enabled,
|
|
299
|
+
selected: n.selected,
|
|
300
|
+
focused: n.focused,
|
|
301
|
+
checked: n.checkable ? n.checked : undefined,
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
const childIndices = tree.childrenOf.get(rawIdx) ?? [];
|
|
306
|
+
const kids = childIndices.map((i, ci) => walk(i, `${path}.${ci}`));
|
|
307
|
+
return {
|
|
308
|
+
nodeId: path,
|
|
309
|
+
accessibilityOrder,
|
|
310
|
+
class: n.className,
|
|
311
|
+
resourceId: n.resourceId,
|
|
312
|
+
text: n.text,
|
|
313
|
+
contentDescription: n.contentDesc,
|
|
314
|
+
hintText: n.hintText,
|
|
315
|
+
roleDescription: m.role,
|
|
316
|
+
role: m.role,
|
|
317
|
+
importantForAccessibility: m.importantForAccessibility,
|
|
318
|
+
screenReaderFocusable: m.screenReaderFocusable,
|
|
319
|
+
bounds: n.bounds,
|
|
320
|
+
state: {
|
|
321
|
+
enabled: n.enabled,
|
|
322
|
+
selected: n.selected,
|
|
323
|
+
focused: n.focused,
|
|
324
|
+
checked: n.checkable ? n.checked : undefined,
|
|
325
|
+
},
|
|
326
|
+
announcement: m.announcement,
|
|
327
|
+
children: kids.length ? kids : undefined,
|
|
328
|
+
};
|
|
329
|
+
};
|
|
330
|
+
const hierarchy = tree.roots.map((r, i) => walk(r, String(i)));
|
|
331
|
+
return { hierarchy, a11ySnapshot: snapshot };
|
|
332
|
+
}
|
|
333
|
+
const WEB_FOCUSABLE_ROLES = new Set([
|
|
334
|
+
'button',
|
|
335
|
+
'link',
|
|
336
|
+
'textbox',
|
|
337
|
+
'searchbox',
|
|
338
|
+
'checkbox',
|
|
339
|
+
'radio',
|
|
340
|
+
'switch',
|
|
341
|
+
'slider',
|
|
342
|
+
'spinbutton',
|
|
343
|
+
'combobox',
|
|
344
|
+
'menuitem',
|
|
345
|
+
'menuitemcheckbox',
|
|
346
|
+
'menuitemradio',
|
|
347
|
+
'tab',
|
|
348
|
+
'option',
|
|
349
|
+
]);
|
|
350
|
+
function isWebFocusable(node) {
|
|
351
|
+
return WEB_FOCUSABLE_ROLES.has(node.role);
|
|
352
|
+
}
|
|
353
|
+
/** Web announcement: `[accessibleName], [role], [state]` — screen readers read role after name. */
|
|
354
|
+
function composeWebAnnouncement(name, role, stateParts) {
|
|
355
|
+
const parts = [];
|
|
356
|
+
if (name)
|
|
357
|
+
parts.push(name);
|
|
358
|
+
if (role && role !== 'generic' && role !== 'none')
|
|
359
|
+
parts.push(role);
|
|
360
|
+
if (stateParts.length)
|
|
361
|
+
parts.push(stateParts.join(', '));
|
|
362
|
+
return parts.join(', ');
|
|
363
|
+
}
|
|
364
|
+
function buildWebA11y(hierarchy) {
|
|
365
|
+
const snapshot = [];
|
|
366
|
+
let order = 0;
|
|
367
|
+
function walk(nodes, basePath) {
|
|
368
|
+
return nodes.map((n, i) => {
|
|
369
|
+
const path = basePath === '' ? String(i) : `${basePath}.${i}`;
|
|
370
|
+
const focusable = isWebFocusable(n);
|
|
371
|
+
const stateParts = [];
|
|
372
|
+
if (n.checked)
|
|
373
|
+
stateParts.push('checked');
|
|
374
|
+
if (n.selected)
|
|
375
|
+
stateParts.push('selected');
|
|
376
|
+
if (!n.enabled)
|
|
377
|
+
stateParts.push('disabled');
|
|
378
|
+
const announcement = composeWebAnnouncement(n.name, n.role, stateParts);
|
|
379
|
+
const kids = n.children ? walk(n.children, path) : undefined;
|
|
380
|
+
let accessibilityOrder = null;
|
|
381
|
+
if (focusable && n.bounds && n.bounds.width > 0 && n.bounds.height > 0) {
|
|
382
|
+
accessibilityOrder = order++;
|
|
383
|
+
snapshot.push({
|
|
384
|
+
nodeId: path,
|
|
385
|
+
order: accessibilityOrder,
|
|
386
|
+
frame: { x: n.bounds.x, y: n.bounds.y, w: n.bounds.width, h: n.bounds.height },
|
|
387
|
+
label: n.name,
|
|
388
|
+
hint: '',
|
|
389
|
+
role: n.role,
|
|
390
|
+
traits: n.role ? [n.role] : [],
|
|
391
|
+
announcement,
|
|
392
|
+
value: '',
|
|
393
|
+
state: {
|
|
394
|
+
enabled: n.enabled,
|
|
395
|
+
selected: !!n.selected,
|
|
396
|
+
focused: n.focused,
|
|
397
|
+
checked: n.checked,
|
|
398
|
+
},
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
const enriched = {
|
|
402
|
+
...n,
|
|
403
|
+
nodeId: path,
|
|
404
|
+
accessibilityOrder,
|
|
405
|
+
accessibleName: n.name,
|
|
406
|
+
ariaLabel: n.name, // Playwright's `name` is the computed accessible name; alias it.
|
|
407
|
+
ariaDescription: '',
|
|
408
|
+
focusable,
|
|
409
|
+
announcement,
|
|
410
|
+
children: kids,
|
|
411
|
+
};
|
|
412
|
+
return enriched;
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
return { hierarchy: walk(hierarchy.elements, ''), a11ySnapshot: snapshot };
|
|
416
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -18,6 +18,7 @@ const scroll_js_1 = require("./commands/scroll.js");
|
|
|
18
18
|
const swipe_js_1 = require("./commands/swipe.js");
|
|
19
19
|
const assert_visible_js_1 = require("./commands/assert-visible.js");
|
|
20
20
|
const screenshot_js_1 = require("./commands/screenshot.js");
|
|
21
|
+
const capture_ui_js_1 = require("./commands/capture-ui.js");
|
|
21
22
|
const inspect_js_1 = require("./commands/inspect.js");
|
|
22
23
|
const focused_js_1 = require("./commands/focused.js");
|
|
23
24
|
const run_flow_js_1 = require("./commands/run-flow.js");
|
|
@@ -79,6 +80,7 @@ const COMMAND_HELP = {
|
|
|
79
80
|
'set-location': set_location_js_1.HELP,
|
|
80
81
|
'set-orientation': set_orientation_js_1.HELP,
|
|
81
82
|
'take-screenshot': screenshot_js_1.HELP,
|
|
83
|
+
'capture-ui': capture_ui_js_1.HELP,
|
|
82
84
|
inspect: inspect_js_1.HELP,
|
|
83
85
|
focused: focused_js_1.HELP,
|
|
84
86
|
'run-flow': run_flow_js_1.HELP,
|
|
@@ -454,6 +456,11 @@ async function main() {
|
|
|
454
456
|
exitCode = await (0, screenshot_js_1.screenshot)(outPath, opts, sessionName);
|
|
455
457
|
break;
|
|
456
458
|
}
|
|
459
|
+
case 'capture-ui': {
|
|
460
|
+
const outPath = argv['output'];
|
|
461
|
+
exitCode = await (0, capture_ui_js_1.captureUI)(outPath, opts, sessionName);
|
|
462
|
+
break;
|
|
463
|
+
}
|
|
457
464
|
case 'inspect':
|
|
458
465
|
exitCode = await (0, inspect_js_1.inspect)(opts, sessionName, { dump: argv['dump'] });
|
|
459
466
|
break;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: conductor
|
|
3
|
-
version: 0.
|
|
3
|
+
version: 0.11.0
|
|
4
4
|
description: "Token-efficient CLI for mobile UI testing (iOS simulator + Android emulator), designed for AI agents"
|
|
5
5
|
metadata.openclaw:
|
|
6
6
|
category: service
|
|
@@ -410,8 +410,54 @@ The hierarchy shows each element's type, text, and accessibility ID (resourceId
|
|
|
410
410
|
|
|
411
411
|
```bash
|
|
412
412
|
conductor inspect
|
|
413
|
+
conductor inspect --dump # full hierarchy as JSON with a11y fields
|
|
413
414
|
```
|
|
414
415
|
|
|
416
|
+
`--dump` emits a11y-enriched JSON on all platforms. Each node carries
|
|
417
|
+
`accessibilityOrder` (0-based screen-reader nav index, or `null`), `traits` / `role`,
|
|
418
|
+
`nodeId` (stable per-capture path), and platform-specific fields:
|
|
419
|
+
`isAccessibilityElement` + `accessibilityLabel/Hint/Value` on iOS,
|
|
420
|
+
`contentDescription` + `screenReaderFocusable` + `importantForAccessibility` on Android,
|
|
421
|
+
`accessibleName` + `focusable` on web. Existing fields (`label`, `frame`, `identifier`,
|
|
422
|
+
`text`, `resource-id`, `bounds`, ...) are preserved.
|
|
423
|
+
|
|
424
|
+
---
|
|
425
|
+
|
|
426
|
+
### `capture-ui`
|
|
427
|
+
|
|
428
|
+
Bundle everything Argus needs for a single UI inspection: screenshot (PNG base64) +
|
|
429
|
+
full UI hierarchy (with a11y fields) + flat accessibility-order snapshot in one JSON
|
|
430
|
+
document. Intended primarily for programmatic consumers (Argus UI panel); humans
|
|
431
|
+
should prefer `inspect` + `take-screenshot`.
|
|
432
|
+
|
|
433
|
+
```bash
|
|
434
|
+
conductor capture-ui # prints bundle as JSON to stdout
|
|
435
|
+
conductor capture-ui --output /tmp/capture.json # writes bundle to file
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
Output shape:
|
|
439
|
+
|
|
440
|
+
```json
|
|
441
|
+
{
|
|
442
|
+
"version": 1,
|
|
443
|
+
"capturedAt": "2026-04-22T12:34:56.000Z",
|
|
444
|
+
"device": { "platform": "ios", "deviceId": "...", "width": 390, "height": 844 },
|
|
445
|
+
"screenshot": { "kind": "composite", "encoding": "png", "data": "<base64>" },
|
|
446
|
+
"hierarchy": { /* platform-native hierarchy with a11y fields on each node */ },
|
|
447
|
+
"a11ySnapshot": [
|
|
448
|
+
{ "nodeId": "0.2.1", "order": 0, "frame": {"x":0,"y":0,"w":80,"h":44},
|
|
449
|
+
"label": "Sign in", "hint": "", "role": "button", "traits": ["button"],
|
|
450
|
+
"announcement": "Sign in, button", "value": "",
|
|
451
|
+
"state": {"enabled": true, "selected": false, "focused": false} }
|
|
452
|
+
],
|
|
453
|
+
"capabilities": { "perViewPixels": false, "depthData": false }
|
|
454
|
+
}
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
`nodeId` is a stable per-capture path (dot-joined child indices) so Argus can
|
|
458
|
+
correlate flat snapshot entries back to hierarchy nodes. `capabilities.perViewPixels`
|
|
459
|
+
is `false` in v1; per-view pixel rendering is a future phase.
|
|
460
|
+
|
|
415
461
|
---
|
|
416
462
|
|
|
417
463
|
### `logs`
|