flow-walker-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +299 -0
- package/CLAUDE.md +81 -0
- package/LICENSE +21 -0
- package/README.md +247 -0
- package/package.json +21 -0
- package/src/agent-bridge.ts +189 -0
- package/src/capture.ts +102 -0
- package/src/cli.ts +352 -0
- package/src/command-schema.ts +178 -0
- package/src/errors.ts +63 -0
- package/src/fingerprint.ts +82 -0
- package/src/flow-parser.ts +222 -0
- package/src/graph.ts +73 -0
- package/src/push.ts +170 -0
- package/src/reporter.ts +211 -0
- package/src/run-schema.ts +71 -0
- package/src/runner.ts +391 -0
- package/src/safety.ts +74 -0
- package/src/types.ts +82 -0
- package/src/validate.ts +115 -0
- package/src/walker.ts +656 -0
- package/src/yaml-writer.ts +194 -0
- package/tests/capture.test.ts +75 -0
- package/tests/command-schema.test.ts +133 -0
- package/tests/errors.test.ts +93 -0
- package/tests/fingerprint.test.ts +85 -0
- package/tests/flow-parser.test.ts +264 -0
- package/tests/graph.test.ts +111 -0
- package/tests/reporter.test.ts +188 -0
- package/tests/run-schema.test.ts +138 -0
- package/tests/runner.test.ts +150 -0
- package/tests/safety.test.ts +115 -0
- package/tests/validate.test.ts +193 -0
- package/tests/yaml-writer.test.ts +146 -0
- package/tsconfig.json +15 -0
package/src/walker.ts
ADDED
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
import type { WalkerConfig, SnapshotElement } from './types.ts';
|
|
2
|
+
import { computeFingerprint, deriveScreenName } from './fingerprint.ts';
|
|
3
|
+
import { filterSafe } from './safety.ts';
|
|
4
|
+
import { NavigationGraph } from './graph.ts';
|
|
5
|
+
import { generateFlows, writeFlows } from './yaml-writer.ts';
|
|
6
|
+
import { AgentBridge } from './agent-bridge.ts';
|
|
7
|
+
import { FlowWalkerError, ErrorCodes } from './errors.ts';
|
|
8
|
+
import { writeFileSync } from 'node:fs';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
|
|
11
|
+
interface WalkResult {
|
|
12
|
+
screensFound: number;
|
|
13
|
+
flowsGenerated: number;
|
|
14
|
+
elementsSkipped: number;
|
|
15
|
+
flowFiles: string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Recursive screen walker.
|
|
20
|
+
* Connects to a Flutter app, explores screens by pressing interactive elements,
|
|
21
|
+
* builds a navigation graph, and outputs YAML flow files.
|
|
22
|
+
*/
|
|
23
|
+
export async function walk(config: WalkerConfig): Promise<WalkResult> {
|
|
24
|
+
const bridge = new AgentBridge(config.agentFlutterPath);
|
|
25
|
+
const graph = new NavigationGraph();
|
|
26
|
+
let totalSkipped = 0;
|
|
27
|
+
|
|
28
|
+
// Connect (skip if using existing session)
|
|
29
|
+
if (config.skipConnect) {
|
|
30
|
+
log(config, `Using existing agent-flutter session...`);
|
|
31
|
+
// Store URI for auto-reconnect even in skip-connect mode
|
|
32
|
+
if (config.appUri) bridge.setUri(config.appUri);
|
|
33
|
+
} else {
|
|
34
|
+
log(config, `Connecting...`);
|
|
35
|
+
if (config.appUri) {
|
|
36
|
+
bridge.connect(config.appUri);
|
|
37
|
+
} else if (config.bundleId) {
|
|
38
|
+
bridge.connectBundle(config.bundleId);
|
|
39
|
+
} else {
|
|
40
|
+
throw new FlowWalkerError(ErrorCodes.INVALID_ARGS, 'Either --app-uri or --bundle-id is required', 'Run: flow-walker schema walk');
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
try {
|
|
45
|
+
// Wait for app to stabilize before capturing home screen
|
|
46
|
+
log(config, `Waiting for app to stabilize...`);
|
|
47
|
+
await sleep(3000); // Initial wait for app to fully load
|
|
48
|
+
let homeSnapshot = bridge.snapshot();
|
|
49
|
+
let prevCount = homeSnapshot.elements.length;
|
|
50
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
51
|
+
await sleep(1500);
|
|
52
|
+
const freshSnapshot = bridge.snapshot();
|
|
53
|
+
log(config, ` Snapshot: ${freshSnapshot.elements.length} elements (prev: ${prevCount})`);
|
|
54
|
+
if (freshSnapshot.elements.length === prevCount && freshSnapshot.elements.length >= 5) break;
|
|
55
|
+
prevCount = freshSnapshot.elements.length;
|
|
56
|
+
homeSnapshot = freshSnapshot;
|
|
57
|
+
}
|
|
58
|
+
const homeFingerprint = computeFingerprint(homeSnapshot.elements);
|
|
59
|
+
const homeName = deriveScreenName(homeSnapshot.elements);
|
|
60
|
+
const homeTypes = homeSnapshot.elements.map(e => e.flutterType || e.type).sort();
|
|
61
|
+
|
|
62
|
+
graph.addScreen(homeFingerprint, homeName, homeTypes, homeSnapshot.elements.length);
|
|
63
|
+
emit(config, { type: 'walk:start', totalElements: homeSnapshot.elements.length, maxDepth: config.maxDepth });
|
|
64
|
+
emit(config, { type: 'screen', id: homeFingerprint, name: homeName, elementCount: homeSnapshot.elements.length });
|
|
65
|
+
log(config, `Home screen: ${homeName} (${homeFingerprint}) — ${homeSnapshot.elements.length} elements`);
|
|
66
|
+
|
|
67
|
+
// Filter safe elements
|
|
68
|
+
const [safeElements, skipped] = filterSafe(homeSnapshot.elements, config.blocklist);
|
|
69
|
+
totalSkipped += skipped.length;
|
|
70
|
+
|
|
71
|
+
for (const s of skipped) {
|
|
72
|
+
emit(config, { type: 'skip', element: s.element.ref, reason: s.reason });
|
|
73
|
+
log(config, ` SKIP ${s.element.ref} "${s.element.text}" — ${s.reason}`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (config.dryRun) {
|
|
77
|
+
log(config, `\nDry run — listing elements without pressing:`);
|
|
78
|
+
for (const el of safeElements) {
|
|
79
|
+
log(config, ` SAFE ${el.ref} [${el.type}] "${el.text}"`);
|
|
80
|
+
}
|
|
81
|
+
for (const s of skipped) {
|
|
82
|
+
log(config, ` BLOCKED ${s.element.ref} [${s.element.type}] "${s.element.text}" — ${s.reason}`);
|
|
83
|
+
}
|
|
84
|
+
log(config, `\nSummary: ${safeElements.length} safe, ${skipped.length} blocked`);
|
|
85
|
+
return { screensFound: 1, flowsGenerated: 0, elementsSkipped: totalSkipped, flowFiles: [] };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// BFS walk: explore from home, always return to home between branches
|
|
89
|
+
try {
|
|
90
|
+
await walkBFS(
|
|
91
|
+
bridge, graph, config,
|
|
92
|
+
homeFingerprint, safeElements,
|
|
93
|
+
{ skipped: totalSkipped },
|
|
94
|
+
);
|
|
95
|
+
} catch (err) {
|
|
96
|
+
log(config, `Walk interrupted: ${err}`);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Always write results (even partial) so we capture what was discovered
|
|
100
|
+
const flows = generateFlows(graph);
|
|
101
|
+
const flowFiles = writeFlows(flows, config.outputDir);
|
|
102
|
+
|
|
103
|
+
const graphPath = join(config.outputDir, '_nav-graph.json');
|
|
104
|
+
writeFileSync(graphPath, JSON.stringify(graph.toJSON(), null, 2));
|
|
105
|
+
flowFiles.push(graphPath);
|
|
106
|
+
|
|
107
|
+
log(config, `\n=== Walk complete ===`);
|
|
108
|
+
log(config, `Screens: ${graph.screenCount()}`);
|
|
109
|
+
log(config, `Flows: ${flows.length}`);
|
|
110
|
+
log(config, `Files: ${flowFiles.join(', ')}`);
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
screensFound: graph.screenCount(),
|
|
114
|
+
flowsGenerated: flows.length,
|
|
115
|
+
elementsSkipped: totalSkipped,
|
|
116
|
+
flowFiles,
|
|
117
|
+
};
|
|
118
|
+
} finally {
|
|
119
|
+
try { bridge.disconnect(); } catch { /* ignore disconnect errors */ }
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* BFS walk from a root screen.
|
|
125
|
+
* For each element: press it, record the transition, return to root.
|
|
126
|
+
* Then for each discovered screen, navigate to it again and explore its children.
|
|
127
|
+
* This avoids the fragile back-navigation issue by always returning to a known screen.
|
|
128
|
+
*/
|
|
129
|
+
async function walkBFS(
|
|
130
|
+
bridge: AgentBridge,
|
|
131
|
+
graph: NavigationGraph,
|
|
132
|
+
config: WalkerConfig,
|
|
133
|
+
rootFingerprint: string,
|
|
134
|
+
rootElements: SnapshotElement[],
|
|
135
|
+
counters: { skipped: number },
|
|
136
|
+
): Promise<void> {
|
|
137
|
+
// Queue: [parentFingerprint, elementToPress, depth, pathFromRoot[]]
|
|
138
|
+
// pathFromRoot stores type+text+bounds for each step — text is stable across rebuilds,
|
|
139
|
+
// bounds is fallback when text is empty or ambiguous
|
|
140
|
+
type PathStep = {
|
|
141
|
+
type: string;
|
|
142
|
+
text: string;
|
|
143
|
+
boundsKey: string;
|
|
144
|
+
};
|
|
145
|
+
type QueueItem = {
|
|
146
|
+
parentFingerprint: string;
|
|
147
|
+
element: SnapshotElement;
|
|
148
|
+
depth: number;
|
|
149
|
+
pathFromRoot: PathStep[];
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const queue: QueueItem[] = [];
|
|
153
|
+
const rootElementCount = rootElements.length;
|
|
154
|
+
// Track known root fingerprints (dynamic content causes fingerprint drift)
|
|
155
|
+
const knownRootFingerprints = new Set<string>([rootFingerprint]);
|
|
156
|
+
|
|
157
|
+
// Detect root screen's bottom nav "home tab" position.
|
|
158
|
+
// Bottom nav elements are at y > 780 with small height.
|
|
159
|
+
// The leftmost one is typically the "home" tab.
|
|
160
|
+
const rootBottomNav = rootElements
|
|
161
|
+
.filter(e => e.bounds && e.bounds.y > 780 && e.bounds.height < 100)
|
|
162
|
+
.sort((a, b) => (a.bounds?.x ?? 0) - (b.bounds?.x ?? 0));
|
|
163
|
+
const homeTabBounds = rootBottomNav.length >= 3 ? rootBottomNav[0].bounds : undefined;
|
|
164
|
+
|
|
165
|
+
// Seed queue with root screen elements
|
|
166
|
+
for (const el of rootElements) {
|
|
167
|
+
queue.push({
|
|
168
|
+
parentFingerprint: rootFingerprint,
|
|
169
|
+
element: el,
|
|
170
|
+
depth: 0,
|
|
171
|
+
pathFromRoot: [],
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Track pressed elements per screen to avoid duplicates
|
|
176
|
+
const pressedPerScreen = new Map<string, Set<string>>();
|
|
177
|
+
pressedPerScreen.set(rootFingerprint, new Set());
|
|
178
|
+
|
|
179
|
+
// Circuit breaker: abort if returnToRoot fails too many times in a row
|
|
180
|
+
let consecutiveRootFailures = 0;
|
|
181
|
+
const MAX_ROOT_FAILURES = 5;
|
|
182
|
+
|
|
183
|
+
// Per-parent failure tracking: skip remaining children after N failures
|
|
184
|
+
const parentFailCounts = new Map<string, number>();
|
|
185
|
+
const MAX_PARENT_FAILURES = 3;
|
|
186
|
+
// Screen mutation aliases: map original fingerprint → mutated fingerprints
|
|
187
|
+
// (e.g., after a switch toggle changes the element count)
|
|
188
|
+
const screenAliases = new Map<string, Set<string>>();
|
|
189
|
+
|
|
190
|
+
async function safeReturnToRoot(): Promise<boolean> {
|
|
191
|
+
const ok = await returnToRoot(bridge, config, rootFingerprint, rootElementCount, homeTabBounds, knownRootFingerprints);
|
|
192
|
+
if (ok) {
|
|
193
|
+
consecutiveRootFailures = 0;
|
|
194
|
+
// Record any new root fingerprint variant
|
|
195
|
+
try {
|
|
196
|
+
const snap = bridge.snapshot();
|
|
197
|
+
const fp = computeFingerprint(snap.elements);
|
|
198
|
+
knownRootFingerprints.add(fp);
|
|
199
|
+
} catch { /* ignore */ }
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// returnToRoot failed — try full app recovery with clean navigation stack.
|
|
204
|
+
// This handles the case where back() exits the app (e.g., bottom nav tabs).
|
|
205
|
+
log(config, ` returnToRoot failed — attempting clean app recovery`);
|
|
206
|
+
try {
|
|
207
|
+
bridge.bringToForeground();
|
|
208
|
+
await sleep(5000);
|
|
209
|
+
try { bridge.reconnect(); } catch { /* ignore */ }
|
|
210
|
+
await sleep(2000);
|
|
211
|
+
const snap = bridge.snapshot();
|
|
212
|
+
const fp = computeFingerprint(snap.elements);
|
|
213
|
+
if (fp === rootFingerprint || knownRootFingerprints.has(fp)) {
|
|
214
|
+
consecutiveRootFailures = 0;
|
|
215
|
+
knownRootFingerprints.add(fp);
|
|
216
|
+
log(config, ` App recovery succeeded (${snap.elements.length} elements)`);
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
log(config, ` App recovery: wrong screen (${snap.elements.length} elements, fp=${fp.slice(0,8)})`);
|
|
220
|
+
} catch {
|
|
221
|
+
log(config, ` App recovery failed`);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
consecutiveRootFailures++;
|
|
225
|
+
log(config, ` returnToRoot failed (${consecutiveRootFailures}/${MAX_ROOT_FAILURES})`);
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
while (queue.length > 0) {
|
|
230
|
+
if (consecutiveRootFailures >= MAX_ROOT_FAILURES) {
|
|
231
|
+
log(config, `\nAborting: returnToRoot failed ${MAX_ROOT_FAILURES} times in a row`);
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const item = queue.shift()!;
|
|
236
|
+
const { parentFingerprint, element, depth, pathFromRoot } = item;
|
|
237
|
+
|
|
238
|
+
if (depth >= config.maxDepth) continue;
|
|
239
|
+
|
|
240
|
+
// Skip if this parent has been unreachable too many times
|
|
241
|
+
if ((parentFailCounts.get(parentFingerprint) ?? 0) >= MAX_PARENT_FAILURES) continue;
|
|
242
|
+
|
|
243
|
+
const boundsKey = element.bounds
|
|
244
|
+
? `${Math.round(element.bounds.x)},${Math.round(element.bounds.y)},${element.type}`
|
|
245
|
+
: `${element.ref},${element.type}`;
|
|
246
|
+
|
|
247
|
+
const pressed = pressedPerScreen.get(parentFingerprint)!;
|
|
248
|
+
if (pressed.has(boundsKey)) continue;
|
|
249
|
+
pressed.add(boundsKey);
|
|
250
|
+
|
|
251
|
+
const indent = ' '.repeat(depth + 1);
|
|
252
|
+
|
|
253
|
+
// Navigate to parent screen from root if not already there
|
|
254
|
+
if (pathFromRoot.length > 0) {
|
|
255
|
+
const navigated = await navigateToScreen(bridge, config, rootFingerprint, pathFromRoot);
|
|
256
|
+
if (!navigated) {
|
|
257
|
+
parentFailCounts.set(parentFingerprint, (parentFailCounts.get(parentFingerprint) ?? 0) + 1);
|
|
258
|
+
log(config, `${indent}SKIP: could not navigate to parent screen (${parentFailCounts.get(parentFingerprint)}/${MAX_PARENT_FAILURES})`);
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Verify we're on the expected screen
|
|
264
|
+
let currentSnapshot;
|
|
265
|
+
try {
|
|
266
|
+
currentSnapshot = bridge.snapshot();
|
|
267
|
+
} catch {
|
|
268
|
+
log(config, `${indent}SKIP: snapshot failed`);
|
|
269
|
+
await safeReturnToRoot();
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const currentFingerprint = computeFingerprint(currentSnapshot.elements);
|
|
274
|
+
// Successfully on target screen — reset failure counters
|
|
275
|
+
if (currentFingerprint === parentFingerprint) {
|
|
276
|
+
consecutiveRootFailures = 0;
|
|
277
|
+
parentFailCounts.delete(parentFingerprint);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// Accept known root fingerprint variants (registered by safeReturnToRoot)
|
|
281
|
+
const isKnownRoot = knownRootFingerprints.has(currentFingerprint);
|
|
282
|
+
if (isKnownRoot && currentFingerprint !== parentFingerprint) {
|
|
283
|
+
consecutiveRootFailures = 0;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// Check screen mutation aliases (e.g., switch toggle changed parent fingerprint)
|
|
287
|
+
const isAlias = screenAliases.get(parentFingerprint)?.has(currentFingerprint) ?? false;
|
|
288
|
+
|
|
289
|
+
if (currentFingerprint !== parentFingerprint && !isKnownRoot && !isAlias) {
|
|
290
|
+
parentFailCounts.set(parentFingerprint, (parentFailCounts.get(parentFingerprint) ?? 0) + 1);
|
|
291
|
+
log(config, `${indent}SKIP: on wrong screen (${currentFingerprint} != ${parentFingerprint}) (${parentFailCounts.get(parentFingerprint)}/${MAX_PARENT_FAILURES})`);
|
|
292
|
+
await safeReturnToRoot();
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Find the element in the fresh snapshot — try text+type first, then bounds
|
|
297
|
+
let freshElement: SnapshotElement | undefined;
|
|
298
|
+
if (element.text) {
|
|
299
|
+
freshElement = currentSnapshot.elements.find(
|
|
300
|
+
e => e.type === element.type && e.text === element.text,
|
|
301
|
+
);
|
|
302
|
+
// Disambiguate if multiple matches
|
|
303
|
+
if (freshElement) {
|
|
304
|
+
const sameTextEls = currentSnapshot.elements.filter(
|
|
305
|
+
e => e.type === element.type && e.text === element.text,
|
|
306
|
+
);
|
|
307
|
+
if (sameTextEls.length > 1 && element.bounds) {
|
|
308
|
+
freshElement = findByBounds(sameTextEls, {
|
|
309
|
+
type: element.type,
|
|
310
|
+
boundsKey: `${Math.round(element.bounds.x)},${Math.round(element.bounds.y)},${element.type}`,
|
|
311
|
+
}) || freshElement;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (!freshElement && element.bounds) {
|
|
316
|
+
freshElement = findByBounds(currentSnapshot.elements, {
|
|
317
|
+
type: element.type,
|
|
318
|
+
boundsKey: `${Math.round(element.bounds.x)},${Math.round(element.bounds.y)},${element.type}`,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if (!freshElement) {
|
|
323
|
+
log(config, `${indent}SKIP: element not found in fresh snapshot`);
|
|
324
|
+
await safeReturnToRoot();
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
log(config, `${indent}Press ${freshElement.ref} [${freshElement.type}] "${freshElement.text}"`);
|
|
329
|
+
|
|
330
|
+
try {
|
|
331
|
+
bridge.press(freshElement.ref);
|
|
332
|
+
} catch {
|
|
333
|
+
log(config, `${indent} FAILED to press, skipping`);
|
|
334
|
+
await safeReturnToRoot();
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
await sleep(1000);
|
|
339
|
+
|
|
340
|
+
let newSnapshot;
|
|
341
|
+
try {
|
|
342
|
+
newSnapshot = bridge.snapshot();
|
|
343
|
+
} catch {
|
|
344
|
+
log(config, `${indent} FAILED to snapshot after press`);
|
|
345
|
+
await safeReturnToRoot();
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const newFingerprint = computeFingerprint(newSnapshot.elements);
|
|
350
|
+
|
|
351
|
+
if (newFingerprint === parentFingerprint
|
|
352
|
+
|| (screenAliases.get(parentFingerprint)?.has(newFingerprint))) {
|
|
353
|
+
log(config, `${indent} Same screen — no-op`);
|
|
354
|
+
// No need to return to root if on root, or already on parent
|
|
355
|
+
if (parentFingerprint !== rootFingerprint) {
|
|
356
|
+
await safeReturnToRoot();
|
|
357
|
+
}
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Detect screen mutation: pressing a toggle/switch changes the parent screen's
|
|
362
|
+
// fingerprint slightly. If element count is close (±3) and the pressed element
|
|
363
|
+
// was a switch/toggle type, register as an alias instead of a new screen.
|
|
364
|
+
const isMutation = (
|
|
365
|
+
freshElement.type === 'switch'
|
|
366
|
+
|| freshElement.flutterType === 'Switch'
|
|
367
|
+
|| freshElement.type === 'checkbox'
|
|
368
|
+
|| freshElement.flutterType === 'Checkbox'
|
|
369
|
+
) && Math.abs(newSnapshot.elements.length - currentSnapshot.elements.length) <= 3;
|
|
370
|
+
|
|
371
|
+
if (isMutation) {
|
|
372
|
+
log(config, `${indent} Screen mutation (${freshElement.type} toggled): ${parentFingerprint.slice(0,8)} → ${newFingerprint.slice(0,8)}`);
|
|
373
|
+
if (!screenAliases.has(parentFingerprint)) screenAliases.set(parentFingerprint, new Set());
|
|
374
|
+
screenAliases.get(parentFingerprint)!.add(newFingerprint);
|
|
375
|
+
// Don't queue children — it's the same screen in a different state
|
|
376
|
+
if (parentFingerprint !== rootFingerprint) {
|
|
377
|
+
await safeReturnToRoot();
|
|
378
|
+
}
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// New screen discovered
|
|
383
|
+
const newName = deriveScreenName(newSnapshot.elements);
|
|
384
|
+
const newTypes = newSnapshot.elements.map(e => e.flutterType || e.type).sort();
|
|
385
|
+
graph.addScreen(newFingerprint, newName, newTypes, newSnapshot.elements.length);
|
|
386
|
+
graph.addEdge(parentFingerprint, newFingerprint, {
|
|
387
|
+
ref: freshElement.ref,
|
|
388
|
+
type: freshElement.type,
|
|
389
|
+
text: freshElement.text,
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
emit(config, { type: 'screen', id: newFingerprint, name: newName, elementCount: newSnapshot.elements.length });
|
|
393
|
+
emit(config, { type: 'edge', source: parentFingerprint, target: newFingerprint, element: freshElement.ref });
|
|
394
|
+
log(config, `${indent} → ${newName} (${newFingerprint}) — ${newSnapshot.elements.length} elements`);
|
|
395
|
+
|
|
396
|
+
// Queue children for exploration if not visited too much
|
|
397
|
+
if (graph.visitCount(newFingerprint) <= 1 && depth + 1 < config.maxDepth) {
|
|
398
|
+
const [safeChildren, skippedChildren] = filterSafe(newSnapshot.elements, config.blocklist);
|
|
399
|
+
counters.skipped += skippedChildren.length;
|
|
400
|
+
for (const s of skippedChildren) {
|
|
401
|
+
emit(config, { type: 'skip', element: s.element.ref, reason: s.reason });
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (!pressedPerScreen.has(newFingerprint)) {
|
|
405
|
+
pressedPerScreen.set(newFingerprint, new Set());
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
const newPath: PathStep[] = [...pathFromRoot, {
|
|
409
|
+
type: freshElement.type,
|
|
410
|
+
text: freshElement.text,
|
|
411
|
+
boundsKey,
|
|
412
|
+
}];
|
|
413
|
+
for (const child of safeChildren) {
|
|
414
|
+
queue.push({
|
|
415
|
+
parentFingerprint: newFingerprint,
|
|
416
|
+
element: child,
|
|
417
|
+
depth: depth + 1,
|
|
418
|
+
pathFromRoot: newPath,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// Return to root
|
|
424
|
+
await returnToRoot(bridge, config, rootFingerprint, rootElementCount, homeTabBounds, knownRootFingerprints);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Navigate from root to a target screen by replaying a path of element presses.
|
|
430
|
+
* Uses text+type matching first (stable across widget rebuilds), bounds as fallback.
|
|
431
|
+
*/
|
|
432
|
+
async function navigateToScreen(
|
|
433
|
+
bridge: AgentBridge,
|
|
434
|
+
config: WalkerConfig,
|
|
435
|
+
rootFingerprint: string,
|
|
436
|
+
path: { type: string; text: string; boundsKey: string }[],
|
|
437
|
+
): Promise<boolean> {
|
|
438
|
+
for (let stepIdx = 0; stepIdx < path.length; stepIdx++) {
|
|
439
|
+
const step = path[stepIdx];
|
|
440
|
+
let snapshot;
|
|
441
|
+
try {
|
|
442
|
+
snapshot = bridge.snapshot();
|
|
443
|
+
} catch {
|
|
444
|
+
return false;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Strategy 1: Match by type + exact text (most stable across rebuilds)
|
|
448
|
+
let el: SnapshotElement | undefined;
|
|
449
|
+
if (step.text) {
|
|
450
|
+
el = snapshot.elements.find(e => e.type === step.type && e.text === step.text);
|
|
451
|
+
// If multiple matches by text, disambiguate by bounds proximity
|
|
452
|
+
if (el && step.boundsKey) {
|
|
453
|
+
const sameTextEls = snapshot.elements.filter(
|
|
454
|
+
e => e.type === step.type && e.text === step.text,
|
|
455
|
+
);
|
|
456
|
+
if (sameTextEls.length > 1) {
|
|
457
|
+
el = findByBounds(sameTextEls, step) || el;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Strategy 2: Match by bounds (fallback for elements without text)
|
|
463
|
+
if (!el) {
|
|
464
|
+
el = findByBounds(snapshot.elements, step);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
if (!el) {
|
|
468
|
+
log(config, ` NAV step ${stepIdx}: element not found (${step.type} "${step.text}")`);
|
|
469
|
+
return false;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
try {
|
|
473
|
+
bridge.press(el.ref);
|
|
474
|
+
} catch {
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
await sleep(1000);
|
|
479
|
+
}
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/** Find element by bounds proximity (±5px tolerance) and type match */
|
|
484
|
+
function findByBounds(
|
|
485
|
+
elements: SnapshotElement[],
|
|
486
|
+
step: { type: string; boundsKey: string },
|
|
487
|
+
): SnapshotElement | undefined {
|
|
488
|
+
const [xStr, yStr] = step.boundsKey.split(',');
|
|
489
|
+
const targetX = parseInt(xStr, 10);
|
|
490
|
+
const targetY = parseInt(yStr, 10);
|
|
491
|
+
if (isNaN(targetX) || isNaN(targetY)) return undefined;
|
|
492
|
+
|
|
493
|
+
// Try exact match first (±3px), then wider tolerances
|
|
494
|
+
for (const tolerance of [3, 8, 15]) {
|
|
495
|
+
const match = elements.find(e =>
|
|
496
|
+
e.bounds
|
|
497
|
+
&& Math.abs(Math.round(e.bounds.x) - targetX) <= tolerance
|
|
498
|
+
&& Math.abs(Math.round(e.bounds.y) - targetY) <= tolerance
|
|
499
|
+
&& e.type === step.type,
|
|
500
|
+
);
|
|
501
|
+
if (match) return match;
|
|
502
|
+
}
|
|
503
|
+
return undefined;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Return to root screen by pressing back repeatedly.
|
|
508
|
+
* Limit attempts to avoid exiting the app entirely.
|
|
509
|
+
* Uses rootElementCount to detect "close enough" instead of a hardcoded threshold.
|
|
510
|
+
*/
|
|
511
|
+
async function returnToRoot(
|
|
512
|
+
bridge: AgentBridge,
|
|
513
|
+
config: WalkerConfig,
|
|
514
|
+
rootFingerprint: string,
|
|
515
|
+
rootElementCount: number = 0,
|
|
516
|
+
homeTabBounds?: { x: number; y: number; width: number; height: number },
|
|
517
|
+
knownRootFingerprints?: Set<string>,
|
|
518
|
+
): Promise<boolean> {
|
|
519
|
+
let lowElementStreak = 0; // Track consecutive low-element snapshots
|
|
520
|
+
let triedBottomNav = false; // Only try bottom nav once per returnToRoot call
|
|
521
|
+
// Max 5 back presses — any more and we risk exiting the app
|
|
522
|
+
for (let i = 0; i < 5; i++) {
|
|
523
|
+
let snapshot;
|
|
524
|
+
try {
|
|
525
|
+
snapshot = bridge.snapshot();
|
|
526
|
+
} catch {
|
|
527
|
+
// Snapshot failed — app may have exited. Try recovery:
|
|
528
|
+
// bringToForeground → wait → reconnect → snapshot
|
|
529
|
+
log(config, ` Snapshot failed — attempting app recovery`);
|
|
530
|
+
bridge.bringToForeground();
|
|
531
|
+
await sleep(3000);
|
|
532
|
+
try { bridge.reconnect(); } catch { /* ignore */ }
|
|
533
|
+
await sleep(2000);
|
|
534
|
+
try {
|
|
535
|
+
snapshot = bridge.snapshot();
|
|
536
|
+
} catch {
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
const fp = computeFingerprint(snapshot.elements);
|
|
542
|
+
if (fp === rootFingerprint) return true;
|
|
543
|
+
// Accept known root fingerprint variants
|
|
544
|
+
if (knownRootFingerprints?.has(fp)) {
|
|
545
|
+
log(config, ` Accepted known root variant (${snapshot.elements.length} elements)`);
|
|
546
|
+
return true;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// If element count dropped very low, might be a dialog/overlay or exited app
|
|
550
|
+
if (snapshot.elements.length < 3) {
|
|
551
|
+
lowElementStreak++;
|
|
552
|
+
log(config, ` Very few elements (${snapshot.elements.length}) — attempt ${lowElementStreak}`);
|
|
553
|
+
|
|
554
|
+
// First attempt: try back() to dismiss overlay/dialog
|
|
555
|
+
if (lowElementStreak === 1) {
|
|
556
|
+
try {
|
|
557
|
+
bridge.back();
|
|
558
|
+
await sleep(1500);
|
|
559
|
+
const check = bridge.snapshot();
|
|
560
|
+
if (check.elements.length >= 5) {
|
|
561
|
+
lowElementStreak = 0;
|
|
562
|
+
continue; // Overlay dismissed, re-check from top of loop
|
|
563
|
+
}
|
|
564
|
+
} catch { /* ignore */ }
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// Second attempt: clean relaunch with cleared navigation stack
|
|
568
|
+
if (lowElementStreak >= 2) {
|
|
569
|
+
log(config, ` Attempting clean app relaunch`);
|
|
570
|
+
try {
|
|
571
|
+
// Use --activity-clear-task to reset navigation to root
|
|
572
|
+
bridge.bringToForeground();
|
|
573
|
+
await sleep(5000); // Give app time to fully load and navigate to home
|
|
574
|
+
// Reconnect in case isolate changed
|
|
575
|
+
try { bridge.reconnect(); } catch { /* ignore */ }
|
|
576
|
+
await sleep(2000);
|
|
577
|
+
const verify = bridge.snapshot();
|
|
578
|
+
if (verify.elements.length >= 10) {
|
|
579
|
+
log(config, ` Clean relaunch recovered (${verify.elements.length} elements)`);
|
|
580
|
+
lowElementStreak = 0;
|
|
581
|
+
continue;
|
|
582
|
+
}
|
|
583
|
+
log(config, ` Clean relaunch didn't recover (${verify.elements.length} elements)`);
|
|
584
|
+
} catch {
|
|
585
|
+
log(config, ` Clean relaunch failed`);
|
|
586
|
+
}
|
|
587
|
+
return false;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
lowElementStreak = 0;
|
|
594
|
+
|
|
595
|
+
// If this screen has bottom nav and we know the home tab position,
|
|
596
|
+
// try pressing the home tab instead of back() (back() on bottom nav exits the app).
|
|
597
|
+
// Only try once per returnToRoot call — if it doesn't work, fall through to back().
|
|
598
|
+
if (!triedBottomNav && homeTabBounds && snapshot.elements.length >= 10) {
|
|
599
|
+
const bottomNavEls = snapshot.elements.filter(
|
|
600
|
+
e => e.bounds && e.bounds.y > 780 && e.bounds.height < 100,
|
|
601
|
+
);
|
|
602
|
+
if (bottomNavEls.length >= 3) {
|
|
603
|
+
// Find element closest to root's home tab x position
|
|
604
|
+
const targetX = homeTabBounds.x;
|
|
605
|
+
const closest = bottomNavEls.reduce((best, el) =>
|
|
606
|
+
Math.abs((el.bounds?.x ?? 999) - targetX) < Math.abs((best.bounds?.x ?? 999) - targetX)
|
|
607
|
+
? el : best,
|
|
608
|
+
);
|
|
609
|
+
if (closest.bounds && Math.abs(closest.bounds.x - targetX) < 20) {
|
|
610
|
+
triedBottomNav = true;
|
|
611
|
+
log(config, ` Pressing home tab (${closest.ref}) at x=${Math.round(closest.bounds.x)}`);
|
|
612
|
+
try {
|
|
613
|
+
bridge.press(closest.ref);
|
|
614
|
+
await sleep(1500);
|
|
615
|
+
continue;
|
|
616
|
+
} catch { /* fall through to back() */ }
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
try {
|
|
622
|
+
bridge.back();
|
|
623
|
+
} catch {
|
|
624
|
+
return false;
|
|
625
|
+
}
|
|
626
|
+
await sleep(1000);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// Check one final time
|
|
630
|
+
try {
|
|
631
|
+
const snapshot = bridge.snapshot();
|
|
632
|
+
const finalFp = computeFingerprint(snapshot.elements);
|
|
633
|
+
if (finalFp === rootFingerprint || knownRootFingerprints?.has(finalFp)) return true;
|
|
634
|
+
} catch { /* ignore */ }
|
|
635
|
+
|
|
636
|
+
return false;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
function log(config: WalkerConfig, message: string): void {
|
|
640
|
+
if (config.json) {
|
|
641
|
+
console.log(JSON.stringify({ type: 'log', message }));
|
|
642
|
+
} else {
|
|
643
|
+
console.error(message);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/** Emit NDJSON event (one JSON object per line) in JSON mode */
|
|
648
|
+
function emit(config: WalkerConfig, event: Record<string, unknown>): void {
|
|
649
|
+
if (config.json) {
|
|
650
|
+
console.log(JSON.stringify(event));
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
function sleep(ms: number): Promise<void> {
|
|
655
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
656
|
+
}
|