jevway 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL-FOR-AGENTS.md +138 -0
- package/LICENSE +11 -0
- package/LICENSE.jev-ultrafast +21 -0
- package/README.md +218 -2
- package/dist/browser/bot-wall.js +105 -0
- package/dist/browser/cdp.js +112 -0
- package/dist/browser/chrome.js +147 -0
- package/dist/browser/devices.js +20 -0
- package/dist/browser/frames.js +516 -0
- package/dist/browser/snapshot.js +611 -0
- package/dist/browser/tab.js +938 -0
- package/dist/browser/words.js +51 -0
- package/dist/cli.js +112 -0
- package/dist/jev/action-space.js +48 -0
- package/dist/jev/box.js +58 -0
- package/dist/jev/decide.js +405 -0
- package/dist/jev/instructions.js +67 -0
- package/dist/jev/model.js +189 -0
- package/dist/jev/replay.js +213 -0
- package/dist/jev/secrets.js +60 -0
- package/dist/journeys/diagnostics.js +85 -0
- package/dist/journeys/journey-doc.js +50 -0
- package/dist/journeys/journey-session.js +503 -0
- package/dist/journeys/pick-next-move.js +59 -0
- package/dist/journeys/render-viewer.js +153 -0
- package/dist/journeys/screen-boxes.js +49 -0
- package/dist/journeys/screen-judgments.js +149 -0
- package/dist/mcp.js +361 -0
- package/dist/setup/doctor-command.js +81 -0
- package/dist/setup/doctor.js +163 -0
- package/dist/setup/env-file.js +50 -0
- package/dist/task/checks.js +156 -0
- package/dist/task/evidence.js +96 -0
- package/dist/task/hosts.js +91 -0
- package/dist/task/look.js +17 -0
- package/dist/task/recording-page.js +310 -0
- package/dist/task/recording.js +232 -0
- package/dist/task/run-limit.js +66 -0
- package/dist/task/run-task.js +1607 -0
- package/dist/task/start-task.js +91 -0
- package/dist/task/upload-files.js +43 -0
- package/package.json +80 -4
- package/skill/jevway/SKILL.md +73 -0
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { DEVICES } from '../browser/devices.js';
|
|
4
|
+
import { StalePage, Tab } from '../browser/tab.js';
|
|
5
|
+
import { buildActionSpace } from '../jev/action-space.js';
|
|
6
|
+
import { decide } from '../jev/decide.js';
|
|
7
|
+
import { readScreenBoxes, screenBoxQuestions } from './screen-boxes.js';
|
|
8
|
+
import { watchDiagnostics } from './diagnostics.js';
|
|
9
|
+
import { buildFlows, hotspotOf } from './journey-doc.js';
|
|
10
|
+
import { pickNextMove } from './pick-next-move.js';
|
|
11
|
+
import { renderViewer } from './render-viewer.js';
|
|
12
|
+
import { qaFindings, readScreenJudgments, screenQuestions, } from './screen-judgments.js';
|
|
13
|
+
const EXPLORE_GOAL = `Explore this site the way a first-time mobile visitor would, one screen at a time.
|
|
14
|
+
Open the next screen that matters most for the site's core user journeys: discovering, searching, comparing,
|
|
15
|
+
and evaluating its main offering. Prefer primary navigation, search, results, and detail screens over legal,
|
|
16
|
+
footer, social, app-download, or external links. Dismiss blocking banners. Never log in, pay, or submit personal data.`;
|
|
17
|
+
/**
|
|
18
|
+
* A mobile user-journey mapping session: every screen is screenshotted, understood by Jev (type, patterns,
|
|
19
|
+
* duplicates, QA states, ranked next steps) and written to `journey.json` + `index.html` after each move.
|
|
20
|
+
* The agent steers with `step()` or lets Jev drive with `autopilot()`.
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* const journey = await JourneySession.start(cdp, ask, { startUrl: 'https://example.com', outDir: 'journeys/example' });
|
|
24
|
+
* journey.report(); // screen s01 + Jev's suggested next tap
|
|
25
|
+
* await journey.step({ next: 4, flow: 'Search', insight: 'Search is a full-screen sheet' });
|
|
26
|
+
* await journey.autopilot({ screens: 5, flow: 'Search/Results' });
|
|
27
|
+
* journey.finish(); // → journey.json, index.html, screens/*.png
|
|
28
|
+
*/
|
|
29
|
+
export class JourneySession {
|
|
30
|
+
doc;
|
|
31
|
+
outDir;
|
|
32
|
+
#tab;
|
|
33
|
+
#ask;
|
|
34
|
+
#device;
|
|
35
|
+
#options;
|
|
36
|
+
#diagnostics;
|
|
37
|
+
#page;
|
|
38
|
+
#decision;
|
|
39
|
+
#judgments;
|
|
40
|
+
#current = '';
|
|
41
|
+
#lastIsNew = true;
|
|
42
|
+
#lastFlow = 'Main';
|
|
43
|
+
#recent = [];
|
|
44
|
+
#paths = new Map();
|
|
45
|
+
#contentKeys = new Map();
|
|
46
|
+
#excerpts = new Map();
|
|
47
|
+
#tried = new Map();
|
|
48
|
+
#skippedRisky = [];
|
|
49
|
+
#moves = [];
|
|
50
|
+
// History index of the start page: "back" never leaves the site for the blank tab it was opened from.
|
|
51
|
+
#startHistoryIndex = 0;
|
|
52
|
+
constructor(tab, ask, diagnostics, options, device) {
|
|
53
|
+
this.#tab = tab;
|
|
54
|
+
this.#ask = ask;
|
|
55
|
+
this.#diagnostics = diagnostics;
|
|
56
|
+
this.#options = options;
|
|
57
|
+
this.#device = device;
|
|
58
|
+
this.outDir = resolve(options.outDir);
|
|
59
|
+
this.doc = {
|
|
60
|
+
site: new URL(options.startUrl).host,
|
|
61
|
+
startUrl: options.startUrl,
|
|
62
|
+
device: device.name,
|
|
63
|
+
...(options.intent && { intent: options.intent }),
|
|
64
|
+
createdAt: new Date().toISOString(),
|
|
65
|
+
screens: [],
|
|
66
|
+
steps: [],
|
|
67
|
+
flows: [],
|
|
68
|
+
};
|
|
69
|
+
mkdirSync(join(this.outDir, 'screens'), { recursive: true });
|
|
70
|
+
}
|
|
71
|
+
static async start(cdp, ask, options) {
|
|
72
|
+
const device = DEVICES[options.device ?? 'iphone-15'];
|
|
73
|
+
const tab = await Tab.open(cdp, 'about:blank', { device, newWindow: options.newWindow });
|
|
74
|
+
const diagnostics = await watchDiagnostics(tab);
|
|
75
|
+
const session = new JourneySession(tab, ask, diagnostics, options, device);
|
|
76
|
+
await tab.navigate(options.startUrl);
|
|
77
|
+
session.#startHistoryIndex = (await tab.send('Page.getNavigationHistory')).currentIndex;
|
|
78
|
+
await session.#capture(null);
|
|
79
|
+
return session;
|
|
80
|
+
}
|
|
81
|
+
get tab() {
|
|
82
|
+
return this.#tab;
|
|
83
|
+
}
|
|
84
|
+
/** The agent's view of the current screen: identity, Jev's understanding and suggestion, QA, element table. */
|
|
85
|
+
report() {
|
|
86
|
+
const screen = this.#screen(this.#current);
|
|
87
|
+
const space = buildActionSpace(this.#page.actions);
|
|
88
|
+
const decision = this.#decision;
|
|
89
|
+
return {
|
|
90
|
+
screen: {
|
|
91
|
+
id: screen.id,
|
|
92
|
+
is_new: this.#lastIsNew,
|
|
93
|
+
url: this.#page.url,
|
|
94
|
+
title: this.#page.title,
|
|
95
|
+
type: screen.type,
|
|
96
|
+
type_confidence: round(screen.typeConfidence),
|
|
97
|
+
patterns: screen.patterns,
|
|
98
|
+
...(screen.primaryAction && { primary_action: screen.primaryAction }),
|
|
99
|
+
},
|
|
100
|
+
screenshot: join(this.outDir, screen.screenshot),
|
|
101
|
+
qa: screen.qa,
|
|
102
|
+
jev: {
|
|
103
|
+
suggests: {
|
|
104
|
+
operation: decision.operation,
|
|
105
|
+
...(decision.action && { index: targetKeyOf(space, decision.action), element: decision.action.label }),
|
|
106
|
+
...(decision.valueKey && { value_key: decision.valueKey }),
|
|
107
|
+
confidence: round(Math.min(decision.confidence, decision.targetConfidence ?? 1)),
|
|
108
|
+
},
|
|
109
|
+
ranked_taps: rankedTaps(decision, space),
|
|
110
|
+
commit_risk: round(decision.commitRisk ?? 0),
|
|
111
|
+
auth_wall: round(this.#judgments.authWall),
|
|
112
|
+
latency_ms: decision.latencyMs,
|
|
113
|
+
},
|
|
114
|
+
elements: elementTable(space, this.#page),
|
|
115
|
+
progress: {
|
|
116
|
+
screens: this.doc.screens.length,
|
|
117
|
+
steps: this.doc.steps.length,
|
|
118
|
+
flows: [...new Set(this.doc.steps.map((step) => step.flow))],
|
|
119
|
+
viewer: join(this.outDir, 'index.html'),
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/** A downscaled JPEG of the viewport for multimodal agents (full-resolution PNGs are on disk). */
|
|
124
|
+
preview() {
|
|
125
|
+
return this.#tab.screenshot({ format: 'jpeg', quality: 55, scale: 0.5 });
|
|
126
|
+
}
|
|
127
|
+
async step(input) {
|
|
128
|
+
this.annotate(this.#current, input);
|
|
129
|
+
const flow = input.flow ?? this.#lastFlow;
|
|
130
|
+
this.#lastFlow = flow;
|
|
131
|
+
const next = String(input.next);
|
|
132
|
+
if (next === 'back')
|
|
133
|
+
return this.#goBack();
|
|
134
|
+
if (next === 'restart') {
|
|
135
|
+
await this.#tab.navigate(this.#options.startUrl);
|
|
136
|
+
await this.#capture(null);
|
|
137
|
+
return this.report();
|
|
138
|
+
}
|
|
139
|
+
if (next.startsWith('screen:')) {
|
|
140
|
+
await this.#replayTo(next.slice('screen:'.length));
|
|
141
|
+
await this.#capture(null);
|
|
142
|
+
return this.report();
|
|
143
|
+
}
|
|
144
|
+
const space = buildActionSpace(this.#page.actions);
|
|
145
|
+
const control = space.controls[next.toUpperCase()];
|
|
146
|
+
if (control) {
|
|
147
|
+
await this.#tab.act(control, this.#page);
|
|
148
|
+
await this.#capture(null);
|
|
149
|
+
return this.report();
|
|
150
|
+
}
|
|
151
|
+
const target = resolveTarget(space, next, input.value);
|
|
152
|
+
try {
|
|
153
|
+
await this.#tab.act(target.action, this.#page, input.value);
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
if (!(error instanceof StalePage))
|
|
157
|
+
throw error;
|
|
158
|
+
await this.#capture(null);
|
|
159
|
+
return {
|
|
160
|
+
...this.report(),
|
|
161
|
+
warning: 'The page changed before the tap, so nothing was tapped. This is the fresh screen.',
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
return this.#arrive(target.action, target.operation, flow, 'agent', input.value, input.insight);
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Jev drives: follow its ranked next steps, skip tried taps and off-site links, back out of explored screens,
|
|
168
|
+
* and stop at auth walls, irreversible actions, a reached intent, or after `screens` new screens.
|
|
169
|
+
*/
|
|
170
|
+
async autopilot({ screens = 6, flow, intent }) {
|
|
171
|
+
if (intent && intent !== this.#options.intent) {
|
|
172
|
+
this.#options.intent = intent;
|
|
173
|
+
// A new goal re-opens taps that were already tried while pursuing a different one.
|
|
174
|
+
this.#tried.clear();
|
|
175
|
+
await this.#capture(null);
|
|
176
|
+
}
|
|
177
|
+
const flowName = flow ?? this.#options.intent ?? 'Explore';
|
|
178
|
+
const discovered = [];
|
|
179
|
+
this.#skippedRisky = [];
|
|
180
|
+
this.#moves = [];
|
|
181
|
+
const hasIntent = Boolean(this.#options.intent);
|
|
182
|
+
// No move cap: an intent run ends only when the intent is reached, a stop condition hits, or every
|
|
183
|
+
// candidate is exhausted. Without an intent, `screens` new screens is the check-in point.
|
|
184
|
+
while (hasIntent || discovered.length < screens) {
|
|
185
|
+
const tried = this.#triedOn(this.#current);
|
|
186
|
+
const move = pickNextMove({
|
|
187
|
+
decision: this.#decision,
|
|
188
|
+
judgments: this.#judgments,
|
|
189
|
+
space: buildActionSpace(this.#page.actions),
|
|
190
|
+
tried,
|
|
191
|
+
hasIntent,
|
|
192
|
+
});
|
|
193
|
+
if ('stop' in move)
|
|
194
|
+
return this.#autopilotResult(move.stop, discovered, move.detail);
|
|
195
|
+
if (move.move === 'skip_risky') {
|
|
196
|
+
tried.add(move.key);
|
|
197
|
+
this.#skippedRisky.push(move.label);
|
|
198
|
+
this.#logMove('skip_irreversible', move.label);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (move.move === 'back') {
|
|
202
|
+
if (await this.#canGoBack()) {
|
|
203
|
+
await this.#goBack();
|
|
204
|
+
this.#logMove('back');
|
|
205
|
+
}
|
|
206
|
+
else if (this.#current !== this.doc.screens[0].id) {
|
|
207
|
+
// Sheets and SPA views often add no history entry: reload the start page and keep exploring from there.
|
|
208
|
+
await this.#tab.navigate(this.#options.startUrl);
|
|
209
|
+
await this.#capture(null);
|
|
210
|
+
this.#logMove('restart');
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
return this.#autopilotResult('exhausted', discovered);
|
|
214
|
+
}
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
tried.add(move.key);
|
|
218
|
+
const text = move.valueKey ? this.#options.values?.[move.valueKey] : undefined;
|
|
219
|
+
try {
|
|
220
|
+
await this.#tab.act(move.action, this.#page, text);
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
if (!(error instanceof StalePage))
|
|
224
|
+
throw error;
|
|
225
|
+
await this.#capture(null);
|
|
226
|
+
this.#logMove('stale_retry', move.action.label);
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const scope = this.#options.stayWithin ?? new URL(this.#options.startUrl).origin;
|
|
230
|
+
if (!(await this.#tab.observe()).url.startsWith(scope)) {
|
|
231
|
+
await this.#goBack();
|
|
232
|
+
this.#logMove('left_scope', move.action.label);
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const from = this.#current;
|
|
236
|
+
await this.#arrive(move.action, move.operation, flowName, 'autopilot', text);
|
|
237
|
+
this.#logMove(this.#lastIsNew ? 'new_screen' : from === this.#current ? 'no_change' : 'revisit', move.action.label);
|
|
238
|
+
if (this.#lastIsNew) {
|
|
239
|
+
const screen = this.#screen(this.#current);
|
|
240
|
+
discovered.push({ id: screen.id, type: screen.type, title: screen.title, via: move.action.label });
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return this.#autopilotResult('checkpoint', discovered);
|
|
244
|
+
}
|
|
245
|
+
/** Attach agent insight to any screen, or move a range of steps into a named flow. */
|
|
246
|
+
annotate(screenId, { description, insight }, relabel) {
|
|
247
|
+
const screen = this.#screen(screenId);
|
|
248
|
+
if (description)
|
|
249
|
+
screen.description = description;
|
|
250
|
+
if (insight)
|
|
251
|
+
screen.insight = insight;
|
|
252
|
+
if (relabel) {
|
|
253
|
+
for (const step of this.doc.steps.slice(relabel.fromStep, relabel.toStep + 1))
|
|
254
|
+
step.flow = relabel.flow;
|
|
255
|
+
}
|
|
256
|
+
this.#save();
|
|
257
|
+
}
|
|
258
|
+
async finish({ closeTab = true } = {}) {
|
|
259
|
+
this.#save();
|
|
260
|
+
this.#diagnostics.stop();
|
|
261
|
+
if (closeTab)
|
|
262
|
+
await this.#tab.close();
|
|
263
|
+
const qa = this.doc.screens.flatMap((screen) => screen.qa.map((finding) => ({ screen: screen.id, ...finding })));
|
|
264
|
+
return {
|
|
265
|
+
viewer: join(this.outDir, 'index.html'),
|
|
266
|
+
journey_json: join(this.outDir, 'journey.json'),
|
|
267
|
+
screens: this.doc.screens.length,
|
|
268
|
+
steps: this.doc.steps.length,
|
|
269
|
+
flows: this.doc.flows.map((flow) => ({ name: flow.name, screens: flow.screens.length })),
|
|
270
|
+
qa_by_severity: countBy(qa, (finding) => finding.severity),
|
|
271
|
+
undescribed_screens: this.doc.screens.filter((screen) => !screen.description).map((screen) => screen.id),
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
async #arrive(action, operation, flow, by, value, insight) {
|
|
275
|
+
const from = this.#current;
|
|
276
|
+
const fromPage = this.#page;
|
|
277
|
+
this.#recent.push({ operation, element: action.label, ...(value && { value_key: 'typed' }) });
|
|
278
|
+
await this.#capture({
|
|
279
|
+
from,
|
|
280
|
+
flow,
|
|
281
|
+
by,
|
|
282
|
+
action: { operation, label: action.label, ...(value && { value }) },
|
|
283
|
+
hotspot: hotspotOf(action, fromPage),
|
|
284
|
+
...(insight && { insight }),
|
|
285
|
+
});
|
|
286
|
+
return this.report();
|
|
287
|
+
}
|
|
288
|
+
// Observe → one Jev request (next step + screen understanding) → dedupe → screenshot → QA → save.
|
|
289
|
+
async #capture(arrival) {
|
|
290
|
+
await this.#diagnostics.waitForIdle();
|
|
291
|
+
const page = await this.#settledPage();
|
|
292
|
+
const seesScreen = (this.#options.screenshots ?? 'off') !== 'off';
|
|
293
|
+
const boxed = seesScreen && Boolean(this.#options.box);
|
|
294
|
+
const decision = await decide(this.#ask, {
|
|
295
|
+
page,
|
|
296
|
+
goal: this.#options.intent ?? EXPLORE_GOAL,
|
|
297
|
+
values: this.#options.values ?? {},
|
|
298
|
+
history: this.#recent.slice(-10),
|
|
299
|
+
checkCommitRisk: true,
|
|
300
|
+
extraQuestions: {
|
|
301
|
+
...screenQuestions(this.doc.screens.map((screen) => visitedSummary(screen, this.#excerpts))),
|
|
302
|
+
...(boxed && screenBoxQuestions()),
|
|
303
|
+
},
|
|
304
|
+
compactRules: this.#options.compactRules,
|
|
305
|
+
imageTransport: this.#options.imageTransport,
|
|
306
|
+
...(this.#options.elementRisk && { elementRisk: 'ranked' }),
|
|
307
|
+
...(seesScreen && { screenshot: await this.#tab.markedScreenshot(buildActionSpace(page.actions).marks, page) }),
|
|
308
|
+
});
|
|
309
|
+
const judgments = readScreenJudgments(decision.answers);
|
|
310
|
+
// Mobile emulation widens the layout viewport to fit overflowing content, so compare against the device width.
|
|
311
|
+
const overflow = Number(await this.#tab.evaluate(`document.documentElement.scrollWidth - ${this.#device.width}`).catch(() => 0));
|
|
312
|
+
const findings = qaFindings(page, this.#diagnostics.drain(), judgments, overflow, this.#device.mobile);
|
|
313
|
+
const boxes = boxed ? readScreenBoxes(decision.answers, page) : { primaryAction: undefined, finding: undefined };
|
|
314
|
+
if (boxes.finding)
|
|
315
|
+
findings.push(boxes.finding);
|
|
316
|
+
const contentKey = `${normalizeUrl(page.url)}\n${page.title}\n${page.text.slice(0, 600)}`;
|
|
317
|
+
const existing = this.#contentKeys.get(contentKey) ?? judgments.sameAs;
|
|
318
|
+
const id = existing ?? `s${String(this.doc.screens.length + 1).padStart(2, '0')}`;
|
|
319
|
+
if (existing) {
|
|
320
|
+
mergeFindings(this.#screen(existing).qa, findings);
|
|
321
|
+
}
|
|
322
|
+
else {
|
|
323
|
+
const screenshot = `screens/${id}.png`;
|
|
324
|
+
writeFileSync(join(this.outDir, screenshot), Buffer.from(await this.#tab.screenshot(), 'base64'));
|
|
325
|
+
this.doc.screens.push({
|
|
326
|
+
id,
|
|
327
|
+
url: page.url,
|
|
328
|
+
title: page.title,
|
|
329
|
+
screenshot,
|
|
330
|
+
capturedAt: new Date().toISOString(),
|
|
331
|
+
type: judgments.type,
|
|
332
|
+
typeConfidence: judgments.typeConfidence,
|
|
333
|
+
patterns: judgments.patterns,
|
|
334
|
+
qa: findings,
|
|
335
|
+
...(boxes.primaryAction && { primaryAction: boxes.primaryAction }),
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
this.#contentKeys.set(contentKey, id);
|
|
339
|
+
if (!existing)
|
|
340
|
+
this.#excerpts.set(id, page.text.slice(0, 300));
|
|
341
|
+
// Autopilot taps that leave the screen unchanged (current tab, no-op buttons) are not journey steps.
|
|
342
|
+
if (arrival && !(arrival.by === 'autopilot' && id === arrival.from)) {
|
|
343
|
+
const step = { ...arrival, to: id };
|
|
344
|
+
this.doc.steps.push(step);
|
|
345
|
+
if (!this.#paths.has(id))
|
|
346
|
+
this.#paths.set(id, [...(this.#paths.get(arrival.from) ?? []), step]);
|
|
347
|
+
}
|
|
348
|
+
else if (!this.#paths.has(id)) {
|
|
349
|
+
this.#paths.set(id, []);
|
|
350
|
+
}
|
|
351
|
+
this.#page = page;
|
|
352
|
+
this.#decision = decision;
|
|
353
|
+
this.#judgments = judgments;
|
|
354
|
+
this.#current = id;
|
|
355
|
+
this.#lastIsNew = !existing;
|
|
356
|
+
this.#save();
|
|
357
|
+
}
|
|
358
|
+
// Sheets and pages fade in: wait for finite animations, then for two identical snapshots, so the element
|
|
359
|
+
// table and the screenshot show the arrived screen rather than a transition frame.
|
|
360
|
+
async #settledPage() {
|
|
361
|
+
await this.#tab
|
|
362
|
+
.evaluate(`Promise.race([
|
|
363
|
+
Promise.all(document.getAnimations()
|
|
364
|
+
.filter(a => a.playState === 'running' && a.effect?.getComputedTiming().endTime !== Infinity)
|
|
365
|
+
.map(a => a.finished.catch(() => {}))),
|
|
366
|
+
new Promise(r => setTimeout(r, 1500)),
|
|
367
|
+
]).then(() => true)`)
|
|
368
|
+
.catch(() => undefined);
|
|
369
|
+
let page = await this.#tab.observe();
|
|
370
|
+
for (let attempt = 0; attempt < 6; attempt++) {
|
|
371
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
372
|
+
const next = await this.#tab.observe();
|
|
373
|
+
if (next.fingerprint === page.fingerprint)
|
|
374
|
+
return next;
|
|
375
|
+
page = next;
|
|
376
|
+
}
|
|
377
|
+
return page;
|
|
378
|
+
}
|
|
379
|
+
async #goBack() {
|
|
380
|
+
await this.#tab.evaluate('history.back()').catch(() => undefined);
|
|
381
|
+
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
382
|
+
await this.#capture(null);
|
|
383
|
+
return this.report();
|
|
384
|
+
}
|
|
385
|
+
async #canGoBack() {
|
|
386
|
+
const { currentIndex } = await this.#tab.send('Page.getNavigationHistory');
|
|
387
|
+
return currentIndex > this.#startHistoryIndex;
|
|
388
|
+
}
|
|
389
|
+
// SPAs rarely encode state in the URL: reload the start page and repeat the recorded taps by label.
|
|
390
|
+
async #replayTo(screenId) {
|
|
391
|
+
const path = this.#paths.get(screenId);
|
|
392
|
+
if (!path)
|
|
393
|
+
throw new Error(`No recorded path to screen ${screenId}`);
|
|
394
|
+
await this.#tab.navigate(this.#options.startUrl);
|
|
395
|
+
for (const step of path) {
|
|
396
|
+
await this.#diagnostics.waitForIdle();
|
|
397
|
+
const page = await this.#tab.observe();
|
|
398
|
+
const kind = { CLICK: 'click', TYPE_TEXT: 'fill', SELECT: 'select' }[step.action.operation];
|
|
399
|
+
const action = page.actions.find((candidate) => candidate.kind === kind && candidate.label === step.action.label);
|
|
400
|
+
if (!action)
|
|
401
|
+
throw new Error(`Replay to ${screenId} failed: "${step.action.label}" is no longer on the page`);
|
|
402
|
+
await this.#tab.act(action, page, step.action.value);
|
|
403
|
+
}
|
|
404
|
+
this.#diagnostics.drain();
|
|
405
|
+
}
|
|
406
|
+
#logMove(move, element) {
|
|
407
|
+
this.#moves.push({ move, ...(element && { element: element.slice(0, 80) }), screen: this.#current });
|
|
408
|
+
}
|
|
409
|
+
#autopilotResult(stopped, discovered, detail) {
|
|
410
|
+
return {
|
|
411
|
+
stopped,
|
|
412
|
+
...(detail && { detail }),
|
|
413
|
+
moves: this.#moves.length,
|
|
414
|
+
move_log: this.#moves,
|
|
415
|
+
discovered,
|
|
416
|
+
...(this.#skippedRisky.length && { skipped_irreversible: this.#skippedRisky }),
|
|
417
|
+
...this.report(),
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
#triedOn(screenId) {
|
|
421
|
+
let tried = this.#tried.get(screenId);
|
|
422
|
+
if (!tried)
|
|
423
|
+
this.#tried.set(screenId, (tried = new Set()));
|
|
424
|
+
return tried;
|
|
425
|
+
}
|
|
426
|
+
#screen(id) {
|
|
427
|
+
const screen = this.doc.screens.find((candidate) => candidate.id === id);
|
|
428
|
+
if (!screen)
|
|
429
|
+
throw new Error(`Unknown screen "${id}"`);
|
|
430
|
+
return screen;
|
|
431
|
+
}
|
|
432
|
+
#save() {
|
|
433
|
+
this.doc.flows = buildFlows(this.doc.steps);
|
|
434
|
+
writeFileSync(join(this.outDir, 'journey.json'), JSON.stringify(this.doc, null, 2));
|
|
435
|
+
writeFileSync(join(this.outDir, 'index.html'), renderViewer(this.doc));
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
// "https://site/#/" and "https://site/" are the same route.
|
|
439
|
+
const normalizeUrl = (url) => url.replace(/#\/?$/, '').replace(/\/$/, '');
|
|
440
|
+
const visitedSummary = (screen, excerpts) => ({
|
|
441
|
+
id: screen.id,
|
|
442
|
+
type: screen.type,
|
|
443
|
+
title: screen.title,
|
|
444
|
+
url: screen.url,
|
|
445
|
+
excerpt: screen.description ?? excerpts.get(screen.id) ?? '',
|
|
446
|
+
});
|
|
447
|
+
const resolveTarget = (space, next, value) => {
|
|
448
|
+
const byOperation = (operation) => {
|
|
449
|
+
const action = space.targets[operation]?.[next];
|
|
450
|
+
return action && { action, operation };
|
|
451
|
+
};
|
|
452
|
+
const target = next.includes(':')
|
|
453
|
+
? byOperation('SELECT')
|
|
454
|
+
: value !== undefined
|
|
455
|
+
? byOperation('TYPE_TEXT')
|
|
456
|
+
: (byOperation('CLICK') ?? byOperation('TYPE_TEXT'));
|
|
457
|
+
if (!target)
|
|
458
|
+
throw new Error(`No element [${next}] supports that step. Check "elements" in the last report.`);
|
|
459
|
+
if (target.operation === 'TYPE_TEXT' && value === undefined) {
|
|
460
|
+
throw new Error(`Element [${next}] is a text field: pass "value" with the text to type.`);
|
|
461
|
+
}
|
|
462
|
+
return target;
|
|
463
|
+
};
|
|
464
|
+
const targetKeyOf = (space, action) => {
|
|
465
|
+
for (const candidates of Object.values(space.targets)) {
|
|
466
|
+
const match = Object.entries(candidates).find(([, candidate]) => candidate === action);
|
|
467
|
+
if (match)
|
|
468
|
+
return match[0];
|
|
469
|
+
}
|
|
470
|
+
return action.id.toLowerCase();
|
|
471
|
+
};
|
|
472
|
+
const rankedTaps = (decision, space) => Object.entries((decision.answers.click_target?.probabilities ?? {}))
|
|
473
|
+
.sort(([, a], [, b]) => b - a)
|
|
474
|
+
.slice(0, 6)
|
|
475
|
+
.map(([index, probability]) => ({
|
|
476
|
+
index,
|
|
477
|
+
element: space.targets.CLICK[index].label,
|
|
478
|
+
probability: round(probability),
|
|
479
|
+
}));
|
|
480
|
+
const elementTable = (space, page) => {
|
|
481
|
+
const rows = space.elements.slice(0, 150).map((element) => {
|
|
482
|
+
const value = element.value ? ` = "${String(element.value).slice(0, 40)}"` : '';
|
|
483
|
+
const options = element.options
|
|
484
|
+
? ` options: ${element.options.map((o) => `${o.index} ${o.label.split(' → ')[1]}`).join(', ')}`
|
|
485
|
+
: '';
|
|
486
|
+
return `[${element.index}] ${element.role ?? ''} "${element.label.slice(0, 80)}"${value} (${element.operations.join('/')})${options}`;
|
|
487
|
+
});
|
|
488
|
+
const hidden = space.elements.length - rows.length + page.omitted_actions;
|
|
489
|
+
return hidden > 0 ? [...rows, `… ${hidden} more off-table; scroll_down reveals more`] : rows;
|
|
490
|
+
};
|
|
491
|
+
// Layout checks describe the screen as a whole: a revisit replaces them. Runtime errors accumulate.
|
|
492
|
+
const SCREEN_WIDE_CHECKS = new Set(['small_tap_target', 'unlabeled_control', 'horizontal_overflow']);
|
|
493
|
+
const mergeFindings = (target, findings) => {
|
|
494
|
+
for (const finding of findings) {
|
|
495
|
+
if (SCREEN_WIDE_CHECKS.has(finding.kind)) {
|
|
496
|
+
target.splice(0, target.length, ...target.filter((existing) => existing.kind !== finding.kind));
|
|
497
|
+
}
|
|
498
|
+
if (!target.some((existing) => existing.kind === finding.kind && existing.detail === finding.detail))
|
|
499
|
+
target.push(finding);
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
const countBy = (items, key) => items.reduce((counts, item) => ({ ...counts, [key(item)]: (counts[key(item)] ?? 0) + 1 }), {});
|
|
503
|
+
const round = (value) => Math.round(value * 1000) / 1000;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Beyond this rank a tap is rarely part of a core journey; backing out explores more usefully.
|
|
2
|
+
const MAX_RANKED_TAPS = 12;
|
|
3
|
+
// With an intent, taps Jev considers nearly irrelevant are noise (logos, org switchers), not exploration.
|
|
4
|
+
const INTENT_MIN_PROBABILITY = 0.03;
|
|
5
|
+
/**
|
|
6
|
+
* Autopilot policy for one screen, from the Jev answers already in hand (no extra request):
|
|
7
|
+
* Jev's chosen step first, then remaining taps by Jev's probability, then one scroll, then back.
|
|
8
|
+
* Irreversible taps stop an intent run and are skipped while exploring.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* pickNextMove({ decision, judgments, space, tried: new Set(['CLICK:Search']), hasIntent: false })
|
|
12
|
+
* // { move: 'act', key: 'CLICK:Filters', operation: 'CLICK', action }
|
|
13
|
+
*/
|
|
14
|
+
export const pickNextMove = ({ decision, judgments, space, tried, hasIntent, }) => {
|
|
15
|
+
if (judgments.type === 'auth' || judgments.authWall >= 0.6)
|
|
16
|
+
return { stop: 'auth_wall' };
|
|
17
|
+
if (hasIntent && decision.operation === 'DONE')
|
|
18
|
+
return { stop: 'intent_reached' };
|
|
19
|
+
const ranked = Object.entries((decision.answers.click_target?.probabilities ?? {}))
|
|
20
|
+
.sort(([, a], [, b]) => b - a)
|
|
21
|
+
.filter(([, probability]) => !hasIntent || probability >= INTENT_MIN_PROBABILITY)
|
|
22
|
+
.slice(0, MAX_RANKED_TAPS)
|
|
23
|
+
.map(([index]) => ({
|
|
24
|
+
operation: 'CLICK',
|
|
25
|
+
action: space.targets.CLICK[index],
|
|
26
|
+
valueKey: undefined,
|
|
27
|
+
}));
|
|
28
|
+
const chosen = decision.action && ['CLICK', 'SELECT'].includes(decision.operation)
|
|
29
|
+
? [{ operation: decision.operation, action: decision.action, valueKey: undefined }]
|
|
30
|
+
: decision.action && decision.operation === 'TYPE_TEXT' && decision.valueKey
|
|
31
|
+
? [{ operation: 'TYPE_TEXT', action: decision.action, valueKey: decision.valueKey }]
|
|
32
|
+
: [];
|
|
33
|
+
const topTap = ranked[0]?.action;
|
|
34
|
+
// Per-element risk scores every tap by name; an unscored tap counts as risky. Without them, the
|
|
35
|
+
// page-level commit_risk describes only the top tap.
|
|
36
|
+
const clickIndexOf = (action) => Object.entries(space.targets.CLICK ?? {}).find(([, candidate]) => candidate === action)?.[0];
|
|
37
|
+
const riskOf = (operation, action) => {
|
|
38
|
+
if (operation !== 'CLICK')
|
|
39
|
+
return 0;
|
|
40
|
+
if (decision.elementRisks)
|
|
41
|
+
return decision.elementRisks[clickIndexOf(action) ?? ''] ?? 1;
|
|
42
|
+
return action === topTap ? (decision.commitRisk ?? 0) : 0;
|
|
43
|
+
};
|
|
44
|
+
for (const candidate of [...chosen, ...ranked]) {
|
|
45
|
+
const key = `${candidate.operation}:${candidate.action.label}`;
|
|
46
|
+
if (tried.has(key))
|
|
47
|
+
continue;
|
|
48
|
+
const risky = riskOf(candidate.operation, candidate.action) >= 0.5;
|
|
49
|
+
if (risky && hasIntent)
|
|
50
|
+
return { stop: 'irreversible_action', detail: candidate.action.label };
|
|
51
|
+
if (risky)
|
|
52
|
+
return { move: 'skip_risky', key, label: candidate.action.label };
|
|
53
|
+
return { move: 'act', key, ...candidate };
|
|
54
|
+
}
|
|
55
|
+
if (space.controls.SCROLL_DOWN && !tried.has('SCROLL_DOWN')) {
|
|
56
|
+
return { move: 'act', key: 'SCROLL_DOWN', operation: 'SCROLL_DOWN', action: space.controls.SCROLL_DOWN };
|
|
57
|
+
}
|
|
58
|
+
return { move: 'back' };
|
|
59
|
+
};
|