jevway 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -2
- package/{INSTALL-FOR-AGENTS.md → SETUP.md} +94 -14
- package/dist/browser/code-boxes.js +121 -0
- package/dist/browser/downloads.js +103 -0
- package/dist/browser/frames.js +23 -2
- package/dist/browser/snapshot.js +82 -1
- package/dist/browser/tab.js +100 -12
- package/dist/cli/dispatch.js +2 -0
- package/dist/cli/inbox-command.js +199 -0
- package/dist/cli/look-command.js +10 -3
- package/dist/cli/run-command.js +51 -3
- package/dist/daemon/server.js +22 -3
- package/dist/inbox/inbox-file.js +43 -0
- package/dist/inbox/inbox-registry.js +33 -0
- package/dist/inbox/inbox.js +127 -0
- package/dist/inbox/mail-api.js +52 -0
- package/dist/inbox/message.js +34 -0
- package/dist/jev/secrets.js +4 -3
- package/dist/mcp/inbox-tools.js +82 -0
- package/dist/mcp/instructions.js +11 -2
- package/dist/mcp/server.js +15 -8
- package/dist/mcp/task-tools.js +24 -8
- package/dist/notify/desktop.js +40 -0
- package/dist/notify/person-messages.js +50 -0
- package/dist/output/layout.js +37 -1
- package/dist/output/manifest.js +6 -0
- package/dist/output/run-folder.js +44 -2
- package/dist/recording/describe-step.js +3 -0
- package/dist/session/session-service.js +31 -11
- package/dist/setup/skill-command.js +44 -21
- package/dist/task/ask-user.js +182 -0
- package/dist/task/downloads.js +63 -0
- package/dist/task/person-wait.js +104 -0
- package/dist/task/present.js +32 -11
- package/dist/task/run-task.js +327 -36
- package/dist/task/start-task.js +49 -6
- package/dist/task/totp.js +124 -0
- package/dist/task/upload-files.js +237 -5
- package/package.json +2 -2
- package/skill/jevway/SKILL.md +5 -2
- package/skill/jevway-inbox/SKILL.md +61 -0
- package/skill/jevway-inbox/scripts/inbox.mjs +209 -0
package/dist/task/run-task.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { tmpdir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
1
3
|
import { findBotWall } from '../browser/bot-wall.js';
|
|
4
|
+
import { codeCharacters } from '../browser/code-boxes.js';
|
|
2
5
|
import { PAGE_LEVEL_KINDS, StalePage, TargetUnreachable } from '../browser/tab.js';
|
|
3
6
|
import { showsTarget } from '../browser/words.js';
|
|
4
7
|
import { buildActionSpace } from '../jev/action-space.js';
|
|
@@ -6,14 +9,18 @@ import { centerOf, contains } from '../jev/box.js';
|
|
|
6
9
|
import { decide, evidenceRequest, foldedFavourite, overflowFolds, readEvidence } from '../jev/decide.js';
|
|
7
10
|
import { estimateCostUsd } from '../jev/model.js';
|
|
8
11
|
import { ANY_HOST, narrowHosts, outsideHosts } from './hosts.js';
|
|
9
|
-
import { scrubberFor } from '../jev/secrets.js';
|
|
12
|
+
import { scrubberFor, secretMarker } from '../jev/secrets.js';
|
|
10
13
|
import { describeFailure, runChecks } from './checks.js';
|
|
11
14
|
import { deepLink, quoteSpans } from './evidence.js';
|
|
12
15
|
import { writeCall } from '../output/run-folder.js';
|
|
13
16
|
import { Recording } from '../recording/recording.js';
|
|
14
17
|
import { RunStopped, startRunLimit } from './run-limit.js';
|
|
15
|
-
import { checkUploadFiles, fileNames } from './upload-files.js';
|
|
18
|
+
import { checkUploadFiles, fetchUploadUrls, fileNames } from './upload-files.js';
|
|
19
|
+
import { checkTotpSecrets, totpMasks, typingCode } from './totp.js';
|
|
20
|
+
import { downloadNote, keepDownload, sizeWords } from './downloads.js';
|
|
16
21
|
import { isTransientModelError } from '../jev/retry.js';
|
|
22
|
+
import { notificationText, passedText, waitingText } from '../notify/person-messages.js';
|
|
23
|
+
import { ssoProviderOf, stillSigningIn, waitForPersonLoop } from './person-wait.js';
|
|
17
24
|
// How much of the final page's visible text a summary carries: enough to read a list or a settings section, not a whole page.
|
|
18
25
|
const PAGE_TEXT_CHARS = 2500;
|
|
19
26
|
// An agent must know when the text was cut, to read the rest with another call or chrome-devtools.
|
|
@@ -45,8 +52,13 @@ const DEFAULTS = {
|
|
|
45
52
|
hosts: ANY_HOST,
|
|
46
53
|
// Prior art (jkudish/jev-browser) waits 8 s; npm's headless challenge never cleared in 50 s, so this bounds patience.
|
|
47
54
|
wallWaitMs: 8000,
|
|
55
|
+
// A task built without the runner never waits for a person; the runner sets the call's wait (default 120 s when visible).
|
|
56
|
+
waitForPersonSeconds: 0,
|
|
48
57
|
checks: [],
|
|
49
58
|
};
|
|
59
|
+
// While waiting for a person: how often the page is read (a full observe, so not every 250 ms) and how often the countdown is told.
|
|
60
|
+
const PERSON_POLL_MS = 1000;
|
|
61
|
+
const PERSON_TICK_MS = 10_000;
|
|
50
62
|
// How often a bot check is looked at again while it gets the chance to clear itself.
|
|
51
63
|
const WALL_POLL_MS = 250;
|
|
52
64
|
// How often a page that has not reacted yet is looked at again while waiting for it to change.
|
|
@@ -136,6 +148,8 @@ const BLANK_PAGE = {
|
|
|
136
148
|
export class BrowserTask {
|
|
137
149
|
trace = [];
|
|
138
150
|
#options;
|
|
151
|
+
/** Whether a person can see the browser and how to tell them; fixed when the session starts. */
|
|
152
|
+
#person;
|
|
139
153
|
#history = [];
|
|
140
154
|
#latencies = [];
|
|
141
155
|
#inputTokens = 0;
|
|
@@ -186,6 +200,17 @@ export class BrowserTask {
|
|
|
186
200
|
#toggled = new Map();
|
|
187
201
|
/** Nodes of fields the model chose with no supplied value to type: each is refused once before the caller is asked. */
|
|
188
202
|
#refusedForValue = new Set();
|
|
203
|
+
/** First-box nodes of split-code groups a value of the wrong length was chosen for: each is fed back once before the caller is asked. */
|
|
204
|
+
#codeLengthNoted = new Set();
|
|
205
|
+
/**
|
|
206
|
+
* Boxes of a split code a secret was typed into, with the secret's key and their document (ids restart in a new one):
|
|
207
|
+
* their values read as its marker from then on.
|
|
208
|
+
*/
|
|
209
|
+
#secretBoxes = new Map();
|
|
210
|
+
/** The caller's authenticator secrets by key: only `decide` (as a marker) and the typing see them. */
|
|
211
|
+
#totp = new Map();
|
|
212
|
+
/** Secrets that are not supplied values, by key: each authenticator secret as given and as base32, every code typed. Kept for the session. */
|
|
213
|
+
#moreSecrets = [];
|
|
189
214
|
/** Trace entries before this index are already scrubbed of secret values. */
|
|
190
215
|
#traceScrubbed = 0;
|
|
191
216
|
/** The page each decision was made on: a `done_when` score counts for the caller's checks only on the page it judged. */
|
|
@@ -209,13 +234,22 @@ export class BrowserTask {
|
|
|
209
234
|
/** This call's step listener, and how many of the call's steps it was told. */
|
|
210
235
|
#onStep;
|
|
211
236
|
#stepsTold = 0;
|
|
237
|
+
#onWait;
|
|
212
238
|
/** This call's stop signal (caller cancel or time budget); set while a call runs. */
|
|
213
239
|
#limit;
|
|
214
|
-
|
|
240
|
+
/** URL files this session fetched, by URL: a later call reuses the file. */
|
|
241
|
+
#fetchedUploads = new Map();
|
|
242
|
+
/** Downloads already reported, by guid, and what this call's steps downloaded, for the page text Jev reads. */
|
|
243
|
+
#downloadsSeen = new Set();
|
|
244
|
+
#downloadedThisCall = [];
|
|
245
|
+
constructor(tab, ask, { recordDir, output, totp, person, ...options }) {
|
|
215
246
|
this.#tab = tab;
|
|
247
|
+
this.#person = person ?? { visible: true };
|
|
216
248
|
this.#tabs = [tab];
|
|
217
249
|
this.#ask = ask;
|
|
250
|
+
const authenticators = checkTotpSecrets(totp, options.values);
|
|
218
251
|
this.#options = { ...DEFAULTS, values: {}, secretKeys: [], ...definedOnly(options), files: checkUploadFiles(options.files) };
|
|
252
|
+
this.#takeTotp(authenticators);
|
|
219
253
|
this.#output = output ?? (recordDir ? { dir: recordDir, record: true } : undefined);
|
|
220
254
|
if (this.#output?.record)
|
|
221
255
|
this.#recording = new Recording(this.#output.dir, (text) => this.#scrub(text));
|
|
@@ -231,7 +265,7 @@ export class BrowserTask {
|
|
|
231
265
|
}
|
|
232
266
|
/** Replaces every value the caller marked secret with its marker, in a string or a plain object's strings. */
|
|
233
267
|
#scrub(value) {
|
|
234
|
-
return scrubberFor(this.#options.values, this.#options.secretKeys)(value);
|
|
268
|
+
return scrubberFor(this.#options.values, this.#options.secretKeys, this.#moreSecrets)(value);
|
|
235
269
|
}
|
|
236
270
|
// jevway: a page is scrubbed of secrets as soon as the run reads it, so every label, value, history line, page
|
|
237
271
|
// message and reason is built from scrubbed text before anything cuts it short (a cut secret no longer matches the
|
|
@@ -241,24 +275,33 @@ export class BrowserTask {
|
|
|
241
275
|
const page = await this.#tab.observe();
|
|
242
276
|
if (!this.#options.secretKeys.length)
|
|
243
277
|
return page;
|
|
244
|
-
const scrub = scrubberFor(this.#options.values, this.#options.secretKeys);
|
|
278
|
+
const scrub = scrubberFor(this.#options.values, this.#options.secretKeys, this.#moreSecrets);
|
|
245
279
|
return {
|
|
246
280
|
...page,
|
|
247
281
|
url: scrub(page.url),
|
|
248
282
|
title: scrub(page.title),
|
|
249
283
|
text: scrub(page.text),
|
|
250
284
|
...(page.passwordFields && { passwordFields: page.passwordFields.map((field) => ({ ...field, name: scrub(field.name) })) }),
|
|
251
|
-
actions: page.actions.map((action) =>
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
285
|
+
actions: page.actions.map((action) => {
|
|
286
|
+
// A box of a split secret code may hold part of it, which no value match finds: it shows the key's marker.
|
|
287
|
+
const document = documentOf(page, action);
|
|
288
|
+
const boxKey = [action.node, ...(action.boxes ?? [])]
|
|
289
|
+
.map((node) => this.#secretBoxes.get(node))
|
|
290
|
+
.find((noted) => noted?.document === document)?.key;
|
|
291
|
+
return {
|
|
292
|
+
...action,
|
|
293
|
+
label: scrub(action.label),
|
|
294
|
+
...(action.value !== undefined && action.kind !== 'select' && { value: scrub(action.value) }),
|
|
295
|
+
...(action.current_value !== undefined && { current_value: scrub(action.current_value) }),
|
|
296
|
+
...(boxKey && action.value && { value: secretMarker(boxKey) }),
|
|
297
|
+
};
|
|
298
|
+
}),
|
|
257
299
|
};
|
|
258
300
|
}
|
|
259
|
-
/** Reads the session's tab with no model call and no input, its secret values already masked: what `jevway_look` shows. */
|
|
260
|
-
look() {
|
|
261
|
-
|
|
301
|
+
/** Reads the session's tab with no model call and no input, its secret values already masked: what `jevway_look` shows. With `pdf`, the page as a PDF too. */
|
|
302
|
+
async look({ pdf = false } = {}) {
|
|
303
|
+
const page = await this.#observe();
|
|
304
|
+
return pdf ? { ...page, pdf: await this.#tab.pdf() } : page;
|
|
262
305
|
}
|
|
263
306
|
/** The tab the session is on now: a tab its clicks opened, once followed, or the one it started on. */
|
|
264
307
|
get tab() {
|
|
@@ -311,9 +354,10 @@ export class BrowserTask {
|
|
|
311
354
|
}
|
|
312
355
|
/** Merge new values, a new goal, narrower hosts (a widening list throws), or a one-time confirmation for the next irreversible click. */
|
|
313
356
|
update(patch) {
|
|
314
|
-
const { confirmIrreversible, confirmTarget, values, secretKeys, files, recordDir: _recordDir, output: _output, allowedHosts, blockedHosts, ...options } = patch;
|
|
315
|
-
// First, so a list that would widen the hosts throws before anything else changes.
|
|
357
|
+
const { confirmIrreversible, confirmTarget, values, secretKeys, files, totp, recordDir: _recordDir, output: _output, person: _person, allowedHosts, blockedHosts, ...options } = patch;
|
|
358
|
+
// First, so a list that would widen the hosts, or a bad authenticator secret, throws before anything else changes.
|
|
316
359
|
const hosts = narrowHosts(this.#options.hosts, { allowed: allowedHosts, blocked: blockedHosts });
|
|
360
|
+
const authenticators = checkTotpSecrets(totp, values);
|
|
317
361
|
const newGoal = options.goal !== undefined && options.goal !== this.#options.goal;
|
|
318
362
|
// A new goal may be to flip back what the old one flipped.
|
|
319
363
|
if (newGoal)
|
|
@@ -329,6 +373,7 @@ export class BrowserTask {
|
|
|
329
373
|
secretKeys: [...new Set([...this.#options.secretKeys, ...(secretKeys ?? [])])],
|
|
330
374
|
files: { ...this.#options.files, ...checkUploadFiles(files) },
|
|
331
375
|
};
|
|
376
|
+
this.#takeTotp(authenticators, values);
|
|
332
377
|
if (!confirmIrreversible)
|
|
333
378
|
return;
|
|
334
379
|
// A confirmation answers the question jevway asked, so a flag carried onto a later call allows nothing (an unused earlier one stands).
|
|
@@ -340,13 +385,40 @@ export class BrowserTask {
|
|
|
340
385
|
else if (!this.#confirmedCommit)
|
|
341
386
|
this.#confirmIgnored = this.#lastStatus;
|
|
342
387
|
}
|
|
388
|
+
// The newest plain value or authenticator secret for a key replaces the older one; either stays secret for the session.
|
|
389
|
+
#takeTotp(authenticators, values = {}) {
|
|
390
|
+
for (const key of Object.keys(values))
|
|
391
|
+
this.#totp.delete(key);
|
|
392
|
+
for (const [key, secret] of authenticators)
|
|
393
|
+
this.#totp.set(key, secret);
|
|
394
|
+
this.#moreSecrets.push(...totpMasks(authenticators));
|
|
395
|
+
if (!authenticators.size)
|
|
396
|
+
return;
|
|
397
|
+
this.#options.values = Object.fromEntries(Object.entries(this.#options.values).filter(([key]) => !authenticators.has(key)));
|
|
398
|
+
this.#options.secretKeys = [...new Set([...this.#options.secretKeys, ...authenticators.keys()])];
|
|
399
|
+
}
|
|
400
|
+
// An authenticator key types the code current now (the next one when its period is about to end), masked before it is typed.
|
|
401
|
+
async #textFor(key, values) {
|
|
402
|
+
const secret = this.#totp.get(key);
|
|
403
|
+
if (!secret)
|
|
404
|
+
return values[key];
|
|
405
|
+
const code = await typingCode(secret, { now: Date.now, pause: (ms) => this.#limit?.pause(ms) ?? Promise.resolve() });
|
|
406
|
+
this.#moreSecrets.push([key, code]);
|
|
407
|
+
return code;
|
|
408
|
+
}
|
|
343
409
|
/**
|
|
344
410
|
* Runs until done or a hand-back. `signal` is the caller's cancel (the MCP request's): the run stops at its next
|
|
345
411
|
* check, `blocked` "Cancelled by the caller"; an input already under way finishes first. `onStep` hears each step of
|
|
346
|
-
* this call (numbered from 1) once the run moves past it, with the page's URL then;
|
|
412
|
+
* this call (numbered from 1) once the run moves past it, with the page's URL then; `onWait` the countdown while it
|
|
413
|
+
* waits for a person at a wall. A listener that throws is ignored.
|
|
347
414
|
*/
|
|
348
|
-
async run({ signal, onStep } = {}) {
|
|
415
|
+
async run({ signal, onStep, onWait } = {}) {
|
|
416
|
+
// URL files are fetched before the run reads a page; a refused one fails the call naming its key.
|
|
417
|
+
const uploads = this.#output ? join(this.#output.dir, 'uploads') : join(tmpdir(), `jevway-uploads-${process.pid}`);
|
|
418
|
+
this.#options.files = await fetchUploadUrls(this.#options.files, { dir: uploads, hosts: this.#options.hosts, known: this.#fetchedUploads, signal });
|
|
419
|
+
this.#downloadedThisCall = [];
|
|
349
420
|
this.#onStep = onStep;
|
|
421
|
+
this.#onWait = onWait;
|
|
350
422
|
this.#stepsTold = 0;
|
|
351
423
|
// The recording is for people: a browser failure while recording must not change or hang the run's outcome.
|
|
352
424
|
await this.#recording?.start(this.#tab).catch(() => undefined);
|
|
@@ -366,6 +438,7 @@ export class BrowserTask {
|
|
|
366
438
|
this.trace[this.#traceScrubbed] = this.#scrub(this.trace[this.#traceScrubbed]);
|
|
367
439
|
const summary = this.#scrub(settled);
|
|
368
440
|
this.#onStep = undefined;
|
|
441
|
+
this.#onWait = undefined;
|
|
369
442
|
await this.#recording?.finish(this.#tab, summary, { goal: this.#options.goal, doneWhen: this.#options.doneWhen }).catch(() => undefined);
|
|
370
443
|
if (this.#output)
|
|
371
444
|
await this.#writeCall(this.#output.dir, summary);
|
|
@@ -390,6 +463,61 @@ export class BrowserTask {
|
|
|
390
463
|
const scale = Math.min(1, END_PAGE_WIDTH / (width * ratio));
|
|
391
464
|
return Buffer.from(await this.#tab.screenshot({ format: 'jpeg', quality: 70, scale }), 'base64');
|
|
392
465
|
}
|
|
466
|
+
/** Tells the wait listener the countdown; like a step listener, it only hears. */
|
|
467
|
+
#tellWait(message, secondsLeft, url) {
|
|
468
|
+
try {
|
|
469
|
+
this.#onWait?.({ message: this.#scrub(message), seconds_left: secondsLeft }, this.#scrub(url));
|
|
470
|
+
}
|
|
471
|
+
catch {
|
|
472
|
+
// The caller's progress line never decides how the run goes.
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
// jevway: with the browser in view, a person can get past a wall the run never touches. The tab comes forward, one
|
|
476
|
+
// notification says what to do, and the page is read (no model, no input) until the wall is gone or the time is up.
|
|
477
|
+
async #waitForPerson(wall, page, { isWall, clearReads, limit }) {
|
|
478
|
+
const waitMs = this.#person.visible ? this.#options.waitForPersonSeconds * 1000 : 0;
|
|
479
|
+
if (waitMs <= 0 || limit.stopped())
|
|
480
|
+
return { outcome: 'off', page };
|
|
481
|
+
const tab = this.#tab;
|
|
482
|
+
for (const [method, params] of [['Page.bringToFront', {}], ['Target.activateTarget', { targetId: tab.targetId }]]) {
|
|
483
|
+
try {
|
|
484
|
+
await tab.send(method, params);
|
|
485
|
+
}
|
|
486
|
+
catch {
|
|
487
|
+
// Best effort: an old Chrome or a closed window still gets the notification.
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
try {
|
|
491
|
+
this.#person.notify?.(notificationText(wall));
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
// A notifier only hears.
|
|
495
|
+
}
|
|
496
|
+
const { outcome, page: read } = await waitForPersonLoop({
|
|
497
|
+
waitMs,
|
|
498
|
+
pollMs: this.#person.pollMs ?? PERSON_POLL_MS,
|
|
499
|
+
tickMs: this.#person.tickMs ?? PERSON_TICK_MS,
|
|
500
|
+
clearReads,
|
|
501
|
+
read: async () => {
|
|
502
|
+
// A tab the page opens meanwhile is the person's (a "Sign in with Google" popup): never followed.
|
|
503
|
+
tab.takeOpened();
|
|
504
|
+
if (tab.closed)
|
|
505
|
+
return undefined;
|
|
506
|
+
return this.#observe().catch((error) => {
|
|
507
|
+
if (error instanceof StalePage)
|
|
508
|
+
return page;
|
|
509
|
+
throw error;
|
|
510
|
+
});
|
|
511
|
+
},
|
|
512
|
+
isWall,
|
|
513
|
+
pause: limit.pause,
|
|
514
|
+
stopped: () => Boolean(limit.stopped()),
|
|
515
|
+
tell: (secondsLeft) => this.#tellWait(waitingText(wall, secondsLeft), secondsLeft, page.url),
|
|
516
|
+
});
|
|
517
|
+
if (outcome === 'passed')
|
|
518
|
+
this.#history.push({ operation: 'WAIT_FOR_PERSON', element: passedText(wall), page_changed: true, confidence: 1 });
|
|
519
|
+
return { outcome, page: read ?? page };
|
|
520
|
+
}
|
|
393
521
|
/** Tells the step listener about this call's steps it has not heard yet; the page's URL is the one the run is on now. */
|
|
394
522
|
#tellSteps(firstStep, url) {
|
|
395
523
|
if (!this.#onStep)
|
|
@@ -490,8 +618,8 @@ export class BrowserTask {
|
|
|
490
618
|
const now = await this.#observe().catch(() => undefined);
|
|
491
619
|
if (now && (await findBotWall(this.#tab, now))) {
|
|
492
620
|
page = now;
|
|
493
|
-
const { wall } = await wallAfterWait();
|
|
494
|
-
return wall ? stopAtWall(wall, decision) : undefined;
|
|
621
|
+
const { wall, waited } = await wallAfterWait();
|
|
622
|
+
return wall ? stopAtWall(wall, decision, waited) : undefined;
|
|
495
623
|
}
|
|
496
624
|
}
|
|
497
625
|
return stop('done', reason, decision);
|
|
@@ -507,8 +635,19 @@ export class BrowserTask {
|
|
|
507
635
|
return stop(status, reason, decision, { overridable: true });
|
|
508
636
|
};
|
|
509
637
|
// jevway: a bot check offers nothing to act on, and the run never tries to get past one. It gets wallWaitMs to clear
|
|
510
|
-
// itself (the page read again meanwhile),
|
|
638
|
+
// itself (the page read again meanwhile), then, in a visible browser, a person gets waitForPersonSeconds to pass it;
|
|
639
|
+
// once per call: one that comes back after clearing ends the run at once.
|
|
511
640
|
let wallWaited = false;
|
|
641
|
+
// Each kind of wall waits for a person once per call.
|
|
642
|
+
const personWaited = new Set();
|
|
643
|
+
const waitForPerson = async (wall, isWall, clearReads) => {
|
|
644
|
+
if (personWaited.has(wall.kind))
|
|
645
|
+
return 'off';
|
|
646
|
+
personWaited.add(wall.kind);
|
|
647
|
+
const waited = await this.#waitForPerson(wall, page, { isWall, clearReads, limit });
|
|
648
|
+
page = waited.page;
|
|
649
|
+
return waited.outcome;
|
|
650
|
+
};
|
|
512
651
|
const wallAfterWait = async () => {
|
|
513
652
|
let wall = await findBotWall(this.#tab, page);
|
|
514
653
|
if (!wall)
|
|
@@ -529,14 +668,24 @@ export class BrowserTask {
|
|
|
529
668
|
});
|
|
530
669
|
wall = await findBotWall(this.#tab, page);
|
|
531
670
|
}
|
|
532
|
-
|
|
671
|
+
if (!wall || limit.stopped())
|
|
672
|
+
return { wall, cleared: !wall };
|
|
673
|
+
const outcome = await waitForPerson({ kind: 'bot_check', check: wall.name }, async (now) => Boolean(await findBotWall(this.#tab, now)), 1);
|
|
674
|
+
// A tab that went away is the loop's to handle, as after any step.
|
|
675
|
+
if (outcome === 'passed' || outcome === 'gone')
|
|
676
|
+
return { cleared: true, byPerson: true };
|
|
677
|
+
return { wall, cleared: false, ...(outcome === 'ran_out' && { waited: this.#options.waitForPersonSeconds }) };
|
|
533
678
|
};
|
|
534
|
-
const stopAtWall = (wall, decision) => {
|
|
679
|
+
const stopAtWall = (wall, decision, waited) => {
|
|
535
680
|
// A cancel or the time budget that cut the wait short is the reason the run ended.
|
|
536
681
|
const halt = limit.stopped();
|
|
537
682
|
if (halt)
|
|
538
683
|
return stop(halt.status, halt.reason, decision);
|
|
539
|
-
|
|
684
|
+
const { visible } = this.#person;
|
|
685
|
+
return {
|
|
686
|
+
...stop('blocked', wallReason(wall, { visible, waited }), decision),
|
|
687
|
+
wall: { kind: 'bot_check', origin: originOf(page.url), name: wall.provider, person_can_act: visible, ...(waited !== undefined && { waited_seconds: waited }) },
|
|
688
|
+
};
|
|
540
689
|
};
|
|
541
690
|
// Browser and model errors end the run as `blocked`: the calling agent keeps the session and decides what next.
|
|
542
691
|
try {
|
|
@@ -562,9 +711,31 @@ export class BrowserTask {
|
|
|
562
711
|
for (const tab of this.#tabs)
|
|
563
712
|
tab.disarmConfirm();
|
|
564
713
|
// Checked before a decision is spent: a challenge page has nothing for Jev to pick.
|
|
565
|
-
const { wall } = await wallAfterWait();
|
|
714
|
+
const { wall, waited, byPerson } = await wallAfterWait();
|
|
566
715
|
if (wall)
|
|
567
|
-
return stopAtWall(wall);
|
|
716
|
+
return stopAtWall(wall, undefined, waited);
|
|
717
|
+
// A person passed it (or its tab went away): the checks above run again on the page now.
|
|
718
|
+
if (byPerson)
|
|
719
|
+
continue;
|
|
720
|
+
// An identity provider's sign-in for another site is the person's: with a visible browser it waits for them.
|
|
721
|
+
const provider = ssoProviderOf(page.url);
|
|
722
|
+
if (provider) {
|
|
723
|
+
const origin = originOf(page.url);
|
|
724
|
+
const outcome = await waitForPerson({ kind: 'sso', provider }, (now) => Boolean(ssoProviderOf(now.url)), 2);
|
|
725
|
+
if (outcome === 'passed' || outcome === 'gone')
|
|
726
|
+
continue;
|
|
727
|
+
if (outcome === 'stopped') {
|
|
728
|
+
const halt = limit.stopped();
|
|
729
|
+
return stop(halt.status, halt.reason);
|
|
730
|
+
}
|
|
731
|
+
if (outcome === 'ran_out') {
|
|
732
|
+
const waited = this.#options.waitForPersonSeconds;
|
|
733
|
+
return {
|
|
734
|
+
...stop('blocked', ssoReason(provider, origin, waited)),
|
|
735
|
+
wall: { kind: 'sign_in', origin, name: provider.toLowerCase(), person_can_act: true, waited_seconds: waited },
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
}
|
|
568
739
|
if (++decisions > maxSteps * 2)
|
|
569
740
|
return stop('step_budget', `Made ${maxSteps * 2} decisions without finishing`);
|
|
570
741
|
let decision;
|
|
@@ -626,9 +797,18 @@ export class BrowserTask {
|
|
|
626
797
|
continue;
|
|
627
798
|
}
|
|
628
799
|
if (signIn) {
|
|
800
|
+
const wallPage = page;
|
|
801
|
+
const outcome = await waitForPerson({ kind: 'sign_in', host: hostOf(page.url) }, (now) => stillSigningIn(now, wallPage), 2);
|
|
802
|
+
if (outcome === 'passed' || outcome === 'gone')
|
|
803
|
+
continue;
|
|
804
|
+
const halt = limit.stopped();
|
|
805
|
+
if (halt)
|
|
806
|
+
return stop(halt.status, halt.reason, decision);
|
|
807
|
+
const { visible } = this.#person;
|
|
808
|
+
const waited = outcome === 'ran_out' ? this.#options.waitForPersonSeconds : undefined;
|
|
629
809
|
return {
|
|
630
|
-
...stop('blocked',
|
|
631
|
-
wall: signIn,
|
|
810
|
+
...stop('blocked', signInReason(signIn.origin, { visible, waited }), decision),
|
|
811
|
+
wall: { ...signIn, person_can_act: visible, ...(waited !== undefined && { waited_seconds: waited }) },
|
|
632
812
|
};
|
|
633
813
|
}
|
|
634
814
|
return stop('blocked', error, decision);
|
|
@@ -748,8 +928,42 @@ export class BrowserTask {
|
|
|
748
928
|
continue;
|
|
749
929
|
}
|
|
750
930
|
const action = decision.action;
|
|
751
|
-
const text = decision.valueKey ?
|
|
931
|
+
const text = decision.valueKey ? await this.#textFor(decision.valueKey, values) : undefined;
|
|
932
|
+
// A CLICK_AT carries no action.
|
|
933
|
+
if (action?.boxes && decision.valueKey && text !== undefined) {
|
|
934
|
+
const characters = codeCharacters(text, action.boxes.length);
|
|
935
|
+
if (!characters) {
|
|
936
|
+
// A value that doesn't fit the boxes is a wrong turn first, and only the second time a question for the caller.
|
|
937
|
+
record(false);
|
|
938
|
+
const [count, boxes] = [[...text].length, action.boxes.length];
|
|
939
|
+
if (this.#codeLengthNoted.has(action.node)) {
|
|
940
|
+
// A hand-back decided on a page that has since changed is stale: read it again instead.
|
|
941
|
+
if (!(await this.#tab.fresh(page))) {
|
|
942
|
+
page = await this.#observe();
|
|
943
|
+
continue;
|
|
944
|
+
}
|
|
945
|
+
return stop('needs_input', `"${action.label}" has ${boxes} boxes but the "${decision.valueKey}" value has ${count} characters; supply it as ${boxes} characters`, decision);
|
|
946
|
+
}
|
|
947
|
+
this.#codeLengthNoted.add(action.node);
|
|
948
|
+
this.#history.push({
|
|
949
|
+
operation: decision.operation,
|
|
950
|
+
element: action.label,
|
|
951
|
+
not_executed: `the "${decision.valueKey}" value has ${count} characters but this field has ${boxes} boxes; type nothing here — choose another value or step, or BLOCKED if the goal needs a value you were not given`,
|
|
952
|
+
confidence: round(decision.confidence),
|
|
953
|
+
});
|
|
954
|
+
page = await this.#observe();
|
|
955
|
+
continue;
|
|
956
|
+
}
|
|
957
|
+
// Masked before it is typed, so the page read right after typing already shows the marker.
|
|
958
|
+
if (this.#options.secretKeys.includes(decision.valueKey)) {
|
|
959
|
+
for (const node of action.boxes)
|
|
960
|
+
this.#secretBoxes.set(node, { key: decision.valueKey, document: documentOf(page, action) });
|
|
961
|
+
if (characters.join('') !== text)
|
|
962
|
+
this.#moreSecrets.push([decision.valueKey, characters.join('')]);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
752
965
|
let element;
|
|
966
|
+
let kept;
|
|
753
967
|
// jevway: the dialog a confirmed click opens is the same action asking again; any other dialog still gets the answer that changes nothing.
|
|
754
968
|
if (isRiskyClick(decision) && this.#confirmedCommit)
|
|
755
969
|
this.#tab.armConfirm([this.#confirmTarget ?? '', bareLabel(decision.action?.label ?? '')]);
|
|
@@ -781,6 +995,7 @@ export class BrowserTask {
|
|
|
781
995
|
const outcome = action.kind === 'upload'
|
|
782
996
|
? await this.#tab.upload(action, page, this.#options.files[decision.fileKey])
|
|
783
997
|
: await this.#tab.act(action, page, text, forced ? { evenIfChanged: true } : undefined);
|
|
998
|
+
kept = outcome?.kept;
|
|
784
999
|
// A scroll that moved nothing says so, or Jev keeps asking for it.
|
|
785
1000
|
element = outcome?.scrolled === 0 ? `${action.label} (nothing moved: there is no more to scroll this way)` : action.label;
|
|
786
1001
|
}
|
|
@@ -905,7 +1120,8 @@ export class BrowserTask {
|
|
|
905
1120
|
this.#history.push({
|
|
906
1121
|
...ran,
|
|
907
1122
|
// A field that is gone (navigated, submitted) can't say; one still on the page must show what was typed.
|
|
908
|
-
|
|
1123
|
+
// Split code boxes report what they hold right after typing; any other field, on the page read after it.
|
|
1124
|
+
...(kept !== undefined ? { value_kept: kept } : field && { value_kept: field.value === this.#scrub(text) }),
|
|
909
1125
|
page_changed: page.fingerprint !== before,
|
|
910
1126
|
confidence,
|
|
911
1127
|
});
|
|
@@ -930,6 +1146,7 @@ export class BrowserTask {
|
|
|
930
1146
|
if (decision.operation === 'CLICK' && decision.via !== 'disclosure' && action.expanded === 'false' && page.actions.some((a) => a.node === action.node && a.expanded === 'true')) {
|
|
931
1147
|
this.#openedByRun.add(action.node);
|
|
932
1148
|
}
|
|
1149
|
+
await this.#reportDownloads(firstStep);
|
|
933
1150
|
const repeatedDialog = this.#reportDialogs(firstStep, decision.operation === 'CLICK' ? { action, path: pathBefore, unsaved: unsavedBefore } : undefined);
|
|
934
1151
|
this.#tab.disarmConfirm();
|
|
935
1152
|
if (repeatedDialog)
|
|
@@ -1101,6 +1318,34 @@ export class BrowserTask {
|
|
|
1101
1318
|
this.#history[last] = { ...this.#history[last], not_executed: saveFirst };
|
|
1102
1319
|
return repeated;
|
|
1103
1320
|
}
|
|
1321
|
+
// jevway: the downloads the last action started, once finished (30 s at most), kept in the run folder and noted on its step.
|
|
1322
|
+
async #reportDownloads(firstStep) {
|
|
1323
|
+
const notes = [];
|
|
1324
|
+
for (const tab of this.#tabs) {
|
|
1325
|
+
for (const download of await tab.downloadsAfterInput()) {
|
|
1326
|
+
// A tab this one opened reports its own downloads, which its opener claimed too.
|
|
1327
|
+
if (this.#downloadsSeen.has(download.guid))
|
|
1328
|
+
continue;
|
|
1329
|
+
this.#downloadsSeen.add(download.guid);
|
|
1330
|
+
const named = { ...download, name: this.#scrub(download.name) };
|
|
1331
|
+
let kept;
|
|
1332
|
+
try {
|
|
1333
|
+
kept = keepDownload(named, this.#output?.dir);
|
|
1334
|
+
}
|
|
1335
|
+
catch {
|
|
1336
|
+
kept = { name: named.name, bytes: named.bytes, state: named.state, ...(named.filePath && { path: named.filePath }) };
|
|
1337
|
+
}
|
|
1338
|
+
notes.push(downloadNote(kept));
|
|
1339
|
+
if (kept.state === 'completed')
|
|
1340
|
+
this.#downloadedThisCall.push(`${kept.name} (${sizeWords(kept.bytes)})`);
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
const last = this.#history.length - 1;
|
|
1344
|
+
if (notes.length && last >= firstStep) {
|
|
1345
|
+
const entry = this.#history[last];
|
|
1346
|
+
this.#history[last] = { ...entry, element: `${entry.element} ${notes.map((note) => `(${note})`).join(' ')}` };
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1104
1349
|
// What the run left unsaved on the page a click was made on: the last field typed there, else the control last ticked.
|
|
1105
1350
|
#unsavedOn(path, ticked) {
|
|
1106
1351
|
const typed = this.#typedFields.filter((field) => field.path === path).at(-1);
|
|
@@ -1188,7 +1433,8 @@ export class BrowserTask {
|
|
|
1188
1433
|
const decision = await decide(ask, {
|
|
1189
1434
|
page: this.#withSaid(this.#withKeys(page)),
|
|
1190
1435
|
goal,
|
|
1191
|
-
|
|
1436
|
+
// An authenticator key is offered like a secret value: Jev sees its marker, never the secret.
|
|
1437
|
+
values: { ...values, ...Object.fromEntries([...this.#totp].map(([key, secret]) => [key, secret.base32])) },
|
|
1192
1438
|
secretKeys,
|
|
1193
1439
|
files: fileNames(this.#options.files),
|
|
1194
1440
|
history: this.#history.slice(-10),
|
|
@@ -1238,12 +1484,17 @@ export class BrowserTask {
|
|
|
1238
1484
|
// jevway: what the page said after the last submit is the first line of the text Jev reads. The done check judges the current
|
|
1239
1485
|
// page's text only, so a "saved" message that scrolled away (or sits above the viewport) is no evidence unless the text carries it.
|
|
1240
1486
|
// It stays while the run is on the path where the message appeared; the caller's page_text is never changed.
|
|
1487
|
+
// A download shows nothing on the page, so what this call downloaded leads the text too.
|
|
1241
1488
|
#withSaid(page) {
|
|
1242
1489
|
if (this.#said && this.#said.path !== pathOf(page.url))
|
|
1243
1490
|
this.#said = undefined;
|
|
1244
|
-
|
|
1491
|
+
const lines = [
|
|
1492
|
+
...(this.#downloadedThisCall.length ? [`[Downloaded this call] ${this.#downloadedThisCall.join(', ')}`] : []),
|
|
1493
|
+
...(this.#said ? [`[Page message after "${this.#said.label}"] ${this.#said.text}`] : []),
|
|
1494
|
+
];
|
|
1495
|
+
if (!lines.length)
|
|
1245
1496
|
return page;
|
|
1246
|
-
return { ...page, text:
|
|
1497
|
+
return { ...page, text: `${lines.join('\n')}\n${page.text}` };
|
|
1247
1498
|
}
|
|
1248
1499
|
// The lines the page now shows that this run never saw on its path. The previous message of a submit doesn't count as seen, so a repeated one is quoted again.
|
|
1249
1500
|
#saidAfter(label, page, textBefore) {
|
|
@@ -1537,12 +1788,25 @@ const controlOf = (decision, page) => ({
|
|
|
1537
1788
|
label: decision.operation === 'CLICK_AT' ? '' : bareLabel(decision.action?.label ?? ''),
|
|
1538
1789
|
coordinate: decision.operation === 'CLICK_AT',
|
|
1539
1790
|
});
|
|
1791
|
+
// How many of a field's choices a needs_input lists, and how much of each: page text, so kept short.
|
|
1792
|
+
const FIELD_CHOICES = 10;
|
|
1793
|
+
const FIELD_CHOICE_CHARS = 80;
|
|
1794
|
+
// The choices the page lists for a field: a select of the same name, or the option rows of its open list (an expanded combobox).
|
|
1795
|
+
const choicesFor = (field, page) => {
|
|
1796
|
+
const prefix = `${field.label} → `;
|
|
1797
|
+
const selectOptions = page.actions.filter((action) => action.kind === 'select' && action.label.startsWith(prefix)).map((action) => action.label.slice(prefix.length));
|
|
1798
|
+
const openRows = field.expanded === 'true' ? page.actions.filter((action) => action.kind === 'click' && action.role === 'option').map((action) => bareLabel(action.label)) : [];
|
|
1799
|
+
return [...new Set([...selectOptions, ...openRows].map((label) => label.trim().slice(0, FIELD_CHOICE_CHARS)).filter(Boolean))].slice(0, FIELD_CHOICES);
|
|
1800
|
+
};
|
|
1540
1801
|
// The fields a needs_input wants: the one Jev chose to type into, or every empty field a BLOCKED found no value for, in page order.
|
|
1541
1802
|
const missingFields = (decision, page) => {
|
|
1542
1803
|
const fields = decision.operation === 'TYPE_TEXT' && decision.action
|
|
1543
1804
|
? [decision.action]
|
|
1544
1805
|
: decision.fieldsWithoutValue.map((label) => page.actions.find((action) => action.kind === 'fill' && action.label === label) ?? { label });
|
|
1545
|
-
return fields.map((field) =>
|
|
1806
|
+
return fields.map((field) => {
|
|
1807
|
+
const options = choicesFor(field, page);
|
|
1808
|
+
return { label: bareLabel(field.label), ...(field.role && { role: field.role }), path: pathnameOf(page.url), ...(options.length && { options }) };
|
|
1809
|
+
});
|
|
1546
1810
|
};
|
|
1547
1811
|
const sameControl = (a, b) => a.path === b.path && a.label === b.label && a.coordinate === b.coordinate;
|
|
1548
1812
|
const pathnameOf = (url) => {
|
|
@@ -1563,6 +1827,8 @@ const decoded = (url) => {
|
|
|
1563
1827
|
return url;
|
|
1564
1828
|
}
|
|
1565
1829
|
};
|
|
1830
|
+
// The document an action's node belongs to (its frame's, or the page's): its start time, as the reader's keys hold it.
|
|
1831
|
+
const documentOf = (page, action) => JSON.stringify(action.frame ? page.frames?.find((frame) => frame.id === action.frame)?.identity : page.page_key?.[0]);
|
|
1566
1832
|
const bareLabel = (label) => label.replace(/ \(appears on hover\)$/, '');
|
|
1567
1833
|
/** Up to 3 labels, in list order and cut at 40 characters, of the listed controls inside the layer covering a target; none when nothing usable names it. */
|
|
1568
1834
|
const coveringLabels = (coverNodes, page) => page.actions
|
|
@@ -1664,7 +1930,32 @@ const round = (value) => Math.round(value * 1000) / 1000;
|
|
|
1664
1930
|
class ModelCallError extends Error {
|
|
1665
1931
|
}
|
|
1666
1932
|
const messageOf = (error) => (error instanceof Error ? error.message : String(error));
|
|
1667
|
-
// The hand-back for a bot wall: what it is, what made it one, and who can pass it.
|
|
1668
|
-
const wallReason = (wall
|
|
1933
|
+
// The hand-back for a bot wall: what it is, what made it one, whether a person was waited for, and who can pass it.
|
|
1934
|
+
const wallReason = (wall, { visible, waited }) => {
|
|
1935
|
+
const seen = wall.evidence.map((e) => (e.startsWith('title ') ? e : `"${e}"`)).join(', ');
|
|
1936
|
+
const wait = waited === undefined ? '' : `, and nobody passed it in the ${waited} s jevway waited with the tab in front`;
|
|
1937
|
+
const who = visible
|
|
1938
|
+
? 'A person can pass it in this tab, then continue the session.'
|
|
1939
|
+
: 'This browser is headless, so nobody can pass it here: a visible browser is needed (JEVWAY_BROWSER=auto or launch).';
|
|
1940
|
+
return `The page is a bot check (${wall.name}; seen: ${seen}) that did not clear by itself${wait}; jevway never tries to get past one. ${who}`;
|
|
1941
|
+
};
|
|
1942
|
+
// The hand-back for a sign-in wall no supplied value passes.
|
|
1943
|
+
const signInReason = (origin, { visible, waited }) => {
|
|
1944
|
+
const wait = waited === undefined ? '' : `, and nobody signed in during the ${waited} s jevway waited with the tab in front`;
|
|
1945
|
+
const what = visible
|
|
1946
|
+
? 'sign in there (or pass the values), then call again.'
|
|
1947
|
+
: 'pass the values, or sign in with a visible browser (JEVWAY_BROWSER=auto or launch; this one is headless), then call again.';
|
|
1948
|
+
return `The page asks to sign in at ${origin} and no supplied value fits its fields${wait}; ${what}`;
|
|
1949
|
+
};
|
|
1950
|
+
// The hand-back for an SSO sign-in nobody finished while the run waited.
|
|
1951
|
+
const ssoReason = (provider, origin, waited) => `The page asks to sign in with ${provider} at ${origin} for another site, and nobody finished it during the ${waited} s jevway waited with the tab in front; sign in there, then call again.`;
|
|
1952
|
+
const hostOf = (url) => {
|
|
1953
|
+
try {
|
|
1954
|
+
return new URL(url).host;
|
|
1955
|
+
}
|
|
1956
|
+
catch {
|
|
1957
|
+
return url;
|
|
1958
|
+
}
|
|
1959
|
+
};
|
|
1669
1960
|
// Anything that is not a model error came from the tab (CDP, page never settling, an unconfirmed action).
|
|
1670
1961
|
const describeError = (error) => `${error instanceof ModelCallError ? 'Model' : 'Browser'} error: ${messageOf(error).slice(0, 200)}`;
|