jevway 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +37 -2
  2. package/{INSTALL-FOR-AGENTS.md → SETUP.md} +94 -14
  3. package/dist/browser/code-boxes.js +121 -0
  4. package/dist/browser/downloads.js +103 -0
  5. package/dist/browser/frames.js +23 -2
  6. package/dist/browser/snapshot.js +82 -1
  7. package/dist/browser/tab.js +100 -12
  8. package/dist/cli/dispatch.js +2 -0
  9. package/dist/cli/inbox-command.js +199 -0
  10. package/dist/cli/look-command.js +10 -3
  11. package/dist/cli/run-command.js +51 -3
  12. package/dist/daemon/server.js +22 -3
  13. package/dist/inbox/inbox-file.js +43 -0
  14. package/dist/inbox/inbox-registry.js +33 -0
  15. package/dist/inbox/inbox.js +127 -0
  16. package/dist/inbox/mail-api.js +52 -0
  17. package/dist/inbox/message.js +34 -0
  18. package/dist/jev/secrets.js +4 -3
  19. package/dist/mcp/inbox-tools.js +82 -0
  20. package/dist/mcp/instructions.js +11 -2
  21. package/dist/mcp/server.js +15 -8
  22. package/dist/mcp/task-tools.js +24 -8
  23. package/dist/notify/desktop.js +40 -0
  24. package/dist/notify/person-messages.js +50 -0
  25. package/dist/output/layout.js +37 -1
  26. package/dist/output/manifest.js +6 -0
  27. package/dist/output/run-folder.js +44 -2
  28. package/dist/recording/describe-step.js +3 -0
  29. package/dist/session/session-service.js +31 -11
  30. package/dist/setup/skill-command.js +44 -21
  31. package/dist/task/ask-user.js +182 -0
  32. package/dist/task/downloads.js +63 -0
  33. package/dist/task/person-wait.js +104 -0
  34. package/dist/task/present.js +32 -11
  35. package/dist/task/run-task.js +327 -36
  36. package/dist/task/start-task.js +49 -6
  37. package/dist/task/totp.js +124 -0
  38. package/dist/task/upload-files.js +237 -5
  39. package/package.json +2 -2
  40. package/skill/jevway/SKILL.md +5 -2
  41. package/skill/jevway-inbox/SKILL.md +61 -0
  42. package/skill/jevway-inbox/scripts/inbox.mjs +209 -0
@@ -1,4 +1,7 @@
1
+ import { tmpdir } from 'node:os';
2
+ import { join } from 'node:path';
1
3
  import { findBotWall } from '../browser/bot-wall.js';
4
+ import { codeCharacters } from '../browser/code-boxes.js';
2
5
  import { PAGE_LEVEL_KINDS, StalePage, TargetUnreachable } from '../browser/tab.js';
3
6
  import { showsTarget } from '../browser/words.js';
4
7
  import { buildActionSpace } from '../jev/action-space.js';
@@ -6,14 +9,18 @@ import { centerOf, contains } from '../jev/box.js';
6
9
  import { decide, evidenceRequest, foldedFavourite, overflowFolds, readEvidence } from '../jev/decide.js';
7
10
  import { estimateCostUsd } from '../jev/model.js';
8
11
  import { ANY_HOST, narrowHosts, outsideHosts } from './hosts.js';
9
- import { scrubberFor } from '../jev/secrets.js';
12
+ import { scrubberFor, secretMarker } from '../jev/secrets.js';
10
13
  import { describeFailure, runChecks } from './checks.js';
11
14
  import { deepLink, quoteSpans } from './evidence.js';
12
15
  import { writeCall } from '../output/run-folder.js';
13
16
  import { Recording } from '../recording/recording.js';
14
17
  import { RunStopped, startRunLimit } from './run-limit.js';
15
- import { checkUploadFiles, fileNames } from './upload-files.js';
18
+ import { checkUploadFiles, fetchUploadUrls, fileNames } from './upload-files.js';
19
+ import { checkTotpSecrets, totpMasks, typingCode } from './totp.js';
20
+ import { downloadNote, keepDownload, sizeWords } from './downloads.js';
16
21
  import { isTransientModelError } from '../jev/retry.js';
22
+ import { notificationText, passedText, waitingText } from '../notify/person-messages.js';
23
+ import { ssoProviderOf, stillSigningIn, waitForPersonLoop } from './person-wait.js';
17
24
  // How much of the final page's visible text a summary carries: enough to read a list or a settings section, not a whole page.
18
25
  const PAGE_TEXT_CHARS = 2500;
19
26
  // An agent must know when the text was cut, to read the rest with another call or chrome-devtools.
@@ -45,8 +52,13 @@ const DEFAULTS = {
45
52
  hosts: ANY_HOST,
46
53
  // Prior art (jkudish/jev-browser) waits 8 s; npm's headless challenge never cleared in 50 s, so this bounds patience.
47
54
  wallWaitMs: 8000,
55
+ // A task built without the runner never waits for a person; the runner sets the call's wait (default 120 s when visible).
56
+ waitForPersonSeconds: 0,
48
57
  checks: [],
49
58
  };
59
+ // While waiting for a person: how often the page is read (a full observe, so not every 250 ms) and how often the countdown is told.
60
+ const PERSON_POLL_MS = 1000;
61
+ const PERSON_TICK_MS = 10_000;
50
62
  // How often a bot check is looked at again while it gets the chance to clear itself.
51
63
  const WALL_POLL_MS = 250;
52
64
  // How often a page that has not reacted yet is looked at again while waiting for it to change.
@@ -136,6 +148,8 @@ const BLANK_PAGE = {
136
148
  export class BrowserTask {
137
149
  trace = [];
138
150
  #options;
151
+ /** Whether a person can see the browser and how to tell them; fixed when the session starts. */
152
+ #person;
139
153
  #history = [];
140
154
  #latencies = [];
141
155
  #inputTokens = 0;
@@ -186,6 +200,17 @@ export class BrowserTask {
186
200
  #toggled = new Map();
187
201
  /** Nodes of fields the model chose with no supplied value to type: each is refused once before the caller is asked. */
188
202
  #refusedForValue = new Set();
203
+ /** First-box nodes of split-code groups a value of the wrong length was chosen for: each is fed back once before the caller is asked. */
204
+ #codeLengthNoted = new Set();
205
+ /**
206
+ * Boxes of a split code a secret was typed into, with the secret's key and their document (ids restart in a new one):
207
+ * their values read as its marker from then on.
208
+ */
209
+ #secretBoxes = new Map();
210
+ /** The caller's authenticator secrets by key: only `decide` (as a marker) and the typing see them. */
211
+ #totp = new Map();
212
+ /** Secrets that are not supplied values, by key: each authenticator secret as given and as base32, every code typed. Kept for the session. */
213
+ #moreSecrets = [];
189
214
  /** Trace entries before this index are already scrubbed of secret values. */
190
215
  #traceScrubbed = 0;
191
216
  /** The page each decision was made on: a `done_when` score counts for the caller's checks only on the page it judged. */
@@ -209,13 +234,22 @@ export class BrowserTask {
209
234
  /** This call's step listener, and how many of the call's steps it was told. */
210
235
  #onStep;
211
236
  #stepsTold = 0;
237
+ #onWait;
212
238
  /** This call's stop signal (caller cancel or time budget); set while a call runs. */
213
239
  #limit;
214
- constructor(tab, ask, { recordDir, output, ...options }) {
240
+ /** URL files this session fetched, by URL: a later call reuses the file. */
241
+ #fetchedUploads = new Map();
242
+ /** Downloads already reported, by guid, and what this call's steps downloaded, for the page text Jev reads. */
243
+ #downloadsSeen = new Set();
244
+ #downloadedThisCall = [];
245
+ constructor(tab, ask, { recordDir, output, totp, person, ...options }) {
215
246
  this.#tab = tab;
247
+ this.#person = person ?? { visible: true };
216
248
  this.#tabs = [tab];
217
249
  this.#ask = ask;
250
+ const authenticators = checkTotpSecrets(totp, options.values);
218
251
  this.#options = { ...DEFAULTS, values: {}, secretKeys: [], ...definedOnly(options), files: checkUploadFiles(options.files) };
252
+ this.#takeTotp(authenticators);
219
253
  this.#output = output ?? (recordDir ? { dir: recordDir, record: true } : undefined);
220
254
  if (this.#output?.record)
221
255
  this.#recording = new Recording(this.#output.dir, (text) => this.#scrub(text));
@@ -231,7 +265,7 @@ export class BrowserTask {
231
265
  }
232
266
  /** Replaces every value the caller marked secret with its marker, in a string or a plain object's strings. */
233
267
  #scrub(value) {
234
- return scrubberFor(this.#options.values, this.#options.secretKeys)(value);
268
+ return scrubberFor(this.#options.values, this.#options.secretKeys, this.#moreSecrets)(value);
235
269
  }
236
270
  // jevway: a page is scrubbed of secrets as soon as the run reads it, so every label, value, history line, page
237
271
  // message and reason is built from scrubbed text before anything cuts it short (a cut secret no longer matches the
@@ -241,24 +275,33 @@ export class BrowserTask {
241
275
  const page = await this.#tab.observe();
242
276
  if (!this.#options.secretKeys.length)
243
277
  return page;
244
- const scrub = scrubberFor(this.#options.values, this.#options.secretKeys);
278
+ const scrub = scrubberFor(this.#options.values, this.#options.secretKeys, this.#moreSecrets);
245
279
  return {
246
280
  ...page,
247
281
  url: scrub(page.url),
248
282
  title: scrub(page.title),
249
283
  text: scrub(page.text),
250
284
  ...(page.passwordFields && { passwordFields: page.passwordFields.map((field) => ({ ...field, name: scrub(field.name) })) }),
251
- actions: page.actions.map((action) => ({
252
- ...action,
253
- label: scrub(action.label),
254
- ...(action.value !== undefined && action.kind !== 'select' && { value: scrub(action.value) }),
255
- ...(action.current_value !== undefined && { current_value: scrub(action.current_value) }),
256
- })),
285
+ actions: page.actions.map((action) => {
286
+ // A box of a split secret code may hold part of it, which no value match finds: it shows the key's marker.
287
+ const document = documentOf(page, action);
288
+ const boxKey = [action.node, ...(action.boxes ?? [])]
289
+ .map((node) => this.#secretBoxes.get(node))
290
+ .find((noted) => noted?.document === document)?.key;
291
+ return {
292
+ ...action,
293
+ label: scrub(action.label),
294
+ ...(action.value !== undefined && action.kind !== 'select' && { value: scrub(action.value) }),
295
+ ...(action.current_value !== undefined && { current_value: scrub(action.current_value) }),
296
+ ...(boxKey && action.value && { value: secretMarker(boxKey) }),
297
+ };
298
+ }),
257
299
  };
258
300
  }
259
- /** Reads the session's tab with no model call and no input, its secret values already masked: what `jevway_look` shows. */
260
- look() {
261
- return this.#observe();
301
+ /** Reads the session's tab with no model call and no input, its secret values already masked: what `jevway_look` shows. With `pdf`, the page as a PDF too. */
302
+ async look({ pdf = false } = {}) {
303
+ const page = await this.#observe();
304
+ return pdf ? { ...page, pdf: await this.#tab.pdf() } : page;
262
305
  }
263
306
  /** The tab the session is on now: a tab its clicks opened, once followed, or the one it started on. */
264
307
  get tab() {
@@ -311,9 +354,10 @@ export class BrowserTask {
311
354
  }
312
355
  /** Merge new values, a new goal, narrower hosts (a widening list throws), or a one-time confirmation for the next irreversible click. */
313
356
  update(patch) {
314
- const { confirmIrreversible, confirmTarget, values, secretKeys, files, recordDir: _recordDir, output: _output, allowedHosts, blockedHosts, ...options } = patch;
315
- // First, so a list that would widen the hosts throws before anything else changes.
357
+ const { confirmIrreversible, confirmTarget, values, secretKeys, files, totp, recordDir: _recordDir, output: _output, person: _person, allowedHosts, blockedHosts, ...options } = patch;
358
+ // First, so a list that would widen the hosts, or a bad authenticator secret, throws before anything else changes.
316
359
  const hosts = narrowHosts(this.#options.hosts, { allowed: allowedHosts, blocked: blockedHosts });
360
+ const authenticators = checkTotpSecrets(totp, values);
317
361
  const newGoal = options.goal !== undefined && options.goal !== this.#options.goal;
318
362
  // A new goal may be to flip back what the old one flipped.
319
363
  if (newGoal)
@@ -329,6 +373,7 @@ export class BrowserTask {
329
373
  secretKeys: [...new Set([...this.#options.secretKeys, ...(secretKeys ?? [])])],
330
374
  files: { ...this.#options.files, ...checkUploadFiles(files) },
331
375
  };
376
+ this.#takeTotp(authenticators, values);
332
377
  if (!confirmIrreversible)
333
378
  return;
334
379
  // A confirmation answers the question jevway asked, so a flag carried onto a later call allows nothing (an unused earlier one stands).
@@ -340,13 +385,40 @@ export class BrowserTask {
340
385
  else if (!this.#confirmedCommit)
341
386
  this.#confirmIgnored = this.#lastStatus;
342
387
  }
388
+ // The newest plain value or authenticator secret for a key replaces the older one; either stays secret for the session.
389
+ #takeTotp(authenticators, values = {}) {
390
+ for (const key of Object.keys(values))
391
+ this.#totp.delete(key);
392
+ for (const [key, secret] of authenticators)
393
+ this.#totp.set(key, secret);
394
+ this.#moreSecrets.push(...totpMasks(authenticators));
395
+ if (!authenticators.size)
396
+ return;
397
+ this.#options.values = Object.fromEntries(Object.entries(this.#options.values).filter(([key]) => !authenticators.has(key)));
398
+ this.#options.secretKeys = [...new Set([...this.#options.secretKeys, ...authenticators.keys()])];
399
+ }
400
+ // An authenticator key types the code current now (the next one when its period is about to end), masked before it is typed.
401
+ async #textFor(key, values) {
402
+ const secret = this.#totp.get(key);
403
+ if (!secret)
404
+ return values[key];
405
+ const code = await typingCode(secret, { now: Date.now, pause: (ms) => this.#limit?.pause(ms) ?? Promise.resolve() });
406
+ this.#moreSecrets.push([key, code]);
407
+ return code;
408
+ }
343
409
  /**
344
410
  * Runs until done or a hand-back. `signal` is the caller's cancel (the MCP request's): the run stops at its next
345
411
  * check, `blocked` "Cancelled by the caller"; an input already under way finishes first. `onStep` hears each step of
346
- * this call (numbered from 1) once the run moves past it, with the page's URL then; a listener that throws is ignored.
412
+ * this call (numbered from 1) once the run moves past it, with the page's URL then; `onWait` the countdown while it
413
+ * waits for a person at a wall. A listener that throws is ignored.
347
414
  */
348
- async run({ signal, onStep } = {}) {
415
+ async run({ signal, onStep, onWait } = {}) {
416
+ // URL files are fetched before the run reads a page; a refused one fails the call naming its key.
417
+ const uploads = this.#output ? join(this.#output.dir, 'uploads') : join(tmpdir(), `jevway-uploads-${process.pid}`);
418
+ this.#options.files = await fetchUploadUrls(this.#options.files, { dir: uploads, hosts: this.#options.hosts, known: this.#fetchedUploads, signal });
419
+ this.#downloadedThisCall = [];
349
420
  this.#onStep = onStep;
421
+ this.#onWait = onWait;
350
422
  this.#stepsTold = 0;
351
423
  // The recording is for people: a browser failure while recording must not change or hang the run's outcome.
352
424
  await this.#recording?.start(this.#tab).catch(() => undefined);
@@ -366,6 +438,7 @@ export class BrowserTask {
366
438
  this.trace[this.#traceScrubbed] = this.#scrub(this.trace[this.#traceScrubbed]);
367
439
  const summary = this.#scrub(settled);
368
440
  this.#onStep = undefined;
441
+ this.#onWait = undefined;
369
442
  await this.#recording?.finish(this.#tab, summary, { goal: this.#options.goal, doneWhen: this.#options.doneWhen }).catch(() => undefined);
370
443
  if (this.#output)
371
444
  await this.#writeCall(this.#output.dir, summary);
@@ -390,6 +463,61 @@ export class BrowserTask {
390
463
  const scale = Math.min(1, END_PAGE_WIDTH / (width * ratio));
391
464
  return Buffer.from(await this.#tab.screenshot({ format: 'jpeg', quality: 70, scale }), 'base64');
392
465
  }
466
+ /** Tells the wait listener the countdown; like a step listener, it only hears. */
467
+ #tellWait(message, secondsLeft, url) {
468
+ try {
469
+ this.#onWait?.({ message: this.#scrub(message), seconds_left: secondsLeft }, this.#scrub(url));
470
+ }
471
+ catch {
472
+ // The caller's progress line never decides how the run goes.
473
+ }
474
+ }
475
+ // jevway: with the browser in view, a person can get past a wall the run never touches. The tab comes forward, one
476
+ // notification says what to do, and the page is read (no model, no input) until the wall is gone or the time is up.
477
+ async #waitForPerson(wall, page, { isWall, clearReads, limit }) {
478
+ const waitMs = this.#person.visible ? this.#options.waitForPersonSeconds * 1000 : 0;
479
+ if (waitMs <= 0 || limit.stopped())
480
+ return { outcome: 'off', page };
481
+ const tab = this.#tab;
482
+ for (const [method, params] of [['Page.bringToFront', {}], ['Target.activateTarget', { targetId: tab.targetId }]]) {
483
+ try {
484
+ await tab.send(method, params);
485
+ }
486
+ catch {
487
+ // Best effort: an old Chrome or a closed window still gets the notification.
488
+ }
489
+ }
490
+ try {
491
+ this.#person.notify?.(notificationText(wall));
492
+ }
493
+ catch {
494
+ // A notifier only hears.
495
+ }
496
+ const { outcome, page: read } = await waitForPersonLoop({
497
+ waitMs,
498
+ pollMs: this.#person.pollMs ?? PERSON_POLL_MS,
499
+ tickMs: this.#person.tickMs ?? PERSON_TICK_MS,
500
+ clearReads,
501
+ read: async () => {
502
+ // A tab the page opens meanwhile is the person's (a "Sign in with Google" popup): never followed.
503
+ tab.takeOpened();
504
+ if (tab.closed)
505
+ return undefined;
506
+ return this.#observe().catch((error) => {
507
+ if (error instanceof StalePage)
508
+ return page;
509
+ throw error;
510
+ });
511
+ },
512
+ isWall,
513
+ pause: limit.pause,
514
+ stopped: () => Boolean(limit.stopped()),
515
+ tell: (secondsLeft) => this.#tellWait(waitingText(wall, secondsLeft), secondsLeft, page.url),
516
+ });
517
+ if (outcome === 'passed')
518
+ this.#history.push({ operation: 'WAIT_FOR_PERSON', element: passedText(wall), page_changed: true, confidence: 1 });
519
+ return { outcome, page: read ?? page };
520
+ }
393
521
  /** Tells the step listener about this call's steps it has not heard yet; the page's URL is the one the run is on now. */
394
522
  #tellSteps(firstStep, url) {
395
523
  if (!this.#onStep)
@@ -490,8 +618,8 @@ export class BrowserTask {
490
618
  const now = await this.#observe().catch(() => undefined);
491
619
  if (now && (await findBotWall(this.#tab, now))) {
492
620
  page = now;
493
- const { wall } = await wallAfterWait();
494
- return wall ? stopAtWall(wall, decision) : undefined;
621
+ const { wall, waited } = await wallAfterWait();
622
+ return wall ? stopAtWall(wall, decision, waited) : undefined;
495
623
  }
496
624
  }
497
625
  return stop('done', reason, decision);
@@ -507,8 +635,19 @@ export class BrowserTask {
507
635
  return stop(status, reason, decision, { overridable: true });
508
636
  };
509
637
  // jevway: a bot check offers nothing to act on, and the run never tries to get past one. It gets wallWaitMs to clear
510
- // itself (the page read again meanwhile), once per call: one that comes back after clearing ends the run at once.
638
+ // itself (the page read again meanwhile), then, in a visible browser, a person gets waitForPersonSeconds to pass it;
639
+ // once per call: one that comes back after clearing ends the run at once.
511
640
  let wallWaited = false;
641
+ // Each kind of wall waits for a person once per call.
642
+ const personWaited = new Set();
643
+ const waitForPerson = async (wall, isWall, clearReads) => {
644
+ if (personWaited.has(wall.kind))
645
+ return 'off';
646
+ personWaited.add(wall.kind);
647
+ const waited = await this.#waitForPerson(wall, page, { isWall, clearReads, limit });
648
+ page = waited.page;
649
+ return waited.outcome;
650
+ };
512
651
  const wallAfterWait = async () => {
513
652
  let wall = await findBotWall(this.#tab, page);
514
653
  if (!wall)
@@ -529,14 +668,24 @@ export class BrowserTask {
529
668
  });
530
669
  wall = await findBotWall(this.#tab, page);
531
670
  }
532
- return { wall, cleared: !wall };
671
+ if (!wall || limit.stopped())
672
+ return { wall, cleared: !wall };
673
+ const outcome = await waitForPerson({ kind: 'bot_check', check: wall.name }, async (now) => Boolean(await findBotWall(this.#tab, now)), 1);
674
+ // A tab that went away is the loop's to handle, as after any step.
675
+ if (outcome === 'passed' || outcome === 'gone')
676
+ return { cleared: true, byPerson: true };
677
+ return { wall, cleared: false, ...(outcome === 'ran_out' && { waited: this.#options.waitForPersonSeconds }) };
533
678
  };
534
- const stopAtWall = (wall, decision) => {
679
+ const stopAtWall = (wall, decision, waited) => {
535
680
  // A cancel or the time budget that cut the wait short is the reason the run ended.
536
681
  const halt = limit.stopped();
537
682
  if (halt)
538
683
  return stop(halt.status, halt.reason, decision);
539
- return { ...stop('blocked', wallReason(wall), decision), wall: { kind: 'bot_check', origin: originOf(page.url), name: wall.provider } };
684
+ const { visible } = this.#person;
685
+ return {
686
+ ...stop('blocked', wallReason(wall, { visible, waited }), decision),
687
+ wall: { kind: 'bot_check', origin: originOf(page.url), name: wall.provider, person_can_act: visible, ...(waited !== undefined && { waited_seconds: waited }) },
688
+ };
540
689
  };
541
690
  // Browser and model errors end the run as `blocked`: the calling agent keeps the session and decides what next.
542
691
  try {
@@ -562,9 +711,31 @@ export class BrowserTask {
562
711
  for (const tab of this.#tabs)
563
712
  tab.disarmConfirm();
564
713
  // Checked before a decision is spent: a challenge page has nothing for Jev to pick.
565
- const { wall } = await wallAfterWait();
714
+ const { wall, waited, byPerson } = await wallAfterWait();
566
715
  if (wall)
567
- return stopAtWall(wall);
716
+ return stopAtWall(wall, undefined, waited);
717
+ // A person passed it (or its tab went away): the checks above run again on the page now.
718
+ if (byPerson)
719
+ continue;
720
+ // An identity provider's sign-in for another site is the person's: with a visible browser it waits for them.
721
+ const provider = ssoProviderOf(page.url);
722
+ if (provider) {
723
+ const origin = originOf(page.url);
724
+ const outcome = await waitForPerson({ kind: 'sso', provider }, (now) => Boolean(ssoProviderOf(now.url)), 2);
725
+ if (outcome === 'passed' || outcome === 'gone')
726
+ continue;
727
+ if (outcome === 'stopped') {
728
+ const halt = limit.stopped();
729
+ return stop(halt.status, halt.reason);
730
+ }
731
+ if (outcome === 'ran_out') {
732
+ const waited = this.#options.waitForPersonSeconds;
733
+ return {
734
+ ...stop('blocked', ssoReason(provider, origin, waited)),
735
+ wall: { kind: 'sign_in', origin, name: provider.toLowerCase(), person_can_act: true, waited_seconds: waited },
736
+ };
737
+ }
738
+ }
568
739
  if (++decisions > maxSteps * 2)
569
740
  return stop('step_budget', `Made ${maxSteps * 2} decisions without finishing`);
570
741
  let decision;
@@ -626,9 +797,18 @@ export class BrowserTask {
626
797
  continue;
627
798
  }
628
799
  if (signIn) {
800
+ const wallPage = page;
801
+ const outcome = await waitForPerson({ kind: 'sign_in', host: hostOf(page.url) }, (now) => stillSigningIn(now, wallPage), 2);
802
+ if (outcome === 'passed' || outcome === 'gone')
803
+ continue;
804
+ const halt = limit.stopped();
805
+ if (halt)
806
+ return stop(halt.status, halt.reason, decision);
807
+ const { visible } = this.#person;
808
+ const waited = outcome === 'ran_out' ? this.#options.waitForPersonSeconds : undefined;
629
809
  return {
630
- ...stop('blocked', `The page asks to sign in at ${signIn.origin} and no supplied value fits its fields; sign in there (or pass the values), then call again.`, decision),
631
- wall: signIn,
810
+ ...stop('blocked', signInReason(signIn.origin, { visible, waited }), decision),
811
+ wall: { ...signIn, person_can_act: visible, ...(waited !== undefined && { waited_seconds: waited }) },
632
812
  };
633
813
  }
634
814
  return stop('blocked', error, decision);
@@ -748,8 +928,42 @@ export class BrowserTask {
748
928
  continue;
749
929
  }
750
930
  const action = decision.action;
751
- const text = decision.valueKey ? values[decision.valueKey] : undefined;
931
+ const text = decision.valueKey ? await this.#textFor(decision.valueKey, values) : undefined;
932
+ // A CLICK_AT carries no action.
933
+ if (action?.boxes && decision.valueKey && text !== undefined) {
934
+ const characters = codeCharacters(text, action.boxes.length);
935
+ if (!characters) {
936
+ // A value that doesn't fit the boxes is a wrong turn first, and only the second time a question for the caller.
937
+ record(false);
938
+ const [count, boxes] = [[...text].length, action.boxes.length];
939
+ if (this.#codeLengthNoted.has(action.node)) {
940
+ // A hand-back decided on a page that has since changed is stale: read it again instead.
941
+ if (!(await this.#tab.fresh(page))) {
942
+ page = await this.#observe();
943
+ continue;
944
+ }
945
+ return stop('needs_input', `"${action.label}" has ${boxes} boxes but the "${decision.valueKey}" value has ${count} characters; supply it as ${boxes} characters`, decision);
946
+ }
947
+ this.#codeLengthNoted.add(action.node);
948
+ this.#history.push({
949
+ operation: decision.operation,
950
+ element: action.label,
951
+ not_executed: `the "${decision.valueKey}" value has ${count} characters but this field has ${boxes} boxes; type nothing here — choose another value or step, or BLOCKED if the goal needs a value you were not given`,
952
+ confidence: round(decision.confidence),
953
+ });
954
+ page = await this.#observe();
955
+ continue;
956
+ }
957
+ // Masked before it is typed, so the page read right after typing already shows the marker.
958
+ if (this.#options.secretKeys.includes(decision.valueKey)) {
959
+ for (const node of action.boxes)
960
+ this.#secretBoxes.set(node, { key: decision.valueKey, document: documentOf(page, action) });
961
+ if (characters.join('') !== text)
962
+ this.#moreSecrets.push([decision.valueKey, characters.join('')]);
963
+ }
964
+ }
752
965
  let element;
966
+ let kept;
753
967
  // jevway: the dialog a confirmed click opens is the same action asking again; any other dialog still gets the answer that changes nothing.
754
968
  if (isRiskyClick(decision) && this.#confirmedCommit)
755
969
  this.#tab.armConfirm([this.#confirmTarget ?? '', bareLabel(decision.action?.label ?? '')]);
@@ -781,6 +995,7 @@ export class BrowserTask {
781
995
  const outcome = action.kind === 'upload'
782
996
  ? await this.#tab.upload(action, page, this.#options.files[decision.fileKey])
783
997
  : await this.#tab.act(action, page, text, forced ? { evenIfChanged: true } : undefined);
998
+ kept = outcome?.kept;
784
999
  // A scroll that moved nothing says so, or Jev keeps asking for it.
785
1000
  element = outcome?.scrolled === 0 ? `${action.label} (nothing moved: there is no more to scroll this way)` : action.label;
786
1001
  }
@@ -905,7 +1120,8 @@ export class BrowserTask {
905
1120
  this.#history.push({
906
1121
  ...ran,
907
1122
  // A field that is gone (navigated, submitted) can't say; one still on the page must show what was typed.
908
- ...(field && { value_kept: field.value === this.#scrub(text) }),
1123
+ // Split code boxes report what they hold right after typing; any other field, on the page read after it.
1124
+ ...(kept !== undefined ? { value_kept: kept } : field && { value_kept: field.value === this.#scrub(text) }),
909
1125
  page_changed: page.fingerprint !== before,
910
1126
  confidence,
911
1127
  });
@@ -930,6 +1146,7 @@ export class BrowserTask {
930
1146
  if (decision.operation === 'CLICK' && decision.via !== 'disclosure' && action.expanded === 'false' && page.actions.some((a) => a.node === action.node && a.expanded === 'true')) {
931
1147
  this.#openedByRun.add(action.node);
932
1148
  }
1149
+ await this.#reportDownloads(firstStep);
933
1150
  const repeatedDialog = this.#reportDialogs(firstStep, decision.operation === 'CLICK' ? { action, path: pathBefore, unsaved: unsavedBefore } : undefined);
934
1151
  this.#tab.disarmConfirm();
935
1152
  if (repeatedDialog)
@@ -1101,6 +1318,34 @@ export class BrowserTask {
1101
1318
  this.#history[last] = { ...this.#history[last], not_executed: saveFirst };
1102
1319
  return repeated;
1103
1320
  }
1321
+ // jevway: the downloads the last action started, once finished (30 s at most), kept in the run folder and noted on its step.
1322
+ async #reportDownloads(firstStep) {
1323
+ const notes = [];
1324
+ for (const tab of this.#tabs) {
1325
+ for (const download of await tab.downloadsAfterInput()) {
1326
+ // A tab this one opened reports its own downloads, which its opener claimed too.
1327
+ if (this.#downloadsSeen.has(download.guid))
1328
+ continue;
1329
+ this.#downloadsSeen.add(download.guid);
1330
+ const named = { ...download, name: this.#scrub(download.name) };
1331
+ let kept;
1332
+ try {
1333
+ kept = keepDownload(named, this.#output?.dir);
1334
+ }
1335
+ catch {
1336
+ kept = { name: named.name, bytes: named.bytes, state: named.state, ...(named.filePath && { path: named.filePath }) };
1337
+ }
1338
+ notes.push(downloadNote(kept));
1339
+ if (kept.state === 'completed')
1340
+ this.#downloadedThisCall.push(`${kept.name} (${sizeWords(kept.bytes)})`);
1341
+ }
1342
+ }
1343
+ const last = this.#history.length - 1;
1344
+ if (notes.length && last >= firstStep) {
1345
+ const entry = this.#history[last];
1346
+ this.#history[last] = { ...entry, element: `${entry.element} ${notes.map((note) => `(${note})`).join(' ')}` };
1347
+ }
1348
+ }
1104
1349
  // What the run left unsaved on the page a click was made on: the last field typed there, else the control last ticked.
1105
1350
  #unsavedOn(path, ticked) {
1106
1351
  const typed = this.#typedFields.filter((field) => field.path === path).at(-1);
@@ -1188,7 +1433,8 @@ export class BrowserTask {
1188
1433
  const decision = await decide(ask, {
1189
1434
  page: this.#withSaid(this.#withKeys(page)),
1190
1435
  goal,
1191
- values,
1436
+ // An authenticator key is offered like a secret value: Jev sees its marker, never the secret.
1437
+ values: { ...values, ...Object.fromEntries([...this.#totp].map(([key, secret]) => [key, secret.base32])) },
1192
1438
  secretKeys,
1193
1439
  files: fileNames(this.#options.files),
1194
1440
  history: this.#history.slice(-10),
@@ -1238,12 +1484,17 @@ export class BrowserTask {
1238
1484
  // jevway: what the page said after the last submit is the first line of the text Jev reads. The done check judges the current
1239
1485
  // page's text only, so a "saved" message that scrolled away (or sits above the viewport) is no evidence unless the text carries it.
1240
1486
  // It stays while the run is on the path where the message appeared; the caller's page_text is never changed.
1487
+ // A download shows nothing on the page, so what this call downloaded leads the text too.
1241
1488
  #withSaid(page) {
1242
1489
  if (this.#said && this.#said.path !== pathOf(page.url))
1243
1490
  this.#said = undefined;
1244
- if (!this.#said)
1491
+ const lines = [
1492
+ ...(this.#downloadedThisCall.length ? [`[Downloaded this call] ${this.#downloadedThisCall.join(', ')}`] : []),
1493
+ ...(this.#said ? [`[Page message after "${this.#said.label}"] ${this.#said.text}`] : []),
1494
+ ];
1495
+ if (!lines.length)
1245
1496
  return page;
1246
- return { ...page, text: `[Page message after "${this.#said.label}"] ${this.#said.text}\n${page.text}` };
1497
+ return { ...page, text: `${lines.join('\n')}\n${page.text}` };
1247
1498
  }
1248
1499
  // The lines the page now shows that this run never saw on its path. The previous message of a submit doesn't count as seen, so a repeated one is quoted again.
1249
1500
  #saidAfter(label, page, textBefore) {
@@ -1537,12 +1788,25 @@ const controlOf = (decision, page) => ({
1537
1788
  label: decision.operation === 'CLICK_AT' ? '' : bareLabel(decision.action?.label ?? ''),
1538
1789
  coordinate: decision.operation === 'CLICK_AT',
1539
1790
  });
1791
+ // How many of a field's choices a needs_input lists, and how much of each: page text, so kept short.
1792
+ const FIELD_CHOICES = 10;
1793
+ const FIELD_CHOICE_CHARS = 80;
1794
+ // The choices the page lists for a field: a select of the same name, or the option rows of its open list (an expanded combobox).
1795
+ const choicesFor = (field, page) => {
1796
+ const prefix = `${field.label} → `;
1797
+ const selectOptions = page.actions.filter((action) => action.kind === 'select' && action.label.startsWith(prefix)).map((action) => action.label.slice(prefix.length));
1798
+ const openRows = field.expanded === 'true' ? page.actions.filter((action) => action.kind === 'click' && action.role === 'option').map((action) => bareLabel(action.label)) : [];
1799
+ return [...new Set([...selectOptions, ...openRows].map((label) => label.trim().slice(0, FIELD_CHOICE_CHARS)).filter(Boolean))].slice(0, FIELD_CHOICES);
1800
+ };
1540
1801
  // The fields a needs_input wants: the one Jev chose to type into, or every empty field a BLOCKED found no value for, in page order.
1541
1802
  const missingFields = (decision, page) => {
1542
1803
  const fields = decision.operation === 'TYPE_TEXT' && decision.action
1543
1804
  ? [decision.action]
1544
1805
  : decision.fieldsWithoutValue.map((label) => page.actions.find((action) => action.kind === 'fill' && action.label === label) ?? { label });
1545
- return fields.map((field) => ({ label: bareLabel(field.label), ...(field.role && { role: field.role }), path: pathnameOf(page.url) }));
1806
+ return fields.map((field) => {
1807
+ const options = choicesFor(field, page);
1808
+ return { label: bareLabel(field.label), ...(field.role && { role: field.role }), path: pathnameOf(page.url), ...(options.length && { options }) };
1809
+ });
1546
1810
  };
1547
1811
  const sameControl = (a, b) => a.path === b.path && a.label === b.label && a.coordinate === b.coordinate;
1548
1812
  const pathnameOf = (url) => {
@@ -1563,6 +1827,8 @@ const decoded = (url) => {
1563
1827
  return url;
1564
1828
  }
1565
1829
  };
1830
+ // The document an action's node belongs to (its frame's, or the page's): its start time, as the reader's keys hold it.
1831
+ const documentOf = (page, action) => JSON.stringify(action.frame ? page.frames?.find((frame) => frame.id === action.frame)?.identity : page.page_key?.[0]);
1566
1832
  const bareLabel = (label) => label.replace(/ \(appears on hover\)$/, '');
1567
1833
  /** Up to 3 labels, in list order and cut at 40 characters, of the listed controls inside the layer covering a target; none when nothing usable names it. */
1568
1834
  const coveringLabels = (coverNodes, page) => page.actions
@@ -1664,7 +1930,32 @@ const round = (value) => Math.round(value * 1000) / 1000;
1664
1930
  class ModelCallError extends Error {
1665
1931
  }
1666
1932
  const messageOf = (error) => (error instanceof Error ? error.message : String(error));
1667
- // The hand-back for a bot wall: what it is, what made it one, and who can pass it.
1668
- const wallReason = (wall) => `The page is a bot check (${wall.name}; seen: ${wall.evidence.map((e) => (e.startsWith('title ') ? e : `"${e}"`)).join(', ')}) that did not clear by itself; jevway never tries to get past one. A person can pass it in this tab, then continue the session.`;
1933
+ // The hand-back for a bot wall: what it is, what made it one, whether a person was waited for, and who can pass it.
1934
+ const wallReason = (wall, { visible, waited }) => {
1935
+ const seen = wall.evidence.map((e) => (e.startsWith('title ') ? e : `"${e}"`)).join(', ');
1936
+ const wait = waited === undefined ? '' : `, and nobody passed it in the ${waited} s jevway waited with the tab in front`;
1937
+ const who = visible
1938
+ ? 'A person can pass it in this tab, then continue the session.'
1939
+ : 'This browser is headless, so nobody can pass it here: a visible browser is needed (JEVWAY_BROWSER=auto or launch).';
1940
+ return `The page is a bot check (${wall.name}; seen: ${seen}) that did not clear by itself${wait}; jevway never tries to get past one. ${who}`;
1941
+ };
1942
+ // The hand-back for a sign-in wall no supplied value passes.
1943
+ const signInReason = (origin, { visible, waited }) => {
1944
+ const wait = waited === undefined ? '' : `, and nobody signed in during the ${waited} s jevway waited with the tab in front`;
1945
+ const what = visible
1946
+ ? 'sign in there (or pass the values), then call again.'
1947
+ : 'pass the values, or sign in with a visible browser (JEVWAY_BROWSER=auto or launch; this one is headless), then call again.';
1948
+ return `The page asks to sign in at ${origin} and no supplied value fits its fields${wait}; ${what}`;
1949
+ };
1950
+ // The hand-back for an SSO sign-in nobody finished while the run waited.
1951
+ const ssoReason = (provider, origin, waited) => `The page asks to sign in with ${provider} at ${origin} for another site, and nobody finished it during the ${waited} s jevway waited with the tab in front; sign in there, then call again.`;
1952
+ const hostOf = (url) => {
1953
+ try {
1954
+ return new URL(url).host;
1955
+ }
1956
+ catch {
1957
+ return url;
1958
+ }
1959
+ };
1669
1960
  // Anything that is not a model error came from the tab (CDP, page never settling, an unconfirmed action).
1670
1961
  const describeError = (error) => `${error instanceof ModelCallError ? 'Model' : 'Browser'} error: ${messageOf(error).slice(0, 200)}`;