@geometra/mcp 1.19.22 → 1.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.js CHANGED
@@ -1,8 +1,9 @@
1
1
  import { createHash } from 'node:crypto';
2
+ import { performance } from 'node:perf_hooks';
2
3
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
3
4
  import { z } from 'zod';
4
5
  import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
5
- import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
6
+ import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, sendScreenshot, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
6
7
  function checkedStateInput() {
7
8
  return z
8
9
  .union([z.boolean(), z.literal('mixed')])
@@ -11,10 +12,10 @@ function checkedStateInput() {
11
12
  }
12
13
  function detailInput() {
13
14
  return z
14
- .enum(['minimal', 'verbose'])
15
+ .enum(['terse', 'minimal', 'verbose'])
15
16
  .optional()
16
17
  .default('minimal')
17
- .describe('`minimal` (default) returns terse action summaries. Use `verbose` for a fuller current-UI fallback.');
18
+ .describe('`terse` returns compact machine-friendly JSON. `minimal` (default) returns short human-readable summaries. `verbose` adds fuller fallback context.');
18
19
  }
19
20
  function formSchemaFormatInput() {
20
21
  return z
@@ -23,6 +24,13 @@ function formSchemaFormatInput() {
23
24
  .default('compact')
24
25
  .describe('`compact` (default) returns readable JSON fields. Use `packed` for the smallest schema payload with short keys.');
25
26
  }
27
+ function pageModelModeInput() {
28
+ return z
29
+ .enum(['inline', 'deferred'])
30
+ .optional()
31
+ .default('inline')
32
+ .describe('When returnPageModel=true, `inline` includes the full page model in the connect response. `deferred` returns connect as soon as the transport is ready and lets the caller fetch geometra_page_model separately.');
33
+ }
26
34
  function formSchemaContextInput() {
27
35
  return z
28
36
  .enum(['auto', 'always', 'none'])
@@ -37,6 +45,9 @@ function nodeFilterShape() {
37
45
  name: z.string().optional().describe('Accessible name to match (exact or substring)'),
38
46
  text: z.string().optional().describe('Text content to search for (substring match)'),
39
47
  contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated controls with the same visible name'),
48
+ promptText: z.string().optional().describe('Nearby question/prompt text to disambiguate repeated controls or actions'),
49
+ sectionText: z.string().optional().describe('Containing section/landmark/form/dialog text to disambiguate repeated controls or actions'),
50
+ itemText: z.string().optional().describe('Nearby card/row/item label to disambiguate repeated actions like “Add to cart” or “Open incident”'),
40
51
  value: z.string().optional().describe('Displayed / current field value to match (substring match)'),
41
52
  checked: checkedStateInput(),
42
53
  disabled: z.boolean().optional().describe('Match disabled state'),
@@ -66,15 +77,22 @@ function waitConditionShape() {
66
77
  .describe('Maximum time to wait before returning an error (default 10000ms)'),
67
78
  };
68
79
  }
69
- const GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE = 'Provide at least one filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
80
+ const GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE = 'Provide at least one filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
70
81
  'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text for substring matching. ' +
71
82
  'To wait until text disappears from the UI, use geometra_wait_for with text and present: false, or geometra_wait_for_resume_parse for typical resume “Parsing…” banners.';
72
- const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
83
+ const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
73
84
  'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text with a distinctive substring and present: false to wait until that text is gone ' +
74
85
  '(common for “Parsing…”, “Parsing your resume”, or similar). Passing only present/timeoutMs is not enough without a filter.';
75
86
  /** Strict input so unknown keys (e.g. textGone) fail parse; empty-filter checks happen in handlers / waitForSemanticCondition. */
76
- const geometraQueryInputSchema = z.object(nodeFilterShape()).strict();
77
- const geometraWaitForInputSchema = z.object(waitConditionShape()).strict();
87
+ const geometraQueryInputSchema = z.object({
88
+ ...nodeFilterShape(),
89
+ maxResults: z.number().int().min(1).max(50).optional().describe('Optional cap on returned matches; terse mode defaults to 8'),
90
+ detail: detailInput(),
91
+ }).strict();
92
+ const geometraWaitForInputSchema = z.object({
93
+ ...waitConditionShape(),
94
+ detail: detailInput(),
95
+ }).strict();
78
96
  /** Same upper bound as geometra_wait_for; resume uploads often need the full minute. */
79
97
  const geometraWaitForResumeParseInputSchema = z
80
98
  .object({
@@ -271,7 +289,7 @@ export function createServer() {
271
289
 
272
290
  Use \`url\` (ws://…) only when a Geometra/native server or an already-running proxy is listening. If you accidentally pass \`https://…\` in \`url\`, MCP treats it like \`pageUrl\` and starts the proxy for you.
273
291
 
274
- Chromium opens **visible** by default unless \`headless: true\`. File upload / wheel / native \`<select>\` need the proxy path (\`pageUrl\` or ws to proxy). Set \`returnForms: true\` and/or \`returnPageModel: true\` when you want a lower-turn startup response.`, {
292
+ Chromium opens **visible** by default unless \`headless: true\`. File upload / wheel / native \`<select>\` need the proxy path (\`pageUrl\` or ws to proxy). Set \`returnForms: true\` and/or \`returnPageModel: true\` when you want a lower-turn startup response. When connect first-response latency matters more than inlining the page model, pair \`returnPageModel: true\` with \`pageModelMode: "deferred"\` and call \`geometra_page_model\` next.`, {
275
293
  url: z
276
294
  .string()
277
295
  .optional()
@@ -311,6 +329,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
311
329
  .optional()
312
330
  .default(false)
313
331
  .describe('Include geometra_page_model output in the connect response so exploration can start in one turn.'),
332
+ pageModelMode: pageModelModeInput(),
314
333
  formId: z.string().optional().describe('Optional form id filter when returnForms=true'),
315
334
  maxFields: z.number().int().min(1).max(120).optional().default(80).describe('Cap returned fields per form when returnForms=true'),
316
335
  onlyRequiredFields: z.boolean().optional().default(false).describe('Only include required fields when returnForms=true'),
@@ -341,6 +360,10 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
341
360
  maxPrimaryActions: input.maxPrimaryActions,
342
361
  maxSectionsPerKind: input.maxSectionsPerKind,
343
362
  };
363
+ const deferInlinePageModel = input.returnPageModel
364
+ && input.pageModelMode === 'deferred'
365
+ && !input.returnForms
366
+ && input.detail !== 'verbose';
344
367
  try {
345
368
  if (target.kind === 'proxy') {
346
369
  const session = await connectThroughProxy({
@@ -350,6 +373,8 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
350
373
  width: input.width,
351
374
  height: input.height,
352
375
  slowMo: input.slowMo,
376
+ awaitInitialFrame: deferInlinePageModel ? false : undefined,
377
+ eagerInitialExtract: deferInlinePageModel ? true : undefined,
353
378
  });
354
379
  if (input.returnForms) {
355
380
  await stabilizeInlineFormSchemas(session, formSchema);
@@ -361,6 +386,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
361
386
  detail: input.detail,
362
387
  returnForms: input.returnForms,
363
388
  returnPageModel: input.returnPageModel,
389
+ pageModelMode: input.pageModelMode,
364
390
  formSchema,
365
391
  pageModelOptions,
366
392
  }), null, input.detail === 'verbose' ? 2 : undefined));
@@ -368,6 +394,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
368
394
  const session = await connect(target.wsUrl, {
369
395
  width: input.width,
370
396
  height: input.height,
397
+ awaitInitialFrame: deferInlinePageModel ? false : undefined,
371
398
  });
372
399
  if (input.returnForms) {
373
400
  await stabilizeInlineFormSchemas(session, formSchema);
@@ -379,6 +406,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
379
406
  detail: input.detail,
380
407
  returnForms: input.returnForms,
381
408
  returnPageModel: input.returnPageModel,
409
+ pageModelMode: input.pageModelMode,
382
410
  formSchema,
383
411
  pageModelOptions,
384
412
  }), null, input.detail === 'verbose' ? 2 : undefined));
@@ -387,6 +415,43 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
387
415
  return err(`Failed to connect: ${formatConnectFailureMessage(e, target)}`);
388
416
  }
389
417
  });
418
+ // ── prepare browser ──────────────────────────────────────────
419
+ server.tool('geometra_prepare_browser', `Pre-launch and pre-navigate a reusable geometra-proxy browser for a normal web page without creating an active MCP session.
420
+
421
+ Use this when you can prepare ahead of the user-facing task so the next \`geometra_connect\` or one-call \`geometra_run_actions\` on the same \`pageUrl\` / viewport / headless settings skips the cold browser launch.`, {
422
+ pageUrl: z
423
+ .string()
424
+ .url()
425
+ .refine(isHttpUrl, 'pageUrl must use http:// or https://')
426
+ .describe('HTTP(S) page to open and keep warm for the next proxy-backed task.'),
427
+ port: z
428
+ .number()
429
+ .int()
430
+ .positive()
431
+ .max(65535)
432
+ .optional()
433
+ .describe('Preferred local port for spawned proxy (default: ephemeral OS-assigned port).'),
434
+ headless: z
435
+ .boolean()
436
+ .optional()
437
+ .describe('Run Chromium headless (default false = visible window).'),
438
+ width: z.number().int().positive().optional().describe('Viewport width for the warmed browser.'),
439
+ height: z.number().int().positive().optional().describe('Viewport height for the warmed browser.'),
440
+ slowMo: z
441
+ .number()
442
+ .int()
443
+ .nonnegative()
444
+ .optional()
445
+ .describe('Playwright slowMo (ms) for the warmed browser.'),
446
+ }, async ({ pageUrl, port, headless, width, height, slowMo }) => {
447
+ try {
448
+ const prepared = await prewarmProxy({ pageUrl, port, headless, width, height, slowMo });
449
+ return ok(JSON.stringify(prepared));
450
+ }
451
+ catch (e) {
452
+ return err(`Failed to prepare browser: ${e instanceof Error ? e.message : String(e)}`);
453
+ }
454
+ });
390
455
  // ── query ────────────────────────────────────────────────────
391
456
  server.registerTool('geometra_query', {
392
457
  description: `Find elements in the current Geometra UI by stable id, role, name, text content, current value, or semantic state. Returns matching elements with their exact pixel bounds {x, y, width, height}, visible in-viewport bounds, an on-screen center point, visibility / scroll-reveal hints, role, name, value, state, and tree path.
@@ -395,11 +460,11 @@ This is the Geometra equivalent of Playwright's locator — but instant, structu
395
460
 
396
461
  Unknown parameter names are rejected (strict schema). To wait until visible text goes away (e.g. a parsing banner), use geometra_wait_for with that substring in text and present: false — there is no textGone field.`,
397
462
  inputSchema: geometraQueryInputSchema,
398
- }, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy }) => {
463
+ }, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, maxResults, detail }) => {
399
464
  const session = getSession();
400
- if (!session?.tree || !session?.layout)
465
+ if (!session)
401
466
  return err('Not connected. Call geometra_connect first.');
402
- const a11y = sessionA11y(session);
467
+ const a11y = await sessionA11yWhenReady(session);
403
468
  if (!a11y)
404
469
  return err('No UI tree available');
405
470
  const filter = {
@@ -408,6 +473,9 @@ Unknown parameter names are rejected (strict schema). To wait until visible text
408
473
  name,
409
474
  text,
410
475
  contextText,
476
+ promptText,
477
+ sectionText,
478
+ itemText,
411
479
  value,
412
480
  checked,
413
481
  disabled,
@@ -422,19 +490,70 @@ Unknown parameter names are rejected (strict schema). To wait until visible text
422
490
  return err(GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE);
423
491
  const matches = findNodes(a11y, filter);
424
492
  if (matches.length === 0) {
493
+ if (detail === 'terse') {
494
+ return ok(JSON.stringify({ matchCount: 0, filter: compactFilterPayload(filter) }));
495
+ }
425
496
  return ok(`No elements found matching ${JSON.stringify(filter)}`);
426
497
  }
427
- const result = sortA11yNodes(matches).map(node => formatNode(node, a11y, a11y.bounds));
498
+ const formatted = sortA11yNodes(matches).map(node => formatNode(node, a11y, a11y.bounds));
499
+ if (detail === 'terse') {
500
+ const limited = formatted.slice(0, maxResults ?? 8);
501
+ return ok(JSON.stringify({
502
+ matchCount: formatted.length,
503
+ matches: limited.map(compactFormattedNode),
504
+ }));
505
+ }
506
+ const result = typeof maxResults === 'number' ? formatted.slice(0, maxResults) : formatted;
428
507
  return ok(JSON.stringify(result, null, 2));
429
508
  });
509
+ server.tool('geometra_find_action', `Resolve a clickable action by action label plus optional section, prompt, or item/card text. This is a narrower, lower-token path for repeated actions like "Open incident" in a queue row or "Add to cart" inside a product card.
510
+
511
+ Use this when geometra_page_model tells you the page shape, but you want one direct semantic action target instead of expanding a whole section.`, {
512
+ name: z.string().describe('Action label / accessible name to match'),
513
+ role: z.enum(['button', 'link']).optional().describe('Optional action role hint (button or link)'),
514
+ sectionText: z.string().optional().describe('Containing section/landmark/form/dialog text to disambiguate repeated actions'),
515
+ promptText: z.string().optional().describe('Nearby question/prompt text to disambiguate repeated actions'),
516
+ itemText: z.string().optional().describe('Nearby card/row/item label to disambiguate repeated actions'),
517
+ maxResults: z.number().int().min(1).max(12).optional().default(6).describe('Maximum number of matches to return'),
518
+ detail: detailInput(),
519
+ }, async ({ name, role, sectionText, promptText, itemText, maxResults, detail }) => {
520
+ const session = getSession();
521
+ if (!session)
522
+ return err('Not connected. Call geometra_connect first.');
523
+ const a11y = await sessionA11yWhenReady(session);
524
+ if (!a11y)
525
+ return err('No UI tree available');
526
+ const filter = {
527
+ ...(role ? { role } : {}),
528
+ name,
529
+ ...(sectionText ? { sectionText } : {}),
530
+ ...(promptText ? { promptText } : {}),
531
+ ...(itemText ? { itemText } : {}),
532
+ };
533
+ const matches = sortA11yNodes(findNodes(a11y, filter).filter(node => node.focusable && (node.role === 'button' || node.role === 'link')));
534
+ if (matches.length === 0) {
535
+ if (detail === 'terse') {
536
+ return ok(JSON.stringify({ matchCount: 0, filter: compactFilterPayload(filter) }));
537
+ }
538
+ return ok(`No actions found matching ${JSON.stringify(filter)}`);
539
+ }
540
+ const formatted = matches.slice(0, maxResults).map(node => formatNode(node, a11y, a11y.bounds));
541
+ if (detail === 'terse') {
542
+ return ok(JSON.stringify({
543
+ matchCount: matches.length,
544
+ matches: formatted.map(compactFormattedNode),
545
+ }));
546
+ }
547
+ return ok(JSON.stringify(formatted));
548
+ });
430
549
  server.registerTool('geometra_wait_for', {
431
550
  description: `Wait for a semantic UI condition without guessing sleep durations. Use this for slow SPA transitions, resume parsing, custom validation alerts, disabled submit buttons, and value/state confirmation before submit.
432
551
 
433
552
  The filter matches the same fields as geometra_query (strict schema — unknown keys error). Set \`present: false\` to wait until **no** node matches — for example Ashby/Lever-style “Parsing your resume” or any “Parsing…” banner: \`{ "text": "Parsing", "present": false }\` (tune the substring to the site). Do not use a textGone parameter; use \`text\` + \`present: false\`, or \`geometra_wait_for_resume_parse\` for the usual post-upload parsing banner.`,
434
553
  inputSchema: geometraWaitForInputSchema,
435
- }, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs }) => {
554
+ }, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs, detail }) => {
436
555
  const session = getSession();
437
- if (!session?.tree || !session?.layout)
556
+ if (!session)
438
557
  return err('Not connected. Call geometra_connect first.');
439
558
  const filterProbe = {
440
559
  id,
@@ -442,6 +561,9 @@ The filter matches the same fields as geometra_query (strict schema — unknown
442
561
  name,
443
562
  text,
444
563
  contextText,
564
+ promptText,
565
+ sectionText,
566
+ itemText,
445
567
  value,
446
568
  checked,
447
569
  disabled,
@@ -461,6 +583,16 @@ The filter matches the same fields as geometra_query (strict schema — unknown
461
583
  });
462
584
  if (!waited.ok)
463
585
  return err(waited.error);
586
+ if (detail === 'terse') {
587
+ const compact = waitConditionCompact(waited.value);
588
+ const matches = waited.value.matches
589
+ .slice(0, 3)
590
+ .map(match => compactFormattedNode(match));
591
+ return ok(JSON.stringify({
592
+ ...compact,
593
+ ...(matches.length > 0 ? { matches } : {}),
594
+ }));
595
+ }
464
596
  if (!waited.value.present) {
465
597
  return ok(waitConditionSuccessLine(waited.value));
466
598
  }
@@ -473,7 +605,7 @@ Equivalent to \`geometra_wait_for\` with \`present: false\` and \`text\` set to
473
605
  inputSchema: geometraWaitForResumeParseInputSchema,
474
606
  }, async ({ text, timeoutMs }) => {
475
607
  const session = getSession();
476
- if (!session?.tree || !session?.layout)
608
+ if (!session)
477
609
  return err('Not connected. Call geometra_connect first.');
478
610
  const filter = { text };
479
611
  const waited = await waitForSemanticCondition(session, {
@@ -508,6 +640,30 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
508
640
  const resolvedFields = resolveFillFieldInputs(session, fields);
509
641
  if (!resolvedFields.ok)
510
642
  return err(resolvedFields.error);
643
+ if (!includeSteps) {
644
+ try {
645
+ const batched = await tryBatchedResolvedFields(session, resolvedFields.fields, detail);
646
+ if (batched.ok) {
647
+ const payload = {
648
+ completed: true,
649
+ execution: 'batched',
650
+ finalSource: batched.finalSource,
651
+ fieldCount: resolvedFields.fields.length,
652
+ successCount: resolvedFields.fields.length,
653
+ errorCount: 0,
654
+ final: batched.final,
655
+ };
656
+ if (failOnInvalid && batched.invalidRemaining > 0) {
657
+ return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
658
+ }
659
+ return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
660
+ }
661
+ }
662
+ catch (e) {
663
+ const message = e instanceof Error ? e.message : String(e);
664
+ return err(message);
665
+ }
666
+ }
511
667
  const steps = [];
512
668
  let stoppedAt;
513
669
  for (let index = 0; index < resolvedFields.fields.length; index++) {
@@ -570,8 +726,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
570
726
  .optional()
571
727
  .default(false)
572
728
  .describe('Include per-field step results in the JSON payload (default false for the smallest response)'),
729
+ resumeFromIndex: z
730
+ .number()
731
+ .int()
732
+ .min(0)
733
+ .optional()
734
+ .describe('Resume a partial fill from this field index (from a previous stoppedAt + 1). Skips already-filled fields.'),
573
735
  detail: detailInput(),
574
- }, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, detail }) => {
736
+ }, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, resumeFromIndex, detail }) => {
575
737
  const directFields = !includeSteps && !formId && Object.keys(valuesById ?? {}).length === 0
576
738
  ? directLabelBatchFields(valuesByLabel)
577
739
  : null;
@@ -720,7 +882,8 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
720
882
  }
721
883
  const steps = [];
722
884
  let stoppedAt;
723
- for (let index = 0; index < planned.fields.length; index++) {
885
+ const startIndex = resumeFromIndex ?? 0;
886
+ for (let index = startIndex; index < planned.fields.length; index++) {
724
887
  const field = planned.fields[index];
725
888
  try {
726
889
  const result = await executeFillField(session, field, detail);
@@ -744,15 +907,16 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
744
907
  const errorCount = steps.length - successCount;
745
908
  const payload = {
746
909
  ...connection,
747
- completed: stoppedAt === undefined && steps.length === planned.fields.length,
910
+ completed: stoppedAt === undefined && (startIndex + steps.length) === planned.fields.length,
748
911
  execution: 'sequential',
749
912
  formId: schema.formId,
750
913
  requestedValueCount: entryCount,
751
914
  fieldCount: planned.fields.length,
752
915
  successCount,
753
916
  errorCount,
917
+ ...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
754
918
  ...(includeSteps ? { steps } : {}),
755
- ...(stoppedAt !== undefined ? { stoppedAt } : {}),
919
+ ...(stoppedAt !== undefined ? { stoppedAt, resumeFromIndex: stoppedAt + 1 } : {}),
756
920
  ...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
757
921
  };
758
922
  if (failOnInvalid && invalidRemaining > 0) {
@@ -762,7 +926,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
762
926
  });
763
927
  server.tool('geometra_run_actions', `Execute several Geometra actions in one MCP round trip and return one consolidated result. This is the preferred path for long, multi-step form fills where one-tool-per-field would otherwise create too much chatter.
764
928
 
765
- Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_listbox_option\`, \`select_option\`, \`set_checked\`, \`wheel\`, \`wait_for\`, and \`fill_fields\`. \`click\` steps can also carry a nested \`waitFor\` condition.`, {
929
+ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_listbox_option\`, \`select_option\`, \`set_checked\`, \`wheel\`, \`wait_for\`, and \`fill_fields\`. \`click\` steps can also carry a nested \`waitFor\` condition. Pass \`pageUrl\` / \`url\` to auto-connect so an entire flow can run in one MCP call.`, {
930
+ url: z.string().optional().describe('Optional target URL. Use a ws:// Geometra server URL or an http(s) page URL to auto-connect before running actions.'),
931
+ pageUrl: z.string().optional().describe('Optional http(s) page URL to auto-connect before running actions. Prefer this over url for browser pages.'),
932
+ port: z.number().int().min(0).max(65535).optional().describe('Preferred local port for an auto-spawned proxy (default: ephemeral OS-assigned port).'),
933
+ headless: z.boolean().optional().describe('Run Chromium headless when auto-spawning a proxy (default false = visible window).'),
934
+ width: z.number().int().positive().optional().describe('Viewport width for auto-connected sessions.'),
935
+ height: z.number().int().positive().optional().describe('Viewport height for auto-connected sessions.'),
936
+ slowMo: z.number().int().nonnegative().optional().describe('Playwright slowMo (ms) when auto-spawning a proxy.'),
766
937
  actions: z.array(batchActionSchema).min(1).max(80).describe('Ordered high-level action steps to run sequentially'),
767
938
  stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing step (default true)'),
768
939
  includeSteps: z
@@ -770,24 +941,81 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
770
941
  .optional()
771
942
  .default(true)
772
943
  .describe('Include per-action step results in the JSON payload (default true). Set false for the smallest batch response.'),
944
+ output: z.enum(['full', 'final']).optional().default('full').describe('`full` (default) returns counts and optional step listings. `final` keeps only completion state plus final semantic signals.'),
773
945
  detail: detailInput(),
774
- }, async ({ actions, stopOnError, includeSteps, detail }) => {
775
- const session = getSession();
776
- if (!session)
777
- return err('Not connected. Call geometra_connect first.');
946
+ }, async ({ url, pageUrl, port, headless, width, height, slowMo, actions, stopOnError, includeSteps, output, detail }) => {
947
+ const resolved = await ensureToolSession({
948
+ url,
949
+ pageUrl,
950
+ port,
951
+ headless,
952
+ width,
953
+ height,
954
+ slowMo,
955
+ awaitInitialFrame: canDeferInitialFrameForRunActions(actions) ? false : undefined,
956
+ }, 'Not connected. Call geometra_connect first, or pass pageUrl/url to geometra_run_actions.');
957
+ if (!resolved.ok)
958
+ return err(resolved.error);
959
+ const session = resolved.session;
960
+ const connection = autoConnectionPayload(resolved);
778
961
  const steps = [];
779
962
  let stoppedAt;
963
+ const batchStartedAt = performance.now();
780
964
  for (let index = 0; index < actions.length; index++) {
781
965
  const action = actions[index];
966
+ const startedAt = performance.now();
967
+ let uiTreeWaitMs = 0;
782
968
  try {
969
+ if (actionNeedsUiTree(action) && (!session.tree || !session.layout)) {
970
+ const uiTreeWaitStartedAt = performance.now();
971
+ await waitForUiCondition(session, () => Boolean(session.tree && session.layout), 2_000);
972
+ uiTreeWaitMs = performance.now() - uiTreeWaitStartedAt;
973
+ }
783
974
  const result = await executeBatchAction(session, action, detail, includeSteps);
975
+ const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
976
+ const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
977
+ const stepSignals = includeSteps ? (() => {
978
+ const a = sessionA11y(session);
979
+ if (!a)
980
+ return undefined;
981
+ const s = collectSessionSignals(a);
982
+ return { invalidCount: s.invalidFields.length, alertCount: s.alerts.length, dialogCount: s.dialogCount, busyCount: s.busyCount };
983
+ })() : undefined;
784
984
  steps.push(detail === 'verbose'
785
- ? { index, type: action.type, ok: true, summary: result.summary }
786
- : { index, type: action.type, ok: true, ...result.compact });
985
+ ? {
986
+ index,
987
+ type: action.type,
988
+ ok: true,
989
+ elapsedMs,
990
+ cumulativeMs,
991
+ ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
992
+ summary: result.summary,
993
+ ...(stepSignals ? { signals: stepSignals } : {}),
994
+ }
995
+ : {
996
+ index,
997
+ type: action.type,
998
+ ok: true,
999
+ elapsedMs,
1000
+ cumulativeMs,
1001
+ ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
1002
+ ...result.compact,
1003
+ ...(stepSignals ? { signals: stepSignals } : {}),
1004
+ });
787
1005
  }
788
1006
  catch (e) {
789
1007
  const message = e instanceof Error ? e.message : String(e);
790
- steps.push({ index, type: action.type, ok: false, error: message });
1008
+ const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
1009
+ const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
1010
+ steps.push({
1011
+ index,
1012
+ type: action.type,
1013
+ ok: false,
1014
+ elapsedMs,
1015
+ cumulativeMs,
1016
+ ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
1017
+ error: message,
1018
+ });
791
1019
  if (stopOnError) {
792
1020
  stoppedAt = index;
793
1021
  break;
@@ -797,15 +1025,23 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
797
1025
  const after = sessionA11y(session);
798
1026
  const successCount = steps.filter(step => step.ok === true).length;
799
1027
  const errorCount = steps.length - successCount;
800
- const payload = {
801
- completed: stoppedAt === undefined && steps.length === actions.length,
802
- stepCount: actions.length,
803
- successCount,
804
- errorCount,
805
- ...(includeSteps ? { steps } : {}),
806
- ...(stoppedAt !== undefined ? { stoppedAt } : {}),
807
- ...(after ? { final: sessionSignalsPayload(collectSessionSignals(after), detail) } : {}),
808
- };
1028
+ const payload = output === 'final'
1029
+ ? {
1030
+ ...connection,
1031
+ completed: stoppedAt === undefined && steps.length === actions.length,
1032
+ ...(stoppedAt !== undefined ? { stoppedAt } : {}),
1033
+ ...(after ? { final: sessionSignalsPayload(collectSessionSignals(after), detail) } : {}),
1034
+ }
1035
+ : {
1036
+ ...connection,
1037
+ completed: stoppedAt === undefined && steps.length === actions.length,
1038
+ stepCount: actions.length,
1039
+ successCount,
1040
+ errorCount,
1041
+ ...(includeSteps ? { steps } : {}),
1042
+ ...(stoppedAt !== undefined ? { stoppedAt } : {}),
1043
+ ...(after ? { final: sessionSignalsPayload(collectSessionSignals(after), detail) } : {}),
1044
+ };
809
1045
  return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
810
1046
  });
811
1047
  // ── page model ────────────────────────────────────────────────
@@ -828,15 +1064,21 @@ Use this first on normal HTML pages when you want to understand the page shape w
828
1064
  .optional()
829
1065
  .default(8)
830
1066
  .describe('Cap returned landmarks/forms/dialogs/lists per kind (default 8).'),
831
- }, async ({ maxPrimaryActions, maxSectionsPerKind }) => {
1067
+ includeScreenshot: z
1068
+ .boolean()
1069
+ .optional()
1070
+ .default(false)
1071
+ .describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy. Use when geometry alone is ambiguous (icon-only buttons, visual styling cues).'),
1072
+ }, async ({ maxPrimaryActions, maxSectionsPerKind, includeScreenshot }) => {
832
1073
  const session = getSession();
833
- if (!session?.tree || !session?.layout)
1074
+ if (!session)
834
1075
  return err('Not connected. Call geometra_connect first.');
835
- const a11y = sessionA11y(session);
1076
+ const a11y = await sessionA11yWhenReady(session);
836
1077
  if (!a11y)
837
1078
  return err('No UI tree available');
838
1079
  const model = buildPageModel(a11y, { maxPrimaryActions, maxSectionsPerKind });
839
- return ok(JSON.stringify(model));
1080
+ const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
1081
+ return ok(JSON.stringify(model), screenshot);
840
1082
  });
841
1083
  server.tool('geometra_form_schema', `Get a compact, fill-oriented schema for forms on the page. This is the preferred discovery step before geometra_fill_form.
842
1084
 
@@ -861,6 +1103,9 @@ Unlike geometra_expand_section, this collapses repeated radio/button groups into
861
1103
  if (!resolved.ok)
862
1104
  return err(resolved.error);
863
1105
  const session = resolved.session;
1106
+ if (!(await ensureSessionUiTree(session, 4_000))) {
1107
+ return err('Timed out waiting for the initial UI tree after connect.');
1108
+ }
864
1109
  const payload = formSchemaResponsePayload(session, {
865
1110
  formId,
866
1111
  maxFields,
@@ -898,9 +1143,9 @@ Use this after geometra_page_model when you know which form/dialog/list/landmark
898
1143
  includeBounds: z.boolean().optional().default(false).describe('Include bounds for fields/actions/headings/items'),
899
1144
  }, async ({ id, maxHeadings, maxFields, fieldOffset, onlyRequiredFields, onlyInvalidFields, maxActions, actionOffset, maxLists, listOffset, maxItems, itemOffset, maxTextPreview, includeBounds, }) => {
900
1145
  const session = getSession();
901
- if (!session?.tree || !session?.layout)
1146
+ if (!session)
902
1147
  return err('Not connected. Call geometra_connect first.');
903
- const a11y = sessionA11y(session);
1148
+ const a11y = await sessionA11yWhenReady(session);
904
1149
  if (!a11y)
905
1150
  return err('No UI tree available');
906
1151
  const detail = expandPageSection(a11y, id, {
@@ -937,7 +1182,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
937
1182
  .optional()
938
1183
  .default(2_500)
939
1184
  .describe('Per-scroll wait timeout (default 2500ms)'),
940
- }, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxSteps, timeoutMs }) => {
1185
+ }, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxSteps, timeoutMs }) => {
941
1186
  const session = getSession();
942
1187
  if (!session)
943
1188
  return err('Not connected. Call geometra_connect first.');
@@ -947,6 +1192,9 @@ Use the same filters as geometra_query, plus an optional match index when repeat
947
1192
  name,
948
1193
  text,
949
1194
  contextText,
1195
+ promptText,
1196
+ sectionText,
1197
+ itemText,
950
1198
  value,
951
1199
  checked,
952
1200
  disabled,
@@ -958,7 +1206,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
958
1206
  busy,
959
1207
  };
960
1208
  if (!hasNodeFilter(filter))
961
- return err('Provide at least one reveal filter (id, role, name, text, contextText, value, or state)');
1209
+ return err('Provide at least one reveal filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, or state)');
962
1210
  const revealed = await revealSemanticTarget(session, {
963
1211
  filter,
964
1212
  index: index ?? 0,
@@ -1001,7 +1249,7 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
1001
1249
  .optional()
1002
1250
  .describe('Optional action wait timeout (use a longer value for slow submits or route transitions)'),
1003
1251
  detail: detailInput(),
1004
- }, async ({ x, y, id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxRevealSteps, revealTimeoutMs, waitFor, timeoutMs, detail }) => {
1252
+ }, async ({ x, y, id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxRevealSteps, revealTimeoutMs, waitFor, timeoutMs, detail }) => {
1005
1253
  const session = getSession();
1006
1254
  if (!session)
1007
1255
  return err('Not connected. Call geometra_connect first.');
@@ -1015,6 +1263,9 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
1015
1263
  name,
1016
1264
  text,
1017
1265
  contextText,
1266
+ promptText,
1267
+ sectionText,
1268
+ itemText,
1018
1269
  value,
1019
1270
  checked,
1020
1271
  disabled,
@@ -1046,6 +1297,9 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
1046
1297
  name: waitFor.name,
1047
1298
  text: waitFor.text,
1048
1299
  contextText: waitFor.contextText,
1300
+ promptText: waitFor.promptText,
1301
+ sectionText: waitFor.sectionText,
1302
+ itemText: waitFor.itemText,
1049
1303
  value: waitFor.value,
1050
1304
  checked: waitFor.checked,
1051
1305
  disabled: waitFor.disabled,
@@ -1062,8 +1316,20 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
1062
1316
  if (!postWait.ok)
1063
1317
  return err([...lines, postWait.error].join('\n'));
1064
1318
  lines.push(`Post-click ${waitConditionSuccessLine(postWait.value)}`);
1319
+ const compact = {
1320
+ at: { x: resolved.value.x, y: resolved.value.y },
1321
+ ...(resolved.value.target ? { target: compactNodeReference(resolved.value.target), revealSteps: resolved.value.revealAttempts ?? 0 } : {}),
1322
+ ...waitStatusPayload(wait),
1323
+ postWait: waitConditionCompact(postWait.value),
1324
+ };
1325
+ return ok(detailText(lines.filter(Boolean).join('\n'), compact, detail));
1065
1326
  }
1066
- return ok(lines.filter(Boolean).join('\n'));
1327
+ const compact = {
1328
+ at: { x: resolved.value.x, y: resolved.value.y },
1329
+ ...(resolved.value.target ? { target: compactNodeReference(resolved.value.target), revealSteps: resolved.value.revealAttempts ?? 0 } : {}),
1330
+ ...waitStatusPayload(wait),
1331
+ };
1332
+ return ok(detailText(lines.filter(Boolean).join('\n'), compact, detail));
1067
1333
  });
1068
1334
  // ── type ─────────────────────────────────────────────────────
1069
1335
  server.tool('geometra_type', `Type text into the currently focused element. First click a textbox/input with geometra_click to focus it, then use this to type.
@@ -1085,7 +1351,10 @@ Each character is sent as a key event through the geometry protocol. Returns a c
1085
1351
  const before = sessionA11y(session);
1086
1352
  const wait = await sendType(session, text, timeoutMs);
1087
1353
  const summary = postActionSummary(session, before, wait, detail);
1088
- return ok(`Typed "${text}".\n${summary}`);
1354
+ return ok(detailText(`Typed "${text}".\n${summary}`, {
1355
+ ...compactTextValue(text),
1356
+ ...waitStatusPayload(wait),
1357
+ }, detail));
1089
1358
  });
1090
1359
  // ── key ──────────────────────────────────────────────────────
1091
1360
  server.tool('geometra_key', `Send a special key press (Enter, Tab, Escape, ArrowDown, etc.) to the Geometra UI. Useful for form submission, focus navigation, and keyboard shortcuts.`, {
@@ -1109,7 +1378,10 @@ Each character is sent as a key event through the geometry protocol. Returns a c
1109
1378
  const before = sessionA11y(session);
1110
1379
  const wait = await sendKey(session, key, { shift, ctrl, meta, alt }, timeoutMs);
1111
1380
  const summary = postActionSummary(session, before, wait, detail);
1112
- return ok(`Pressed ${formatKeyCombo(key, { shift, ctrl, meta, alt })}.\n${summary}`);
1381
+ return ok(detailText(`Pressed ${formatKeyCombo(key, { shift, ctrl, meta, alt })}.\n${summary}`, {
1382
+ key: formatKeyCombo(key, { shift, ctrl, meta, alt }),
1383
+ ...waitStatusPayload(wait),
1384
+ }, detail));
1113
1385
  });
1114
1386
  // ── upload files (proxy) ───────────────────────────────────────
1115
1387
  server.tool('geometra_upload_files', `Attach local files to a file input. Requires \`@geometra/proxy\` (paths exist on the proxy host).
@@ -1126,6 +1398,8 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
1126
1398
  .describe('Upload strategy (default auto)'),
1127
1399
  dropX: z.number().optional().describe('Drop target X (viewport) for strategy drop'),
1128
1400
  dropY: z.number().optional().describe('Drop target Y (viewport) for strategy drop'),
1401
+ contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated file inputs'),
1402
+ sectionText: z.string().optional().describe('Containing section text to disambiguate repeated file inputs'),
1129
1403
  timeoutMs: z
1130
1404
  .number()
1131
1405
  .int()
@@ -1134,7 +1408,7 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
1134
1408
  .optional()
1135
1409
  .describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
1136
1410
  detail: detailInput(),
1137
- }, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, timeoutMs, detail }) => {
1411
+ }, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
1138
1412
  const session = getSession();
1139
1413
  if (!session)
1140
1414
  return err('Not connected. Call geometra_connect first.');
@@ -1148,7 +1422,13 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
1148
1422
  drop: dropX !== undefined && dropY !== undefined ? { x: dropX, y: dropY } : undefined,
1149
1423
  }, timeoutMs ?? 8_000);
1150
1424
  const summary = postActionSummary(session, before, wait, detail);
1151
- return ok(`Uploaded ${paths.length} file(s).\n${summary}`);
1425
+ return ok(detailText(`Uploaded ${paths.length} file(s).\n${summary}`, {
1426
+ fileCount: paths.length,
1427
+ ...(fieldLabel ? { fieldLabel } : {}),
1428
+ ...(strategy ? { strategy } : {}),
1429
+ ...waitStatusPayload(wait),
1430
+ ...(fieldLabel ? { readback: fieldStatePayload(session, fieldLabel) } : {}),
1431
+ }, detail));
1152
1432
  }
1153
1433
  catch (e) {
1154
1434
  return err(e.message);
@@ -1162,6 +1442,8 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
1162
1442
  openX: z.number().optional().describe('Click to open dropdown'),
1163
1443
  openY: z.number().optional().describe('Click to open dropdown'),
1164
1444
  fieldLabel: z.string().optional().describe('Field label of the dropdown/combobox to open semantically (e.g. "Location")'),
1445
+ contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated dropdowns with the same label'),
1446
+ sectionText: z.string().optional().describe('Containing section text to disambiguate repeated dropdowns'),
1165
1447
  query: z.string().optional().describe('Optional text to type into a searchable combobox before selecting'),
1166
1448
  timeoutMs: z
1167
1449
  .number()
@@ -1171,7 +1453,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
1171
1453
  .optional()
1172
1454
  .describe('Optional action wait timeout for slow dropdowns / remote search results'),
1173
1455
  detail: detailInput(),
1174
- }, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs, detail }) => {
1456
+ }, async ({ label, exact, openX, openY, fieldLabel, contextText, sectionText, query, timeoutMs, detail }) => {
1175
1457
  const session = getSession();
1176
1458
  if (!session)
1177
1459
  return err('Not connected. Call geometra_connect first.');
@@ -1185,11 +1467,17 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
1185
1467
  }, timeoutMs);
1186
1468
  const summary = postActionSummary(session, before, wait, detail);
1187
1469
  const fieldSummary = fieldLabel ? summarizeFieldLabelState(session, fieldLabel) : undefined;
1188
- return ok([
1470
+ const summaryText = [
1189
1471
  `Picked listbox option "${label}".`,
1190
1472
  fieldSummary,
1191
1473
  summary,
1192
- ].filter(Boolean).join('\n'));
1474
+ ].filter(Boolean).join('\n');
1475
+ return ok(detailText(summaryText, {
1476
+ label,
1477
+ ...(fieldLabel ? { fieldLabel } : {}),
1478
+ ...waitStatusPayload(wait),
1479
+ ...(fieldLabel ? { readback: fieldStatePayload(session, fieldLabel) } : {}),
1480
+ }, detail));
1193
1481
  }
1194
1482
  catch (e) {
1195
1483
  return err(e.message);
@@ -1204,6 +1492,8 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
1204
1492
  value: z.string().optional().describe('Option value= attribute'),
1205
1493
  label: z.string().optional().describe('Visible option label (substring match)'),
1206
1494
  index: z.number().int().min(0).optional().describe('Zero-based option index'),
1495
+ contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated selects'),
1496
+ sectionText: z.string().optional().describe('Containing section text to disambiguate repeated selects'),
1207
1497
  timeoutMs: z
1208
1498
  .number()
1209
1499
  .int()
@@ -1212,7 +1502,7 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
1212
1502
  .optional()
1213
1503
  .describe('Optional action wait timeout'),
1214
1504
  detail: detailInput(),
1215
- }, async ({ x, y, value, label, index, timeoutMs, detail }) => {
1505
+ }, async ({ x, y, value, label, index, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
1216
1506
  const session = getSession();
1217
1507
  if (!session)
1218
1508
  return err('Not connected. Call geometra_connect first.');
@@ -1223,7 +1513,13 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
1223
1513
  try {
1224
1514
  const wait = await sendSelectOption(session, x, y, { value, label, index }, timeoutMs);
1225
1515
  const summary = postActionSummary(session, before, wait, detail);
1226
- return ok(`Selected option.\n${summary}`);
1516
+ return ok(detailText(`Selected option.\n${summary}`, {
1517
+ at: { x, y },
1518
+ ...(value !== undefined ? { value } : {}),
1519
+ ...(label !== undefined ? { label } : {}),
1520
+ ...(index !== undefined ? { index } : {}),
1521
+ ...waitStatusPayload(wait),
1522
+ }, detail));
1227
1523
  }
1228
1524
  catch (e) {
1229
1525
  return err(e.message);
@@ -1236,6 +1532,8 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
1236
1532
  checked: z.boolean().optional().default(true).describe('Desired checked state (radios only support true)'),
1237
1533
  exact: z.boolean().optional().describe('Exact label match'),
1238
1534
  controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
1535
+ contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated checkboxes/radios'),
1536
+ sectionText: z.string().optional().describe('Containing section text to disambiguate repeated checkboxes/radios'),
1239
1537
  timeoutMs: z
1240
1538
  .number()
1241
1539
  .int()
@@ -1244,7 +1542,7 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
1244
1542
  .optional()
1245
1543
  .describe('Optional action wait timeout'),
1246
1544
  detail: detailInput(),
1247
- }, async ({ label, checked, exact, controlType, timeoutMs, detail }) => {
1545
+ }, async ({ label, checked, exact, controlType, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
1248
1546
  const session = getSession();
1249
1547
  if (!session)
1250
1548
  return err('Not connected. Call geometra_connect first.');
@@ -1252,7 +1550,12 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
1252
1550
  try {
1253
1551
  const wait = await sendSetChecked(session, label, { checked, exact, controlType }, timeoutMs);
1254
1552
  const summary = postActionSummary(session, before, wait, detail);
1255
- return ok(`Set ${controlType ?? 'checkbox/radio'} "${label}" to ${String(checked ?? true)}.\n${summary}`);
1553
+ return ok(detailText(`Set ${controlType ?? 'checkbox/radio'} "${label}" to ${String(checked ?? true)}.\n${summary}`, {
1554
+ label,
1555
+ checked: checked ?? true,
1556
+ ...(controlType ? { controlType } : {}),
1557
+ ...waitStatusPayload(wait),
1558
+ }, detail));
1256
1559
  }
1257
1560
  catch (e) {
1258
1561
  return err(e.message);
@@ -1280,12 +1583,68 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
1280
1583
  try {
1281
1584
  const wait = await sendWheel(session, deltaY, { deltaX, x, y }, timeoutMs);
1282
1585
  const summary = postActionSummary(session, before, wait, detail);
1283
- return ok(`Wheel delta (${deltaX ?? 0}, ${deltaY}).\n${summary}`);
1586
+ return ok(detailText(`Wheel delta (${deltaX ?? 0}, ${deltaY}).\n${summary}`, {
1587
+ deltaY,
1588
+ ...(deltaX !== undefined ? { deltaX } : {}),
1589
+ ...(x !== undefined && y !== undefined ? { at: { x, y } } : {}),
1590
+ ...waitStatusPayload(wait),
1591
+ }, detail));
1284
1592
  }
1285
1593
  catch (e) {
1286
1594
  return err(e.message);
1287
1595
  }
1288
1596
  });
1597
+ // ── list items (virtualized list pagination) ─────────────────
1598
+ server.tool('geometra_list_items', `Auto-scroll a virtualized or long list and collect all visible items across scroll positions. Requires \`@geometra/proxy\`.
1599
+
1600
+ Use this for dropdowns, location pickers, or any scrollable list where items are rendered on demand. Scrolls down in steps, collecting new items each time, until no new items appear or the cap is reached.`, {
1601
+ listId: z.string().optional().describe('Stable section id from geometra_page_model (e.g. ls:2.1) to scope item collection'),
1602
+ role: z.string().optional().describe('Role filter for list items (default: listitem)'),
1603
+ scrollX: z.number().optional().describe('X coordinate to position mouse for scrolling (default: viewport center)'),
1604
+ scrollY: z.number().optional().describe('Y coordinate to position mouse for scrolling (default: viewport center)'),
1605
+ maxItems: z.number().int().min(1).max(500).optional().default(100).describe('Cap collected items (default 100)'),
1606
+ maxScrollSteps: z.number().int().min(1).max(50).optional().default(20).describe('Max scroll steps before stopping (default 20)'),
1607
+ scrollDelta: z.number().optional().default(300).describe('Vertical scroll delta per step (default 300)'),
1608
+ }, async ({ listId: _listId, role, scrollX, scrollY, maxItems, maxScrollSteps, scrollDelta }) => {
1609
+ const session = getSession();
1610
+ if (!session)
1611
+ return err('Not connected. Call geometra_connect first.');
1612
+ const itemRole = role ?? 'listitem';
1613
+ const collected = new Map();
1614
+ const cx = scrollX ?? 400;
1615
+ const cy = scrollY ?? 400;
1616
+ for (let step = 0; step < maxScrollSteps; step++) {
1617
+ const a11y = await sessionA11yWhenReady(session);
1618
+ if (!a11y)
1619
+ break;
1620
+ const items = findNodes(a11y, { role: itemRole });
1621
+ let newCount = 0;
1622
+ for (const item of items) {
1623
+ const id = nodeIdForPath(item.path);
1624
+ if (!collected.has(id)) {
1625
+ collected.set(id, {
1626
+ ...(item.name ? { name: item.name } : {}),
1627
+ ...(item.value ? { value: item.value } : {}),
1628
+ });
1629
+ newCount++;
1630
+ }
1631
+ }
1632
+ if (collected.size >= maxItems || newCount === 0)
1633
+ break;
1634
+ try {
1635
+ await sendWheel(session, scrollDelta, { x: cx, y: cy }, 1_000);
1636
+ }
1637
+ catch {
1638
+ break;
1639
+ }
1640
+ }
1641
+ const items = [...collected.entries()].slice(0, maxItems).map(([id, data]) => ({ id, ...data }));
1642
+ return ok(JSON.stringify({
1643
+ itemCount: items.length,
1644
+ items,
1645
+ truncated: collected.size > maxItems,
1646
+ }));
1647
+ });
1289
1648
  // ── snapshot ─────────────────────────────────────────────────
1290
1649
  server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout. Use **form-required** to list required fields across forms, including offscreen ones, with bounds + visibility + scroll hints for long application flows.
1291
1650
 
@@ -1306,15 +1665,21 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
1306
1665
  formId: z.string().optional().describe('Optional form id from geometra_form_schema / geometra_page_model when view=form-required'),
1307
1666
  maxFields: z.number().int().min(1).max(200).optional().default(80).describe('Per-form field cap when view=form-required'),
1308
1667
  includeOptions: z.boolean().optional().default(false).describe('Include explicit choice option labels when view=form-required'),
1309
- }, async ({ view, maxNodes, formId, maxFields, includeOptions }) => {
1668
+ includeScreenshot: z
1669
+ .boolean()
1670
+ .optional()
1671
+ .default(false)
1672
+ .describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy.'),
1673
+ }, async ({ view, maxNodes, formId, maxFields, includeOptions, includeScreenshot }) => {
1310
1674
  const session = getSession();
1311
- if (!session?.tree || !session?.layout)
1675
+ if (!session)
1312
1676
  return err('Not connected. Call geometra_connect first.');
1313
- const a11y = sessionA11y(session);
1677
+ const a11y = await sessionA11yWhenReady(session);
1314
1678
  if (!a11y)
1315
1679
  return err('No UI tree available');
1680
+ const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
1316
1681
  if (view === 'full') {
1317
- return ok(JSON.stringify(a11y, null, 2));
1682
+ return ok(JSON.stringify(a11y, null, 2), screenshot);
1318
1683
  }
1319
1684
  if (view === 'form-required') {
1320
1685
  const payload = {
@@ -1327,7 +1692,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
1327
1692
  includeContext: 'auto',
1328
1693
  }),
1329
1694
  };
1330
- return ok(JSON.stringify(payload));
1695
+ return ok(JSON.stringify(payload), screenshot);
1331
1696
  }
1332
1697
  const { nodes, truncated, context } = buildCompactUiIndex(a11y, { maxNodes });
1333
1698
  const payload = {
@@ -1337,7 +1702,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
1337
1702
  nodes,
1338
1703
  truncated,
1339
1704
  };
1340
- return ok(JSON.stringify(payload));
1705
+ return ok(JSON.stringify(payload), screenshot);
1341
1706
  });
1342
1707
  // ── layout ───────────────────────────────────────────────────
1343
1708
  server.tool('geometra_layout', `Get the raw computed layout geometry — the exact {x, y, width, height} for every node in the UI tree. This is the lowest-level view, useful for pixel-precise assertions in tests.
@@ -1365,7 +1730,7 @@ function compactSessionSummary(session) {
1365
1730
  return sessionOverviewFromA11y(a11y);
1366
1731
  }
1367
1732
  function connectPayload(session, opts) {
1368
- const a11y = sessionA11y(session);
1733
+ const a11y = opts.detail === 'verbose' ? sessionA11y(session) : null;
1369
1734
  return {
1370
1735
  connected: true,
1371
1736
  transport: opts.transport,
@@ -1387,6 +1752,17 @@ function sessionA11y(session) {
1387
1752
  session.cachedA11yRevision = session.updateRevision;
1388
1753
  return a11y;
1389
1754
  }
1755
+ async function ensureSessionUiTree(session, timeoutMs = 4_000) {
1756
+ if (session.tree && session.layout)
1757
+ return true;
1758
+ return await waitForUiCondition(session, () => Boolean(session.tree && session.layout), timeoutMs);
1759
+ }
1760
+ async function sessionA11yWhenReady(session, timeoutMs = 4_000) {
1761
+ const ready = await ensureSessionUiTree(session, timeoutMs);
1762
+ if (!ready)
1763
+ return null;
1764
+ return sessionA11y(session);
1765
+ }
1390
1766
  function shortHash(value) {
1391
1767
  return createHash('sha1').update(value).digest('hex').slice(0, 12);
1392
1768
  }
@@ -1438,8 +1814,10 @@ function packedFormSchemas(forms) {
1438
1814
  ...(field.valueLength !== undefined ? { vl: field.valueLength } : {}),
1439
1815
  ...(field.checked !== undefined ? { c: field.checked ? 1 : 0 } : {}),
1440
1816
  ...(field.values && field.values.length > 0 ? { vs: field.values } : {}),
1817
+ ...(field.aliases ? { al: field.aliases } : {}),
1441
1818
  ...(field.context ? { x: field.context } : {}),
1442
1819
  })),
1820
+ ...(form.sections ? { s: form.sections.map(s => ({ n: s.name, fi: s.fieldIds })) } : {}),
1443
1821
  }));
1444
1822
  }
1445
1823
  function formSchemaResponsePayload(session, opts) {
@@ -1506,10 +1884,23 @@ function connectResponsePayload(session, opts) {
1506
1884
  nextPayload.formSchema = formSchemaResponsePayload(session, opts.formSchema ?? {});
1507
1885
  }
1508
1886
  if (opts.returnPageModel) {
1509
- nextPayload.pageModel = pageModelResponsePayload(session, opts.pageModelOptions);
1887
+ nextPayload.pageModel = opts.pageModelMode === 'deferred'
1888
+ ? deferredPageModelConnectPayload(session, opts.pageModelOptions)
1889
+ : pageModelResponsePayload(session, opts.pageModelOptions);
1510
1890
  }
1511
1891
  return nextPayload;
1512
1892
  }
1893
+ function deferredPageModelConnectPayload(session, options) {
1894
+ return {
1895
+ deferred: true,
1896
+ ready: Boolean(session.tree && session.layout),
1897
+ tool: 'geometra_page_model',
1898
+ options: {
1899
+ maxPrimaryActions: options?.maxPrimaryActions ?? 6,
1900
+ maxSectionsPerKind: options?.maxSectionsPerKind ?? 8,
1901
+ },
1902
+ };
1903
+ }
1513
1904
  function pageModelResponsePayload(session, options) {
1514
1905
  const a11y = sessionA11y(session);
1515
1906
  if (!a11y) {
@@ -1737,8 +2128,15 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
1737
2128
  busyCount: signals.busyCount,
1738
2129
  alertCount: signals.alerts.length,
1739
2130
  invalidCount: signals.invalidFields.length,
1740
- alerts: detail === 'verbose' ? signals.alerts : signals.alerts.slice(0, 2),
1741
- invalidFields: detail === 'verbose' ? signals.invalidFields : signals.invalidFields.slice(0, 4),
2131
+ ...(detail === 'verbose'
2132
+ ? {
2133
+ alerts: signals.alerts,
2134
+ invalidFields: signals.invalidFields,
2135
+ }
2136
+ : {
2137
+ alerts: signals.alerts.slice(0, 2),
2138
+ invalidFields: signals.invalidFields.slice(0, 6),
2139
+ }),
1742
2140
  };
1743
2141
  }
1744
2142
  function compactTextValue(value, inlineLimit = 48) {
@@ -1834,6 +2232,10 @@ function inferRevealStepBudget(target, viewport) {
1834
2232
  return clamp(Math.max(6, Math.max(verticalSteps, horizontalSteps) + 1), 6, 48);
1835
2233
  }
1836
2234
  async function revealSemanticTarget(session, options) {
2235
+ const initialTreeReady = await ensureSessionUiTree(session, Math.max(4_000, options.timeoutMs));
2236
+ if (!initialTreeReady) {
2237
+ return { ok: false, error: 'Timed out waiting for the initial UI tree after connect.' };
2238
+ }
1837
2239
  let attempts = 0;
1838
2240
  let stepBudget = options.maxSteps;
1839
2241
  while (attempts <= (stepBudget ?? 48)) {
@@ -1904,7 +2306,7 @@ async function resolveClickLocation(session, options) {
1904
2306
  if (!hasNodeFilter(options.filter)) {
1905
2307
  return {
1906
2308
  ok: false,
1907
- error: 'Provide x and y, or at least one semantic target filter (id, role, name, text, contextText, value, or state)',
2309
+ error: 'Provide x and y, or at least one semantic target filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, or state)',
1908
2310
  };
1909
2311
  }
1910
2312
  const revealed = await revealSemanticTarget(session, {
@@ -1936,6 +2338,18 @@ function compactNodeReference(node) {
1936
2338
  ...(node.name ? { name: node.name } : {}),
1937
2339
  };
1938
2340
  }
2341
+ function compactFormattedNode(node) {
2342
+ return {
2343
+ ...compactNodeReference(node),
2344
+ ...(node.context ? { context: node.context } : {}),
2345
+ ...(node.value ? { value: node.value } : {}),
2346
+ center: node.center,
2347
+ bounds: node.bounds,
2348
+ };
2349
+ }
2350
+ function detailText(summary, compact, detail) {
2351
+ return detail === 'terse' ? JSON.stringify(compact) : summary;
2352
+ }
1939
2353
  function normalizeLookupKey(value) {
1940
2354
  return value.replace(/\s+/g, ' ').trim().toLowerCase();
1941
2355
  }
@@ -2182,12 +2596,14 @@ function parseProxyFillAckResult(value) {
2182
2596
  typeof candidate.busyCount !== 'number') {
2183
2597
  return undefined;
2184
2598
  }
2599
+ const invalidFields = Array.isArray(candidate.invalidFields) ? candidate.invalidFields : undefined;
2185
2600
  return {
2186
2601
  ...(typeof candidate.pageUrl === 'string' ? { pageUrl: candidate.pageUrl } : {}),
2187
2602
  invalidCount: candidate.invalidCount,
2188
2603
  alertCount: candidate.alertCount,
2189
2604
  dialogCount: candidate.dialogCount,
2190
2605
  busyCount: candidate.busyCount,
2606
+ ...(invalidFields && invalidFields.length > 0 ? { invalidFields } : {}),
2191
2607
  };
2192
2608
  }
2193
2609
  function directLabelBatchFields(valuesByLabel) {
@@ -2202,6 +2618,44 @@ function directLabelBatchFields(valuesByLabel) {
2202
2618
  }
2203
2619
  return fields;
2204
2620
  }
2621
+ async function tryBatchedResolvedFields(session, fields, detail) {
2622
+ let batchAckResult;
2623
+ try {
2624
+ const startRevision = session.updateRevision;
2625
+ const wait = await sendFillFields(session, fields);
2626
+ const ackResult = parseProxyFillAckResult(wait.result);
2627
+ batchAckResult = ackResult;
2628
+ if (ackResult && ackResult.invalidCount === 0) {
2629
+ return {
2630
+ ok: true,
2631
+ finalSource: 'proxy',
2632
+ final: ackResult,
2633
+ invalidRemaining: 0,
2634
+ };
2635
+ }
2636
+ await waitForDeferredBatchUpdate(session, startRevision, wait);
2637
+ await waitForBatchFieldReadback(session, fields);
2638
+ }
2639
+ catch (e) {
2640
+ if (canFallbackToSequentialFill(e))
2641
+ return { ok: false };
2642
+ throw e;
2643
+ }
2644
+ const after = sessionA11y(session);
2645
+ if (!after)
2646
+ return { ok: false };
2647
+ const signals = collectSessionSignals(after);
2648
+ const invalidRemaining = signals.invalidFields.length;
2649
+ if ((!batchAckResult || batchAckResult.invalidCount > 0) && invalidRemaining > 0) {
2650
+ return { ok: false };
2651
+ }
2652
+ return {
2653
+ ok: true,
2654
+ finalSource: 'session',
2655
+ final: sessionSignalsPayload(signals, detail),
2656
+ invalidRemaining,
2657
+ };
2658
+ }
2205
2659
  async function waitForDeferredBatchUpdate(session, startRevision, wait) {
2206
2660
  if (wait.status !== 'acknowledged' || session.updateRevision > startRevision)
2207
2661
  return;
@@ -2242,6 +2696,22 @@ function batchFieldReadbackMatches(a11y, field) {
2242
2696
  }
2243
2697
  }
2244
2698
  }
2699
+ function actionNeedsUiTree(action) {
2700
+ switch (action.type) {
2701
+ case 'wait_for':
2702
+ return true;
2703
+ case 'click':
2704
+ return action.x === undefined || action.y === undefined || Boolean(action.waitFor);
2705
+ default:
2706
+ return false;
2707
+ }
2708
+ }
2709
+ function canDeferInitialFrameForRunActions(actions) {
2710
+ const first = actions[0];
2711
+ if (!first)
2712
+ return false;
2713
+ return first.type === 'fill_fields';
2714
+ }
2245
2715
  async function executeBatchAction(session, action, detail, includeSteps) {
2246
2716
  switch (action.type) {
2247
2717
  case 'click': {
@@ -2255,6 +2725,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2255
2725
  name: action.name,
2256
2726
  text: action.text,
2257
2727
  contextText: action.contextText,
2728
+ promptText: action.promptText,
2729
+ sectionText: action.sectionText,
2730
+ itemText: action.itemText,
2258
2731
  value: action.value,
2259
2732
  checked: action.checked,
2260
2733
  disabled: action.disabled,
@@ -2286,6 +2759,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2286
2759
  name: action.waitFor.name,
2287
2760
  text: action.waitFor.text,
2288
2761
  contextText: action.waitFor.contextText,
2762
+ promptText: action.waitFor.promptText,
2763
+ sectionText: action.waitFor.sectionText,
2764
+ itemText: action.waitFor.itemText,
2289
2765
  value: action.waitFor.value,
2290
2766
  checked: action.waitFor.checked,
2291
2767
  disabled: action.waitFor.disabled,
@@ -2442,6 +2918,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2442
2918
  name: action.name,
2443
2919
  text: action.text,
2444
2920
  contextText: action.contextText,
2921
+ promptText: action.promptText,
2922
+ sectionText: action.sectionText,
2923
+ itemText: action.itemText,
2445
2924
  value: action.value,
2446
2925
  checked: action.checked,
2447
2926
  disabled: action.disabled,
@@ -2479,6 +2958,20 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2479
2958
  const resolvedFields = resolveFillFieldInputs(session, action.fields);
2480
2959
  if (!resolvedFields.ok)
2481
2960
  throw new Error(resolvedFields.error);
2961
+ if (!includeSteps) {
2962
+ const batched = await tryBatchedResolvedFields(session, resolvedFields.fields, detail);
2963
+ if (batched.ok) {
2964
+ return {
2965
+ summary: `Filled ${resolvedFields.fields.length} field(s) in one proxy batch.`,
2966
+ compact: {
2967
+ fieldCount: resolvedFields.fields.length,
2968
+ execution: 'batched',
2969
+ finalSource: batched.finalSource,
2970
+ final: batched.final,
2971
+ },
2972
+ };
2973
+ }
2974
+ }
2482
2975
  const steps = [];
2483
2976
  for (let index = 0; index < resolvedFields.fields.length; index++) {
2484
2977
  const field = resolvedFields.fields[index];
@@ -2571,8 +3064,24 @@ async function executeFillField(session, field, detail) {
2571
3064
  }
2572
3065
  }
2573
3066
  }
2574
- function ok(text) {
2575
- return { content: [{ type: 'text', text }] };
3067
+ function ok(text, screenshot) {
3068
+ const content = [
3069
+ { type: 'text', text },
3070
+ ];
3071
+ if (screenshot) {
3072
+ content.push({ type: 'image', data: screenshot, mimeType: 'image/png' });
3073
+ }
3074
+ return { content };
3075
+ }
3076
+ async function captureScreenshotBase64(session) {
3077
+ try {
3078
+ const wait = await sendScreenshot(session);
3079
+ const result = wait.result;
3080
+ return typeof result?.screenshot === 'string' ? result.screenshot : undefined;
3081
+ }
3082
+ catch {
3083
+ return undefined;
3084
+ }
2576
3085
  }
2577
3086
  function err(text) {
2578
3087
  return { content: [{ type: 'text', text }], isError: true };
@@ -2673,10 +3182,10 @@ function sectionContext(root, node) {
2673
3182
  }
2674
3183
  return undefined;
2675
3184
  }
2676
- function nodeContextText(root, node) {
2677
- return [promptContext(root, node), sectionContext(root, node)].filter(Boolean).join(' | ') || undefined;
3185
+ function nodeContextText(context) {
3186
+ return [context?.prompt, context?.section, context?.item].filter(Boolean).join(' | ') || undefined;
2678
3187
  }
2679
- function nodeMatchesFilter(node, filter, contextText) {
3188
+ function nodeMatchesFilter(node, filter, context) {
2680
3189
  if (filter.id && nodeIdForPath(node.path) !== filter.id)
2681
3190
  return false;
2682
3191
  if (filter.role && node.role !== filter.role)
@@ -2688,7 +3197,13 @@ function nodeMatchesFilter(node, filter, contextText) {
2688
3197
  if (filter.text &&
2689
3198
  !textMatches(`${node.name ?? ''} ${node.value ?? ''} ${node.validation?.error ?? ''} ${node.validation?.description ?? ''}`.trim(), filter.text))
2690
3199
  return false;
2691
- if (!textMatches(contextText, filter.contextText))
3200
+ if (!textMatches(nodeContextText(context), filter.contextText))
3201
+ return false;
3202
+ if (!textMatches(context?.prompt, filter.promptText))
3203
+ return false;
3204
+ if (!textMatches(context?.section, filter.sectionText))
3205
+ return false;
3206
+ if (!textMatches(context?.item, filter.itemText))
2692
3207
  return false;
2693
3208
  if (filter.checked !== undefined && node.state?.checked !== filter.checked)
2694
3209
  return false;
@@ -2711,8 +3226,10 @@ function nodeMatchesFilter(node, filter, contextText) {
2711
3226
  export function findNodes(node, filter) {
2712
3227
  const matches = [];
2713
3228
  function walk(n) {
2714
- const contextText = filter.contextText ? nodeContextText(node, n) : undefined;
2715
- if (nodeMatchesFilter(n, filter, contextText) && hasNodeFilter(filter))
3229
+ const context = filter.contextText || filter.promptText || filter.sectionText || filter.itemText
3230
+ ? nodeContextForNode(node, n)
3231
+ : undefined;
3232
+ if (nodeMatchesFilter(n, filter, context) && hasNodeFilter(filter))
2716
3233
  matches.push(n);
2717
3234
  for (const child of n.children)
2718
3235
  walk(child);
@@ -2755,14 +3272,13 @@ function formatNode(node, root, viewport) {
2755
3272
  : Math.round(Math.min(Math.max(node.bounds.y + node.bounds.height / 2, 0), viewport.height));
2756
3273
  const revealDeltaX = Math.round(node.bounds.x + node.bounds.width / 2 - viewport.width / 2);
2757
3274
  const revealDeltaY = Math.round(node.bounds.y + node.bounds.height / 2 - viewport.height / 2);
2758
- const prompt = promptContext(root, node);
2759
- const section = sectionContext(root, node);
3275
+ const context = nodeContextForNode(root, node);
2760
3276
  return {
2761
3277
  id: nodeIdForPath(node.path),
2762
3278
  role: node.role,
2763
3279
  name: node.name,
2764
3280
  ...(node.value ? { value: node.value } : {}),
2765
- ...(prompt || section ? { context: { ...(prompt ? { prompt } : {}), ...(section ? { section } : {}) } } : {}),
3281
+ ...(context ? { context } : {}),
2766
3282
  bounds: node.bounds,
2767
3283
  visibleBounds: {
2768
3284
  x: visibleLeft,