@geometra/mcp 1.19.20 → 1.19.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -27,22 +27,23 @@ Proxy-backed sessions stay warm by default on disconnect, and MCP now keeps a sm
27
27
  | `geometra_connect` | Connect with `url` (ws://…) **or** `pageUrl` (https://…) to auto-start geometra-proxy; can inline `formSchema` and/or `pageModel` for lower-turn starts |
28
28
  | `geometra_query` | Find elements by stable id, role, name, text content, ancestor/prompt context, current value, or semantic state such as `invalid`, `required`, or `busy` |
29
29
  | `geometra_wait_for` | Wait for a semantic condition instead of guessing sleeps (`busy`, `disabled`, alerts, values, etc.). **Strict parameters** — use `text` plus `present: false` to wait until a substring disappears (e.g. “Parsing your resume”); there is no `textGone` field |
30
+ | `geometra_wait_for_resume_parse` | Convenience wait until a parsing banner is gone (defaults: `text`: `"Parsing"`, `present` implied false). Same engine as `geometra_wait_for` |
30
31
  | `geometra_form_schema` | Compact, fill-oriented form schema with stable field ids and collapsed radio/button groups; can auto-connect from `pageUrl` / `url` |
31
32
  | `geometra_fill_form` | Fill a form from `valuesById` / `valuesByLabel` in one MCP call; can auto-connect from `pageUrl` / `url` for the lowest-token known-form path |
32
- | `geometra_fill_fields` | Fill labeled text/choice/toggle/file fields in one MCP call; can return final-only status for the smallest responses |
33
+ | `geometra_fill_fields` | Fill text/choice/toggle/file fields in one MCP call; text/choice/toggle entries can use `fieldId` from `geometra_form_schema` without repeating the label |
33
34
  | `geometra_run_actions` | Execute a batch of high-level actions in one MCP round trip and get one consolidated result, with optional final-only output |
34
35
  | `geometra_page_model` | Summary-first webpage model: archetypes, stable section ids, counts, top-level sections, primary actions |
35
36
  | `geometra_expand_section` | Expand one form/dialog/list/landmark from `geometra_page_model` on demand, with paging/filtering for long sections |
36
- | `geometra_reveal` | Scroll until a matching node is visible instead of guessing wheel deltas |
37
+ | `geometra_reveal` | Scroll until a matching node is visible instead of guessing wheel deltas; auto-scales reveal steps for tall forms when omitted |
37
38
  | `geometra_click` | Click by coordinates or semantic target, optionally waiting for a post-click semantic condition |
38
39
  | `geometra_type` | Type text into the focused element |
39
40
  | `geometra_key` | Send special keys (Enter, Tab, Escape, arrows) |
40
41
  | `geometra_upload_files` | Attach files: labeled field / auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
41
- | `geometra_pick_listbox_option` | Pick an option from a custom dropdown/searchable combobox; can open by field label (`@geometra/proxy` only) |
42
- | `geometra_select_option` | Choose an option on a native `<select>` (`@geometra/proxy` only) |
42
+ | `geometra_pick_listbox_option` | Pick an option from a custom dropdown/searchable combobox; can open by field label, falls back to keyboard selection, and returns visible option hints on failure (`@geometra/proxy` only) |
43
+ | `geometra_select_option` | Choose an option on a native `<select>` only; use `geometra_pick_listbox_option` for custom dropdowns (`@geometra/proxy` only) |
43
44
  | `geometra_set_checked` | Set a checkbox or radio by label instead of coordinate clicks (`@geometra/proxy` only) |
44
45
  | `geometra_wheel` | Mouse wheel / scroll (`@geometra/proxy` only) |
45
- | `geometra_snapshot` | Default **compact**: flat viewport-visible actionable nodes (minified JSON). `view=full` for nested tree |
46
+ | `geometra_snapshot` | Default **compact**: flat viewport-visible actionable nodes (minified JSON). `view=full` for nested tree, `view=form-required` for required fields including offscreen ones |
46
47
  | `geometra_layout` | Raw computed geometry for every node |
47
48
  | `geometra_disconnect` | Close the connection |
48
49
 
@@ -50,6 +51,20 @@ Proxy-backed sessions stay warm by default on disconnect, and MCP now keeps a sm
50
51
 
51
52
  The tool input is **strict**: unknown keys are rejected (so mistaken names like `textGone` fail fast instead of being ignored). To wait until copy such as “Parsing…” or “Parsing your resume” is **gone**, pass a substring that matches the banner in **`text`** and set **`present`** to **`false`**. Example: `{ "text": "Parsing", "present": false }` (adjust the substring per site).
52
53
 
54
+ For the common resume-upload case, prefer **`geometra_wait_for_resume_parse`** (optional `text`, default `"Parsing"`; optional `timeoutMs`, default `60000`) so agents do not have to remember `present: false`.
55
+
56
+ ### Custom dropdowns and searchable comboboxes
57
+
58
+ Use **`geometra_pick_listbox_option`** for React Select / Radix / Headless UI / custom ATS dropdowns. Prefer **`fieldLabel`** so MCP opens the right combobox semantically, and pass **`query`** when the dropdown is searchable. If click-selection cannot be confirmed, the proxy now falls back to keyboard navigation for editable comboboxes.
59
+
60
+ When selection fails, the error payload includes a capped **`visibleOptions`** list so an agent can retry with a real label instead of blindly guessing. Use **`geometra_select_option`** only for native HTML `<select>` controls.
61
+
62
+ ### Long forms and offscreen required fields
63
+
64
+ For tall application forms, **`geometra_snapshot({ view: "form-required" })`** returns required fields even when they are offscreen, with **bounds**, **visibility**, and **scrollHint** data. This is useful for “what required fields are left?” without paging through a full tree.
65
+
66
+ When a target is below the fold, **`geometra_reveal`** and semantic click actions auto-scale reveal steps when `maxSteps` / `maxRevealSteps` are omitted, so long forms do not fail after a hard-coded short scroll budget. If you already have schema ids, **`geometra_fill_fields`** can now use `fieldId` directly for text / choice / toggle entries without repeating `fieldLabel` / `label`.
67
+
53
68
  ## Setup
54
69
 
55
70
  <details>
@@ -304,13 +319,14 @@ Agent: geometra_query({ role: "button", name: "Save" })
304
319
  1. The MCP server connects to a WebSocket peer that speaks GEOM v1 (`frame` with `layout` + `tree`, optional `patch` updates).
305
320
  2. It receives the computed layout (`{ x, y, width, height }` for every node) and the UI tree (`kind`, `semantic`, `props`, `handlers`, `children`).
306
321
  3. It builds an accessibility tree from that data — roles, names, focusable state, bounds.
307
- 4. **`geometra_snapshot`** defaults to a **compact** flat list of viewport-visible actionable nodes (minified JSON) to reduce LLM tokens; use `view: "full"` for the complete nested tree.
322
+ 4. **`geometra_snapshot`** defaults to a **compact** flat list of viewport-visible actionable nodes (minified JSON) to reduce LLM tokens; use `view: "full"` for the complete nested tree or `view: "form-required"` for offscreen required fields plus scroll hints.
308
323
  5. **`geometra_form_schema`** is the compact form-specific path: stable field ids, required/invalid state, current values, and collapsed choice groups without layout-heavy section detail. It can auto-connect when you pass `pageUrl` / `url`.
309
324
  6. **`geometra_fill_form`** turns a compact values object into semantic field operations server-side, so the model does not need to emit one tool call per field. It can also auto-connect, which collapses “open page + fill known values” into a single tool call.
310
- 7. **`geometra_page_model`** is still the right summary-first path for non-form exploration: page archetypes, stable section ids, counts, top-level landmarks/forms/dialogs/lists, and a few primary actions.
311
- 8. **`geometra_expand_section`** fetches richer details only for the section you care about (fields, actions, headings, nested lists, list items, text preview).
312
- 9. After interactions, action tools return a **semantic delta** when possible (dialogs opened/closed, forms appeared/removed, list counts changed, named/focusable nodes added/removed/updated). If nothing meaningful changed, they fall back to a short current-UI overview.
313
- 10. After each interaction, the peer sends updated geometry (full `frame` or `patch`) the MCP tools interpret that into compact summaries.
325
+ 7. **`geometra_fill_fields`** is the lower-level field-intent path when you need direct control; text / choice / toggle entries can now resolve `fieldId` from `geometra_form_schema` without repeating labels.
326
+ 8. **`geometra_page_model`** is still the right summary-first path for non-form exploration: page archetypes, stable section ids, counts, top-level landmarks/forms/dialogs/lists, and a few primary actions.
327
+ 9. **`geometra_expand_section`** fetches richer details only for the section you care about (fields, actions, headings, nested lists, list items, text preview).
328
+ 10. After interactions, action tools return a **semantic delta** when possible (dialogs opened/closed, forms appeared/removed, list counts changed, named/focusable nodes added/removed/updated). If nothing meaningful changed, they fall back to a short current-UI overview.
329
+ 11. After each interaction, the peer sends updated geometry (full `frame` or `patch`) — the MCP tools interpret that into compact summaries.
314
330
 
315
331
  ## Long Forms
316
332
 
@@ -319,11 +335,12 @@ For long application flows, prefer one of these patterns:
319
335
  1. `geometra_fill_form({ pageUrl, valuesByLabel })` when you already know the fields you want to set
320
336
  2. otherwise `geometra_form_schema`
321
337
  3. then `geometra_fill_form`
322
- 3. `geometra_reveal` for far-below-fold targets such as submit buttons
323
- 4. `geometra_click({ ..., waitFor: ... })` when one action should also wait for the next semantic state
324
- 5. `geometra_run_actions` when you need mixed navigation + waits + field entry
325
- 6. `geometra_connect({ pageUrl, returnPageModel: true })` when you want connect + summary-first exploration in one turn
326
- 7. `geometra_page_model` + `geometra_expand_section` when you are still exploring the page rather than filling it
338
+ 4. `geometra_snapshot({ view: "form-required" })` when you need the remaining required fields including offscreen ones
339
+ 5. `geometra_reveal` for far-below-fold targets such as submit buttons (auto-scales reveal steps when you omit `maxSteps`)
340
+ 6. `geometra_click({ ..., waitFor: ... })` when one action should also wait for the next semantic state
341
+ 7. `geometra_run_actions` when you need mixed navigation + waits + field entry
342
+ 8. `geometra_connect({ pageUrl, returnPageModel: true })` when you want connect + summary-first exploration in one turn
343
+ 9. `geometra_page_model` + `geometra_expand_section` when you are still exploring the page rather than filling it
327
344
 
328
345
  Typical batch:
329
346
 
@@ -87,6 +87,7 @@ vi.mock('../session.js', () => ({
87
87
  lists: [],
88
88
  })),
89
89
  buildFormSchemas: vi.fn(() => mockState.formSchemas),
90
+ buildFormRequiredSnapshot: vi.fn(() => []),
90
91
  expandPageSection: vi.fn(() => null),
91
92
  buildUiDelta: vi.fn(() => ({})),
92
93
  hasUiDelta: vi.fn(() => false),
@@ -171,6 +172,42 @@ describe('batch MCP result shaping', () => {
171
172
  readback: { role: 'textbox', value: 'taylor@example.com' },
172
173
  });
173
174
  });
175
+ it('resolves fieldId-only fill_fields entries from the current form schema', async () => {
176
+ const handler = getToolHandler('geometra_fill_fields');
177
+ mockState.formSchemas = [{
178
+ formId: 'fm:0',
179
+ name: 'Application',
180
+ fieldCount: 3,
181
+ requiredCount: 0,
182
+ invalidCount: 0,
183
+ fields: [
184
+ { id: 'ff:0.0', kind: 'text', label: 'Full name' },
185
+ { id: 'ff:0.1', kind: 'choice', label: 'Preferred location', choiceType: 'select' },
186
+ { id: 'ff:0.2', kind: 'toggle', label: 'Share my profile for future roles', controlType: 'checkbox' },
187
+ ],
188
+ }];
189
+ const result = await handler({
190
+ fields: [
191
+ { kind: 'text', fieldId: 'ff:0.0', value: 'Taylor Applicant' },
192
+ { kind: 'choice', fieldId: 'ff:0.1', value: 'Berlin, Germany' },
193
+ { kind: 'toggle', fieldId: 'ff:0.2', checked: true },
194
+ ],
195
+ stopOnError: true,
196
+ failOnInvalid: false,
197
+ includeSteps: true,
198
+ detail: 'minimal',
199
+ });
200
+ const payload = JSON.parse(result.content[0].text);
201
+ expect(mockState.sendFieldText).toHaveBeenCalledWith(mockState.session, 'Full name', 'Taylor Applicant', { exact: undefined, fieldId: 'ff:0.0' }, undefined);
202
+ expect(mockState.sendFieldChoice).toHaveBeenCalledWith(mockState.session, 'Preferred location', 'Berlin, Germany', { exact: undefined, query: undefined, choiceType: 'select', fieldId: 'ff:0.1' }, undefined);
203
+ expect(mockState.sendSetChecked).toHaveBeenCalledWith(mockState.session, 'Share my profile for future roles', { checked: true, exact: undefined, controlType: 'checkbox' }, undefined);
204
+ expect(payload).toMatchObject({
205
+ completed: true,
206
+ fieldCount: 3,
207
+ successCount: 3,
208
+ errorCount: 0,
209
+ });
210
+ });
174
211
  it('lets run_actions omit step listings while keeping capped final validation state', async () => {
175
212
  const handler = getToolHandler('geometra_run_actions');
176
213
  mockState.currentA11yRoot = node('group', undefined, {
@@ -763,6 +800,52 @@ describe('query and reveal tools', () => {
763
800
  },
764
801
  });
765
802
  });
803
+ it('auto-scales reveal steps for tall forms when maxSteps is omitted', async () => {
804
+ const handler = getToolHandler('geometra_reveal');
805
+ let scrollY = 0;
806
+ const setTree = () => {
807
+ mockState.currentA11yRoot = node('group', undefined, {
808
+ bounds: { x: 0, y: 0, width: 1280, height: 800 },
809
+ meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY },
810
+ children: [
811
+ node('form', 'Application', {
812
+ bounds: { x: 20, y: -200 - scrollY, width: 760, height: 6200 },
813
+ path: [0],
814
+ children: [
815
+ node('button', 'Submit application', {
816
+ bounds: { x: 60, y: 5200 - scrollY, width: 180, height: 40 },
817
+ path: [0, 0],
818
+ }),
819
+ ],
820
+ }),
821
+ ],
822
+ });
823
+ };
824
+ setTree();
825
+ mockState.sendWheel.mockImplementation(async () => {
826
+ scrollY += 650;
827
+ setTree();
828
+ bumpMockUiRevision();
829
+ return { status: 'updated', timeoutMs: 2500 };
830
+ });
831
+ const result = await handler({
832
+ role: 'button',
833
+ name: 'Submit application',
834
+ fullyVisible: true,
835
+ timeoutMs: 2500,
836
+ });
837
+ const payload = JSON.parse(result.content[0].text);
838
+ expect(mockState.sendWheel).toHaveBeenCalledTimes(7);
839
+ expect(payload).toMatchObject({
840
+ revealed: true,
841
+ attempts: 7,
842
+ target: {
843
+ role: 'button',
844
+ name: 'Submit application',
845
+ visibility: { fullyVisible: true },
846
+ },
847
+ });
848
+ });
766
849
  it('clicks an offscreen semantic target by revealing it first', async () => {
767
850
  const handler = getToolHandler('geometra_click');
768
851
  mockState.currentA11yRoot = node('group', undefined, {
@@ -1,5 +1,5 @@
1
1
  import { describe, expect, it } from 'vitest';
2
- import { buildA11yTree, buildCompactUiIndex, buildFormSchemas, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
2
+ import { buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildFormSchemas, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
3
3
  function node(role, name, bounds, options) {
4
4
  return {
5
5
  role,
@@ -375,6 +375,78 @@ describe('buildFormSchemas', () => {
375
375
  });
376
376
  });
377
377
  });
378
+ describe('buildFormRequiredSnapshot', () => {
379
+ it('keeps offscreen required fields with bounds, visibility, and scroll hints', () => {
380
+ const tree = node('group', undefined, { x: 0, y: 0, width: 900, height: 700 }, {
381
+ children: [
382
+ node('form', 'Application', { x: 20, y: -160, width: 760, height: 2200 }, {
383
+ path: [0],
384
+ children: [
385
+ node('textbox', 'Full name', { x: 48, y: 120, width: 320, height: 36 }, {
386
+ path: [0, 0],
387
+ state: { required: true },
388
+ }),
389
+ node('combobox', 'Preferred location', { x: 48, y: 940, width: 320, height: 36 }, {
390
+ path: [0, 1],
391
+ state: { required: true },
392
+ meta: { controlTag: 'select' },
393
+ }),
394
+ node('group', undefined, { x: 40, y: 1260, width: 520, height: 96 }, {
395
+ path: [0, 2],
396
+ children: [
397
+ node('text', 'Will you require sponsorship?', { x: 48, y: 1260, width: 320, height: 24 }, {
398
+ path: [0, 2, 0],
399
+ }),
400
+ node('button', 'Yes', { x: 48, y: 1300, width: 88, height: 40 }, {
401
+ path: [0, 2, 1],
402
+ focusable: true,
403
+ state: { required: true },
404
+ }),
405
+ node('button', 'No', { x: 148, y: 1300, width: 88, height: 40 }, {
406
+ path: [0, 2, 2],
407
+ focusable: true,
408
+ }),
409
+ ],
410
+ }),
411
+ ],
412
+ }),
413
+ ],
414
+ });
415
+ const snapshot = buildFormRequiredSnapshot(tree, { includeOptions: true });
416
+ expect(snapshot).toHaveLength(1);
417
+ expect(snapshot[0]).toMatchObject({
418
+ formId: 'fm:0',
419
+ name: 'Application',
420
+ requiredCount: 3,
421
+ fields: [
422
+ {
423
+ id: 'ff:0.0',
424
+ label: 'Full name',
425
+ visibility: { intersectsViewport: true, fullyVisible: true },
426
+ scrollHint: { status: 'visible' },
427
+ bounds: { x: 48, y: 120, width: 320, height: 36 },
428
+ },
429
+ {
430
+ id: 'ff:0.1',
431
+ label: 'Preferred location',
432
+ choiceType: 'select',
433
+ visibility: { intersectsViewport: false, offscreenBelow: true },
434
+ scrollHint: { status: 'offscreen' },
435
+ bounds: { x: 48, y: 940, width: 320, height: 36 },
436
+ },
437
+ {
438
+ id: 'ff:0.2',
439
+ label: 'Will you require sponsorship?',
440
+ choiceType: 'group',
441
+ visibility: { intersectsViewport: false, offscreenBelow: true },
442
+ scrollHint: { status: 'offscreen' },
443
+ bounds: { x: 40, y: 1260, width: 520, height: 96 },
444
+ options: ['Yes', 'No'],
445
+ },
446
+ ],
447
+ });
448
+ });
449
+ });
378
450
  describe('buildUiDelta', () => {
379
451
  it('captures opened dialogs, state changes, and list count changes', () => {
380
452
  const before = node('group', undefined, { x: 0, y: 0, width: 1024, height: 768 }, {
package/dist/server.js CHANGED
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
2
2
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
3
3
  import { z } from 'zod';
4
4
  import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
5
- import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
5
+ import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
6
6
  function checkedStateInput() {
7
7
  return z
8
8
  .union([z.boolean(), z.literal('mixed')])
@@ -68,19 +68,44 @@ function waitConditionShape() {
68
68
  }
69
69
  const GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE = 'Provide at least one filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
70
70
  'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text for substring matching. ' +
71
- 'To wait until text disappears from the UI, use geometra_wait_for with text and present: false.';
71
+ 'To wait until text disappears from the UI, use geometra_wait_for with text and present: false, or geometra_wait_for_resume_parse for typical resume “Parsing…” banners.';
72
72
  const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
73
73
  'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text with a distinctive substring and present: false to wait until that text is gone ' +
74
74
  '(common for “Parsing…”, “Parsing your resume”, or similar). Passing only present/timeoutMs is not enough without a filter.';
75
75
  /** Strict input so unknown keys (e.g. textGone) fail parse; empty-filter checks happen in handlers / waitForSemanticCondition. */
76
76
  const geometraQueryInputSchema = z.object(nodeFilterShape()).strict();
77
77
  const geometraWaitForInputSchema = z.object(waitConditionShape()).strict();
78
+ /** Same upper bound as geometra_wait_for; resume uploads often need the full minute. */
79
+ const geometraWaitForResumeParseInputSchema = z
80
+ .object({
81
+ text: z
82
+ .string()
83
+ .min(1)
84
+ .default('Parsing')
85
+ .describe('Substring that appears in the loading/parsing banner while work is in progress (default matches copy like “Parsing your resume”, “Parsing…”, “Parsing resume”, etc.)'),
86
+ timeoutMs: z
87
+ .number()
88
+ .int()
89
+ .min(50)
90
+ .max(60_000)
91
+ .default(60_000)
92
+ .describe('Maximum time to wait before returning an error (default 60000ms)'),
93
+ })
94
+ .strict();
78
95
  const timeoutMsInput = z.number().int().min(50).max(60_000).optional();
79
- const fillFieldSchema = z.discriminatedUnion('kind', [
96
+ const fillFieldSchema = z.union([
80
97
  z.object({
81
98
  kind: z.literal('text'),
82
99
  fieldId: z.string().optional().describe('Optional stable field id from geometra_form_schema'),
83
- fieldLabel: z.string().describe('Visible field label / accessible name'),
100
+ fieldLabel: z.string().describe('Visible field label / accessible name. Optional to duplicate when fieldId is present.'),
101
+ value: z.string().describe('Text value to set'),
102
+ exact: z.boolean().optional().describe('Exact label match'),
103
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
104
+ }),
105
+ z.object({
106
+ kind: z.literal('text'),
107
+ fieldId: z.string().describe('Stable field id from geometra_form_schema'),
108
+ fieldLabel: z.string().optional().describe('Optional when fieldId is present; MCP resolves the current label from geometra_form_schema'),
84
109
  value: z.string().describe('Text value to set'),
85
110
  exact: z.boolean().optional().describe('Exact label match'),
86
111
  timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
@@ -88,7 +113,20 @@ const fillFieldSchema = z.discriminatedUnion('kind', [
88
113
  z.object({
89
114
  kind: z.literal('choice'),
90
115
  fieldId: z.string().optional().describe('Optional stable field id from geometra_form_schema'),
91
- fieldLabel: z.string().describe('Visible field label / accessible name'),
116
+ fieldLabel: z.string().describe('Visible field label / accessible name. Optional to duplicate when fieldId is present.'),
117
+ value: z.string().describe('Desired option value / answer label'),
118
+ query: z.string().optional().describe('Optional search text for searchable comboboxes'),
119
+ choiceType: z
120
+ .enum(['select', 'group', 'listbox'])
121
+ .optional()
122
+ .describe('Optional choice subtype hint. Use `group` for repeated radio/button answers, `select` for native selects, and `listbox` for searchable dropdowns.'),
123
+ exact: z.boolean().optional().describe('Exact label match'),
124
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
125
+ }),
126
+ z.object({
127
+ kind: z.literal('choice'),
128
+ fieldId: z.string().describe('Stable field id from geometra_form_schema'),
129
+ fieldLabel: z.string().optional().describe('Optional when fieldId is present; MCP resolves the current label from geometra_form_schema'),
92
130
  value: z.string().describe('Desired option value / answer label'),
93
131
  query: z.string().optional().describe('Optional search text for searchable comboboxes'),
94
132
  choiceType: z
@@ -101,7 +139,16 @@ const fillFieldSchema = z.discriminatedUnion('kind', [
101
139
  z.object({
102
140
  kind: z.literal('toggle'),
103
141
  fieldId: z.string().optional().describe('Optional stable field id from geometra_form_schema'),
104
- label: z.string().describe('Visible checkbox/radio label to set'),
142
+ label: z.string().describe('Visible checkbox/radio label to set. Optional to duplicate when fieldId is present.'),
143
+ checked: z.boolean().optional().default(true).describe('Desired checked state (default true)'),
144
+ exact: z.boolean().optional().describe('Exact label match'),
145
+ controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
146
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
147
+ }),
148
+ z.object({
149
+ kind: z.literal('toggle'),
150
+ fieldId: z.string().describe('Stable field id from geometra_form_schema'),
151
+ label: z.string().optional().describe('Optional when fieldId is present; MCP resolves the current label from geometra_form_schema'),
105
152
  checked: z.boolean().optional().default(true).describe('Desired checked state (default true)'),
106
153
  exact: z.boolean().optional().describe('Exact label match'),
107
154
  controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
@@ -115,6 +162,14 @@ const fillFieldSchema = z.discriminatedUnion('kind', [
115
162
  exact: z.boolean().optional().describe('Exact label match'),
116
163
  timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
117
164
  }),
165
+ z.object({
166
+ kind: z.literal('file'),
167
+ fieldId: z.string().describe('Stable field id from geometra_form_schema'),
168
+ fieldLabel: z.string().optional().describe('Optional when fieldId is present; MCP resolves the current label when file fields are exposed by geometra_form_schema'),
169
+ paths: z.array(z.string()).min(1).describe('Absolute paths on the proxy machine'),
170
+ exact: z.boolean().optional().describe('Exact label match'),
171
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
172
+ }),
118
173
  ]);
119
174
  const formValueSchema = z.union([
120
175
  z.string(),
@@ -130,7 +185,7 @@ const batchActionSchema = z.discriminatedUnion('type', [
130
185
  ...nodeFilterShape(),
131
186
  index: z.number().int().min(0).optional().describe('Which matching semantic target to click after sorting top-to-bottom'),
132
187
  fullyVisible: z.boolean().optional().describe('When clicking by semantic target, require full visibility before clicking (default true)'),
133
- maxRevealSteps: z.number().int().min(1).max(12).optional().describe('Maximum reveal attempts before clicking a semantic target'),
188
+ maxRevealSteps: z.number().int().min(1).max(48).optional().describe('Maximum reveal attempts before clicking a semantic target. When omitted, Geometra auto-scales from scroll distance for tall forms.'),
134
189
  revealTimeoutMs: timeoutMsInput.describe('Per-scroll wait timeout while revealing a semantic target'),
135
190
  waitFor: z.object(waitConditionShape()).optional().describe('Optional semantic condition to wait for after the click'),
136
191
  timeoutMs: timeoutMsInput,
@@ -208,7 +263,7 @@ const batchActionSchema = z.discriminatedUnion('type', [
208
263
  }),
209
264
  ]);
210
265
  export function createServer() {
211
- const server = new McpServer({ name: 'geometra', version: '1.19.20' }, { capabilities: { tools: {} } });
266
+ const server = new McpServer({ name: 'geometra', version: '1.19.21' }, { capabilities: { tools: {} } });
212
267
  // ── connect ──────────────────────────────────────────────────
213
268
  server.tool('geometra_connect', `Connect to a Geometra WebSocket peer, or start \`geometra-proxy\` automatically for a normal web page.
214
269
 
@@ -333,13 +388,14 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
333
388
  }
334
389
  });
335
390
  // ── query ────────────────────────────────────────────────────
336
- server.tool('geometra_query', `Find elements in the current Geometra UI by stable id, role, name, text content, current value, or semantic state. Returns matching elements with their exact pixel bounds {x, y, width, height}, visible in-viewport bounds, an on-screen center point, visibility / scroll-reveal hints, role, name, value, state, and tree path.
391
+ server.registerTool('geometra_query', {
392
+ description: `Find elements in the current Geometra UI by stable id, role, name, text content, current value, or semantic state. Returns matching elements with their exact pixel bounds {x, y, width, height}, visible in-viewport bounds, an on-screen center point, visibility / scroll-reveal hints, role, name, value, state, and tree path.
337
393
 
338
394
  This is the Geometra equivalent of Playwright's locator — but instant, structured, and with no browser. Use the returned bounds to click elements or assert on layout.
339
395
 
340
- Unknown parameter names are rejected (strict schema). To wait until visible text goes away (e.g. a parsing banner), use geometra_wait_for with that substring in text and present: false — there is no textGone field.`,
341
- // SDK overload typings only list raw shapes; runtime accepts ZodObject via getZodSchemaObject().
342
- geometraQueryInputSchema, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy }) => {
396
+ Unknown parameter names are rejected (strict schema). To wait until visible text goes away (e.g. a parsing banner), use geometra_wait_for with that substring in text and present: false — there is no textGone field.`,
397
+ inputSchema: geometraQueryInputSchema,
398
+ }, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy }) => {
343
399
  const session = getSession();
344
400
  if (!session?.tree || !session?.layout)
345
401
  return err('Not connected. Call geometra_connect first.');
@@ -371,9 +427,12 @@ Unknown parameter names are rejected (strict schema). To wait until visible text
371
427
  const result = sortA11yNodes(matches).map(node => formatNode(node, a11y, a11y.bounds));
372
428
  return ok(JSON.stringify(result, null, 2));
373
429
  });
374
- server.tool('geometra_wait_for', `Wait for a semantic UI condition without guessing sleep durations. Use this for slow SPA transitions, resume parsing, custom validation alerts, disabled submit buttons, and value/state confirmation before submit.
430
+ server.registerTool('geometra_wait_for', {
431
+ description: `Wait for a semantic UI condition without guessing sleep durations. Use this for slow SPA transitions, resume parsing, custom validation alerts, disabled submit buttons, and value/state confirmation before submit.
375
432
 
376
- The filter matches the same fields as geometra_query (strict schema — unknown keys error). Set \`present: false\` to wait until **no** node matches — for example Ashby/Lever-style “Parsing your resume” or any “Parsing…” banner: \`{ "text": "Parsing", "present": false }\` (tune the substring to the site). Do not use a textGone parameter; use \`text\` + \`present: false\`.`, geometraWaitForInputSchema, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs }) => {
433
+ The filter matches the same fields as geometra_query (strict schema — unknown keys error). Set \`present: false\` to wait until **no** node matches — for example Ashby/Lever-style “Parsing your resume” or any “Parsing…” banner: \`{ "text": "Parsing", "present": false }\` (tune the substring to the site). Do not use a textGone parameter; use \`text\` + \`present: false\`, or \`geometra_wait_for_resume_parse\` for the usual post-upload parsing banner.`,
434
+ inputSchema: geometraWaitForInputSchema,
435
+ }, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs }) => {
377
436
  const session = getSession();
378
437
  if (!session?.tree || !session?.layout)
379
438
  return err('Not connected. Call geometra_connect first.');
@@ -407,10 +466,29 @@ The filter matches the same fields as geometra_query (strict schema — unknown
407
466
  }
408
467
  return ok(JSON.stringify(waited.value.matches.slice(0, 8), null, 2));
409
468
  });
469
+ server.registerTool('geometra_wait_for_resume_parse', {
470
+ description: `Wait until **no** visible text contains the given substring — optimized for ATS “parsing your resume” / file-processing banners after upload.
471
+
472
+ Equivalent to \`geometra_wait_for\` with \`present: false\` and \`text\` set to a banner substring. Default \`text\` is \`Parsing\` (tune per site). Strict schema (unknown keys rejected).`,
473
+ inputSchema: geometraWaitForResumeParseInputSchema,
474
+ }, async ({ text, timeoutMs }) => {
475
+ const session = getSession();
476
+ if (!session?.tree || !session?.layout)
477
+ return err('Not connected. Call geometra_connect first.');
478
+ const filter = { text };
479
+ const waited = await waitForSemanticCondition(session, {
480
+ filter,
481
+ present: false,
482
+ timeoutMs,
483
+ });
484
+ if (!waited.ok)
485
+ return err(waited.error);
486
+ return ok(waitConditionSuccessLine(waited.value));
487
+ });
410
488
  server.tool('geometra_fill_fields', `Fill several labeled form fields in one MCP call. This is the preferred high-level primitive for long forms.
411
489
 
412
- Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / comboboxes / radio-style questions addressed by field label + answer, \`"toggle"\` for individually labeled checkboxes or radios, and \`"file"\` for labeled uploads.`, {
413
- fields: z.array(fillFieldSchema).min(1).max(80).describe('Ordered labeled field operations to apply'),
490
+ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / comboboxes / radio-style questions addressed by field label + answer, \`"toggle"\` for individually labeled checkboxes or radios, and \`"file"\` for labeled uploads. When \`fieldId\` from \`geometra_form_schema\` is present, MCP can resolve the current label server-side so you do not need to duplicate \`fieldLabel\` / \`label\` for text, choice, and toggle fields.`, {
491
+ fields: z.array(fillFieldSchema).min(1).max(80).describe('Ordered field operations to apply. Use fieldId from geometra_form_schema to omit duplicate fieldLabel/label on schema-backed fields.'),
414
492
  stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing field (default true)'),
415
493
  failOnInvalid: z
416
494
  .boolean()
@@ -427,10 +505,13 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
427
505
  const session = getSession();
428
506
  if (!session)
429
507
  return err('Not connected. Call geometra_connect first.');
508
+ const resolvedFields = resolveFillFieldInputs(session, fields);
509
+ if (!resolvedFields.ok)
510
+ return err(resolvedFields.error);
430
511
  const steps = [];
431
512
  let stoppedAt;
432
- for (let index = 0; index < fields.length; index++) {
433
- const field = fields[index];
513
+ for (let index = 0; index < resolvedFields.fields.length; index++) {
514
+ const field = resolvedFields.fields[index];
434
515
  try {
435
516
  const result = await executeFillField(session, field, detail);
436
517
  steps.push(detail === 'verbose'
@@ -452,8 +533,8 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
452
533
  const successCount = steps.filter(step => step.ok === true).length;
453
534
  const errorCount = steps.length - successCount;
454
535
  const payload = {
455
- completed: stoppedAt === undefined && steps.length === fields.length,
456
- fieldCount: fields.length,
536
+ completed: stoppedAt === undefined && steps.length === resolvedFields.fields.length,
537
+ fieldCount: resolvedFields.fields.length,
457
538
  successCount,
458
539
  errorCount,
459
540
  ...(includeSteps ? { steps } : {}),
@@ -847,7 +928,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
847
928
  ...nodeFilterShape(),
848
929
  index: z.number().int().min(0).optional().default(0).describe('Which matching node to reveal after sorting top-to-bottom'),
849
930
  fullyVisible: z.boolean().optional().default(true).describe('Require the target to become fully visible (default true)'),
850
- maxSteps: z.number().int().min(1).max(12).optional().default(6).describe('Maximum reveal attempts before returning an error'),
931
+ maxSteps: z.number().int().min(1).max(48).optional().describe('Maximum reveal attempts before returning an error. When omitted, Geometra auto-scales from scroll distance for tall forms.'),
851
932
  timeoutMs: z
852
933
  .number()
853
934
  .int()
@@ -882,7 +963,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
882
963
  filter,
883
964
  index: index ?? 0,
884
965
  fullyVisible: fullyVisible ?? true,
885
- maxSteps: maxSteps ?? 6,
966
+ maxSteps,
886
967
  timeoutMs: timeoutMs ?? 2_500,
887
968
  });
888
969
  if (!revealed.ok)
@@ -902,7 +983,7 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
902
983
  ...nodeFilterShape(),
903
984
  index: z.number().int().min(0).optional().default(0).describe('Which matching semantic target to click after sorting top-to-bottom'),
904
985
  fullyVisible: z.boolean().optional().default(true).describe('When clicking by semantic target, require full visibility before clicking (default true)'),
905
- maxRevealSteps: z.number().int().min(1).max(12).optional().default(6).describe('Maximum reveal attempts before clicking a semantic target'),
986
+ maxRevealSteps: z.number().int().min(1).max(48).optional().describe('Maximum reveal attempts before clicking a semantic target. When omitted, Geometra auto-scales from scroll distance for tall forms.'),
906
987
  revealTimeoutMs: z
907
988
  .number()
908
989
  .int()
@@ -1075,7 +1156,7 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
1075
1156
  });
1076
1157
  server.tool('geometra_pick_listbox_option', `Pick an option from a custom dropdown / listbox / searchable combobox (Headless UI, React Select, Radix, Ashby-style custom selects, etc.). Requires \`@geometra/proxy\`.
1077
1158
 
1078
- Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying on coordinates. If the opened control is editable, MCP types \`query\` (or the option label by default) before selecting. Uses substring name match unless exact=true, prefers the popup nearest the opened field, and handles a few short affirmative/negative aliases such as \`Yes\` /\`No\` for consent-style copy.`, {
1159
+ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying on coordinates. If the opened control is editable, MCP types \`query\` (or the option label by default) before selecting. Uses fuzzy-ish substring/alias matching unless exact=true, prefers the popup nearest the opened field, can fall back to keyboard navigation for searchable comboboxes, and returns capped \`visibleOptions\` in failure payloads so agents can retry with a real label.`, {
1079
1160
  label: z.string().describe('Accessible name of the option (visible text or aria-label)'),
1080
1161
  exact: z.boolean().optional().describe('Exact name match'),
1081
1162
  openX: z.number().optional().describe('Click to open dropdown'),
@@ -1117,7 +1198,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
1117
1198
  // ── select option (proxy, native <select>) ─────────────────────
1118
1199
  server.tool('geometra_select_option', `Set a native HTML \`<select>\` after clicking its center (x,y from geometra_query). Requires \`@geometra/proxy\`.
1119
1200
 
1120
- Custom React/Vue dropdowns are not supported — open them with geometra_click and pick options by snapshot instead.`, {
1201
+ Custom React/Vue dropdowns are not supported here use \`geometra_pick_listbox_option\` for custom dropdowns / searchable comboboxes.`, {
1121
1202
  x: z.number().describe('X coordinate (e.g. center of the select from geometra_query)'),
1122
1203
  y: z.number().describe('Y coordinate'),
1123
1204
  value: z.string().optional().describe('Option value= attribute'),
@@ -1206,14 +1287,14 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
1206
1287
  }
1207
1288
  });
1208
1289
  // ── snapshot ─────────────────────────────────────────────────
1209
- server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout.
1290
+ server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout. Use **form-required** to list required fields across forms, including offscreen ones, with bounds + visibility + scroll hints for long application flows.
1210
1291
 
1211
1292
  JSON is minified in compact view to save tokens. For a summary-first overview, use geometra_page_model, then geometra_expand_section for just the part you want.`, {
1212
1293
  view: z
1213
- .enum(['compact', 'full'])
1294
+ .enum(['compact', 'full', 'form-required'])
1214
1295
  .optional()
1215
1296
  .default('compact')
1216
- .describe('compact (default): token-efficient flat index. full: nested tree, every node.'),
1297
+ .describe('compact (default): token-efficient flat index. full: nested tree, every node. form-required: required fields across forms, including offscreen controls.'),
1217
1298
  maxNodes: z
1218
1299
  .number()
1219
1300
  .int()
@@ -1222,7 +1303,10 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
1222
1303
  .optional()
1223
1304
  .default(400)
1224
1305
  .describe('Max rows in compact view (default 400).'),
1225
- }, async ({ view, maxNodes }) => {
1306
+ formId: z.string().optional().describe('Optional form id from geometra_form_schema / geometra_page_model when view=form-required'),
1307
+ maxFields: z.number().int().min(1).max(200).optional().default(80).describe('Per-form field cap when view=form-required'),
1308
+ includeOptions: z.boolean().optional().default(false).describe('Include explicit choice option labels when view=form-required'),
1309
+ }, async ({ view, maxNodes, formId, maxFields, includeOptions }) => {
1226
1310
  const session = getSession();
1227
1311
  if (!session?.tree || !session?.layout)
1228
1312
  return err('Not connected. Call geometra_connect first.');
@@ -1232,6 +1316,19 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
1232
1316
  if (view === 'full') {
1233
1317
  return ok(JSON.stringify(a11y, null, 2));
1234
1318
  }
1319
+ if (view === 'form-required') {
1320
+ const payload = {
1321
+ view: 'form-required',
1322
+ viewport: { width: a11y.bounds.width, height: a11y.bounds.height },
1323
+ forms: buildFormRequiredSnapshot(a11y, {
1324
+ formId,
1325
+ maxFields,
1326
+ includeOptions,
1327
+ includeContext: 'auto',
1328
+ }),
1329
+ };
1330
+ return ok(JSON.stringify(payload));
1331
+ }
1235
1332
  const { nodes, truncated, context } = buildCompactUiIndex(a11y, { maxNodes });
1236
1333
  const payload = {
1237
1334
  view: 'compact',
@@ -1731,9 +1828,15 @@ function waitConditionCompact(result) {
1731
1828
  ...(result.present ? { matchCount: result.matchCount } : {}),
1732
1829
  };
1733
1830
  }
1831
+ function inferRevealStepBudget(target, viewport) {
1832
+ const verticalSteps = Math.ceil(Math.abs(target.scrollHint.revealDeltaY) / Math.max(1, viewport.height * 0.75));
1833
+ const horizontalSteps = Math.ceil(Math.abs(target.scrollHint.revealDeltaX) / Math.max(1, viewport.width * 0.7));
1834
+ return clamp(Math.max(6, Math.max(verticalSteps, horizontalSteps) + 1), 6, 48);
1835
+ }
1734
1836
  async function revealSemanticTarget(session, options) {
1735
1837
  let attempts = 0;
1736
- while (attempts <= options.maxSteps) {
1838
+ let stepBudget = options.maxSteps;
1839
+ while (attempts <= (stepBudget ?? 48)) {
1737
1840
  const a11y = sessionA11y(session);
1738
1841
  if (!a11y)
1739
1842
  return { ok: false, error: 'No UI tree available to reveal from' };
@@ -1748,6 +1851,7 @@ async function revealSemanticTarget(session, options) {
1748
1851
  };
1749
1852
  }
1750
1853
  const formatted = formatNode(matches[options.index], a11y, a11y.bounds);
1854
+ stepBudget ??= inferRevealStepBudget(formatted, a11y.bounds);
1751
1855
  const visible = options.fullyVisible ? formatted.visibility.fullyVisible : formatted.visibility.intersectsViewport;
1752
1856
  if (visible) {
1753
1857
  return {
@@ -1758,12 +1862,13 @@ async function revealSemanticTarget(session, options) {
1758
1862
  },
1759
1863
  };
1760
1864
  }
1761
- if (attempts === options.maxSteps) {
1865
+ if (attempts === stepBudget) {
1762
1866
  return {
1763
1867
  ok: false,
1764
1868
  error: JSON.stringify({
1765
1869
  revealed: false,
1766
1870
  attempts,
1871
+ maxSteps: stepBudget,
1767
1872
  target: formatted,
1768
1873
  }, null, 2),
1769
1874
  };
@@ -1806,7 +1911,7 @@ async function resolveClickLocation(session, options) {
1806
1911
  filter: options.filter,
1807
1912
  index: options.index ?? 0,
1808
1913
  fullyVisible: options.fullyVisible ?? true,
1809
- maxSteps: options.maxRevealSteps ?? 6,
1914
+ maxSteps: options.maxRevealSteps,
1810
1915
  timeoutMs: options.revealTimeoutMs ?? 2_500,
1811
1916
  });
1812
1917
  if (!revealed.ok)
@@ -1954,8 +2059,109 @@ function planFormFill(schema, opts) {
1954
2059
  }
1955
2060
  return { ok: true, fields: planned };
1956
2061
  }
2062
+ function isResolvedFillFieldInput(field) {
2063
+ if (field.kind === 'toggle')
2064
+ return typeof field.label === 'string' && field.label.length > 0;
2065
+ return typeof field.fieldLabel === 'string' && field.fieldLabel.length > 0;
2066
+ }
2067
+ function resolveFillFieldInputs(session, fields) {
2068
+ const unresolved = fields.filter(field => !isResolvedFillFieldInput(field));
2069
+ if (unresolved.length === 0) {
2070
+ return { ok: true, fields: fields };
2071
+ }
2072
+ const a11y = sessionA11y(session);
2073
+ if (!a11y)
2074
+ return { ok: false, error: 'No UI tree available to resolve fieldId entries from geometra_form_schema' };
2075
+ const fieldById = new Map();
2076
+ for (const schema of buildFormSchemas(a11y, { includeOptions: true, includeContext: 'always' })) {
2077
+ for (const field of schema.fields)
2078
+ fieldById.set(field.id, field);
2079
+ }
2080
+ const resolved = [];
2081
+ for (const field of fields) {
2082
+ if (isResolvedFillFieldInput(field)) {
2083
+ if (field.kind === 'choice' && field.fieldId && field.choiceType === undefined) {
2084
+ const schemaField = fieldById.get(field.fieldId);
2085
+ resolved.push({
2086
+ ...field,
2087
+ ...(schemaField?.kind === 'choice' && schemaField.choiceType ? { choiceType: schemaField.choiceType } : {}),
2088
+ });
2089
+ }
2090
+ else if (field.kind === 'toggle' && field.fieldId && field.controlType === undefined) {
2091
+ const schemaField = fieldById.get(field.fieldId);
2092
+ resolved.push({
2093
+ ...field,
2094
+ ...(schemaField?.kind === 'toggle' && schemaField.controlType ? { controlType: schemaField.controlType } : {}),
2095
+ });
2096
+ }
2097
+ else {
2098
+ resolved.push(field);
2099
+ }
2100
+ continue;
2101
+ }
2102
+ if (!field.fieldId) {
2103
+ return {
2104
+ ok: false,
2105
+ error: field.kind === 'toggle'
2106
+ ? 'Toggle fields require label, or provide fieldId from geometra_form_schema so MCP can resolve the current label.'
2107
+ : `${field.kind} fields require fieldLabel, or provide fieldId from geometra_form_schema so MCP can resolve the current label.`,
2108
+ };
2109
+ }
2110
+ const schemaField = fieldById.get(field.fieldId);
2111
+ if (!schemaField) {
2112
+ return { ok: false, error: `Unknown form field id ${field.fieldId}. Refresh geometra_form_schema and try again.` };
2113
+ }
2114
+ if (field.kind === 'text') {
2115
+ if (schemaField.kind !== 'text') {
2116
+ return { ok: false, error: `Field id ${field.fieldId} resolves to kind "${schemaField.kind}", not text.` };
2117
+ }
2118
+ resolved.push({ ...field, fieldLabel: schemaField.label });
2119
+ continue;
2120
+ }
2121
+ if (field.kind === 'choice') {
2122
+ if (schemaField.kind !== 'choice') {
2123
+ return { ok: false, error: `Field id ${field.fieldId} resolves to kind "${schemaField.kind}", not choice.` };
2124
+ }
2125
+ resolved.push({
2126
+ ...field,
2127
+ fieldLabel: schemaField.label,
2128
+ ...(field.choiceType === undefined && schemaField.choiceType ? { choiceType: schemaField.choiceType } : {}),
2129
+ });
2130
+ continue;
2131
+ }
2132
+ if (field.kind === 'toggle') {
2133
+ if (schemaField.kind !== 'toggle') {
2134
+ return { ok: false, error: `Field id ${field.fieldId} resolves to kind "${schemaField.kind}", not toggle.` };
2135
+ }
2136
+ resolved.push({
2137
+ ...field,
2138
+ label: schemaField.label,
2139
+ ...(field.controlType === undefined && schemaField.controlType ? { controlType: schemaField.controlType } : {}),
2140
+ });
2141
+ continue;
2142
+ }
2143
+ return {
2144
+ ok: false,
2145
+ error: `File field id ${field.fieldId} still needs fieldLabel. geometra_form_schema does not reliably expose proxy file inputs yet.`,
2146
+ };
2147
+ }
2148
+ return { ok: true, fields: resolved };
2149
+ }
1957
2150
  function canFallbackToSequentialFill(error) {
1958
2151
  const message = error instanceof Error ? error.message : String(error);
2152
+ try {
2153
+ const parsed = JSON.parse(message);
2154
+ if (parsed?.error === 'listboxPick' ||
2155
+ parsed?.error === 'setFieldText' ||
2156
+ parsed?.error === 'setFieldChoice' ||
2157
+ parsed?.error === 'setChecked' ||
2158
+ parsed?.error === 'attachFiles') {
2159
+ return true;
2160
+ }
2161
+ }
2162
+ catch {
2163
+ /* ignore non-JSON errors */
2164
+ }
1959
2165
  return (message.includes('Unsupported client message type "fillFields"') ||
1960
2166
  message.includes('Client message type "fillFields" is not supported') ||
1961
2167
  message.startsWith('setFieldText:') ||
@@ -2270,9 +2476,12 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2270
2476
  };
2271
2477
  }
2272
2478
  case 'fill_fields': {
2479
+ const resolvedFields = resolveFillFieldInputs(session, action.fields);
2480
+ if (!resolvedFields.ok)
2481
+ throw new Error(resolvedFields.error);
2273
2482
  const steps = [];
2274
- for (let index = 0; index < action.fields.length; index++) {
2275
- const field = action.fields[index];
2483
+ for (let index = 0; index < resolvedFields.fields.length; index++) {
2484
+ const field = resolvedFields.fields[index];
2276
2485
  const result = await executeFillField(session, field, detail);
2277
2486
  steps.push(detail === 'verbose'
2278
2487
  ? { index, kind: field.kind, ok: true, summary: result.summary }
@@ -2281,7 +2490,7 @@ async function executeBatchAction(session, action, detail, includeSteps) {
2281
2490
  return {
2282
2491
  summary: steps.map(step => String(step.summary ?? '')).filter(Boolean).join('\n'),
2283
2492
  compact: {
2284
- fieldCount: action.fields.length,
2493
+ fieldCount: resolvedFields.fields.length,
2285
2494
  ...(includeSteps ? { steps } : {}),
2286
2495
  },
2287
2496
  };
package/dist/session.d.ts CHANGED
@@ -271,6 +271,23 @@ export interface FormSchemaModel {
271
271
  invalidCount: number;
272
272
  fields: FormSchemaField[];
273
273
  }
274
+ export interface FormRequiredFieldSnapshot extends FormSchemaField {
275
+ bounds: {
276
+ x: number;
277
+ y: number;
278
+ width: number;
279
+ height: number;
280
+ };
281
+ visibility: NodeVisibilityModel;
282
+ scrollHint: NodeScrollHintModel;
283
+ }
284
+ export interface FormRequiredSnapshotModel {
285
+ formId: string;
286
+ name?: string;
287
+ requiredCount: number;
288
+ invalidCount: number;
289
+ fields: FormRequiredFieldSnapshot[];
290
+ }
274
291
  export interface FormSchemaBuildOptions {
275
292
  formId?: string;
276
293
  maxFields?: number;
@@ -510,6 +527,12 @@ export declare function buildPageModel(root: A11yNode, options?: {
510
527
  maxSectionsPerKind?: number;
511
528
  }): PageModel;
512
529
  export declare function buildFormSchemas(root: A11yNode, options?: FormSchemaBuildOptions): FormSchemaModel[];
530
+ /**
531
+ * Required-field snapshot for automation: every required field in a form, including
532
+ * offscreen entries, annotated with visibility and scroll hints so agents do not
533
+ * mistake long-form fields for missing controls.
534
+ */
535
+ export declare function buildFormRequiredSnapshot(root: A11yNode, options?: Pick<FormSchemaBuildOptions, 'formId' | 'maxFields' | 'includeOptions' | 'includeContext'>): FormRequiredSnapshotModel[];
513
536
  /**
514
537
  * Expand a page-model section by stable ID into richer, on-demand details.
515
538
  */
package/dist/session.js CHANGED
@@ -755,6 +755,12 @@ export function nodeIdForPath(path) {
755
755
  function formFieldIdForPath(path) {
756
756
  return `ff:${encodePath(path)}`;
757
757
  }
758
+ function parseFormFieldId(id) {
759
+ const [prefix, encoded] = id.split(':', 2);
760
+ if (prefix !== 'ff' || !encoded)
761
+ return null;
762
+ return decodePath(encoded);
763
+ }
758
764
  function sectionPrefix(kind) {
759
765
  if (kind === 'landmark')
760
766
  return 'lm';
@@ -1596,6 +1602,45 @@ export function buildFormSchemas(root, options) {
1596
1602
  .filter(form => !options?.formId || sectionIdForPath('form', form.path) === options.formId)
1597
1603
  .map(form => buildFormSchemaForNode(root, form, options));
1598
1604
  }
1605
+ /**
1606
+ * Required-field snapshot for automation: every required field in a form, including
1607
+ * offscreen entries, annotated with visibility and scroll hints so agents do not
1608
+ * mistake long-form fields for missing controls.
1609
+ */
1610
+ export function buildFormRequiredSnapshot(root, options) {
1611
+ const schemas = buildFormSchemas(root, {
1612
+ formId: options?.formId,
1613
+ maxFields: options?.maxFields,
1614
+ onlyRequiredFields: true,
1615
+ includeOptions: options?.includeOptions,
1616
+ includeContext: options?.includeContext,
1617
+ });
1618
+ return schemas.map(schema => {
1619
+ const parsedForm = parseSectionId(schema.formId);
1620
+ const formNode = parsedForm ? findNodeByPath(root, parsedForm.path) : null;
1621
+ const fields = schema.fields
1622
+ .map(field => {
1623
+ const fieldPath = parseFormFieldId(field.id);
1624
+ const target = fieldPath ? findNodeByPath(root, fieldPath) ?? formNode : formNode;
1625
+ if (!target)
1626
+ return null;
1627
+ return {
1628
+ ...field,
1629
+ bounds: cloneBounds(target.bounds),
1630
+ visibility: buildVisibility(target.bounds, root.bounds),
1631
+ scrollHint: buildScrollHint(target.bounds, root.bounds),
1632
+ };
1633
+ })
1634
+ .filter((field) => field !== null);
1635
+ return {
1636
+ formId: schema.formId,
1637
+ ...(schema.name ? { name: schema.name } : {}),
1638
+ requiredCount: schema.requiredCount,
1639
+ invalidCount: schema.invalidCount,
1640
+ fields,
1641
+ };
1642
+ });
1643
+ }
1599
1644
  function headingModels(node, maxHeadings, includeBounds) {
1600
1645
  const headings = sortByBounds(collectDescendants(node, candidate => candidate.role === 'heading' && !!sanitizeInlineName(candidate.name, 80)));
1601
1646
  return headings.slice(0, maxHeadings).map(heading => ({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@geometra/mcp",
3
- "version": "1.19.20",
3
+ "version": "1.19.22",
4
4
  "description": "MCP server for Geometra — interact with running Geometra apps via the geometry protocol, no browser needed",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,7 +30,7 @@
30
30
  "ui-testing"
31
31
  ],
32
32
  "dependencies": {
33
- "@geometra/proxy": "^1.19.20",
33
+ "@geometra/proxy": "^1.19.22",
34
34
  "@modelcontextprotocol/sdk": "^1.12.1",
35
35
  "ws": "^8.18.0",
36
36
  "zod": "^3.23.0"