@geometra/mcp 1.19.9 → 1.19.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -19,13 +19,16 @@ Geometra proxy: Chromium → DOM geometry → same WebSocket as native →
19
19
  | Tool | Description |
20
20
  |---|---|
21
21
  | `geometra_connect` | Connect with `url` (ws://…) **or** `pageUrl` (https://…) to auto-start geometra-proxy; `url: "https://…"` is auto-coerced onto the proxy path |
22
- | `geometra_query` | Find elements by stable id, role, name, or text content |
22
+ | `geometra_query` | Find elements by stable id, role, name, text content, current value, or semantic state such as `invalid`, `required`, or `busy` |
23
+ | `geometra_wait_for` | Wait for a semantic condition instead of guessing sleeps (`busy`, `disabled`, alerts, values, etc.) |
24
+ | `geometra_fill_fields` | Fill labeled text/choice/toggle/file fields in one MCP call |
25
+ | `geometra_run_actions` | Execute a batch of high-level actions in one MCP round trip and get one consolidated result |
23
26
  | `geometra_page_model` | Summary-first webpage model: archetypes, stable section ids, counts, top-level sections, primary actions |
24
27
  | `geometra_expand_section` | Expand one form/dialog/list/landmark from `geometra_page_model` on demand |
25
28
  | `geometra_click` | Click an element by coordinates |
26
29
  | `geometra_type` | Type text into the focused element |
27
30
  | `geometra_key` | Send special keys (Enter, Tab, Escape, arrows) |
28
- | `geometra_upload_files` | Attach files: auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
31
+ | `geometra_upload_files` | Attach files: labeled field / auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
29
32
  | `geometra_pick_listbox_option` | Pick an option from a custom dropdown/searchable combobox; can open by field label (`@geometra/proxy` only) |
30
33
  | `geometra_select_option` | Choose an option on a native `<select>` (`@geometra/proxy` only) |
31
34
  | `geometra_set_checked` | Set a checkbox or radio by label instead of coordinate clicks (`@geometra/proxy` only) |
@@ -173,4 +176,43 @@ Agent: geometra_query({ role: "button", name: "Save" })
173
176
  8. Tools expose query, click, type, snapshot, page-model, and section-expansion operations over this structured data.
174
177
  9. After each interaction, the peer sends updated geometry (full `frame` or `patch`) — the MCP tools interpret that into compact summaries.
175
178
 
179
+ ## Long Forms
180
+
181
+ For long application flows, prefer one of these patterns:
182
+
183
+ 1. `geometra_page_model`
184
+ 2. `geometra_expand_section`
185
+ 3. `geometra_fill_fields` for obvious field entry
186
+ 4. `geometra_run_actions` when you need mixed navigation + waits + field entry
187
+
188
+ Typical batch:
189
+
190
+ ```json
191
+ {
192
+ "actions": [
193
+ { "type": "click", "x": 412, "y": 228 },
194
+ { "type": "type", "text": "Taylor Applicant" },
195
+ { "type": "upload_files", "paths": ["/Users/you/resume.pdf"], "fieldLabel": "Resume" },
196
+ { "type": "wait_for", "text": "Parsing your resume", "present": false, "timeoutMs": 10000 }
197
+ ]
198
+ }
199
+ ```
200
+
201
+ Single action tools now default to terse summaries. Pass `detail: "verbose"` when you need a fuller current-UI fallback for debugging.
202
+
203
+ Typical field fill:
204
+
205
+ ```json
206
+ {
207
+ "fields": [
208
+ { "kind": "text", "fieldLabel": "Full name", "value": "Taylor Applicant" },
209
+ { "kind": "text", "fieldLabel": "Email", "value": "taylor@example.com" },
210
+ { "kind": "choice", "fieldLabel": "Country", "value": "Germany" },
211
+ { "kind": "choice", "fieldLabel": "Will you require sponsorship?", "value": "No" },
212
+ { "kind": "file", "fieldLabel": "Resume", "paths": ["/Users/you/resume.pdf"] }
213
+ ],
214
+ "failOnInvalid": true
215
+ }
216
+ ```
217
+
176
218
  With a **native** Geometra server, layout comes from Textura/Yoga. With **`@geometra/proxy`**, layout comes from the browser’s computed DOM geometry; the MCP layer is the same.
@@ -6,6 +6,7 @@ function node(role, bounds, options) {
6
6
  ...(options?.name ? { name: options.name } : {}),
7
7
  ...(options?.value ? { value: options.value } : {}),
8
8
  ...(options?.state ? { state: options.state } : {}),
9
+ ...(options?.validation ? { validation: options.validation } : {}),
9
10
  bounds,
10
11
  path: options?.path ?? [],
11
12
  children: options?.children ?? [],
@@ -22,8 +23,15 @@ describe('findNodes', () => {
22
23
  value: 'Austin, Texas, United States',
23
24
  focusable: true,
24
25
  }),
25
- node('checkbox', { x: 20, y: 72, width: 24, height: 24 }, {
26
+ node('textbox', { x: 20, y: 120, width: 260, height: 36 }, {
26
27
  path: [1],
28
+ name: 'Email',
29
+ focusable: true,
30
+ state: { invalid: true, required: true, busy: true },
31
+ validation: { error: 'Please enter a valid email address.' },
32
+ }),
33
+ node('checkbox', { x: 20, y: 72, width: 24, height: 24 }, {
34
+ path: [2],
27
35
  name: 'Notion Website',
28
36
  state: { checked: true },
29
37
  focusable: true,
@@ -36,6 +44,12 @@ describe('findNodes', () => {
36
44
  expect(findNodes(tree, { text: 'United States' })).toEqual([
37
45
  expect.objectContaining({ role: 'combobox', value: 'Austin, Texas, United States' }),
38
46
  ]);
47
+ expect(findNodes(tree, { invalid: true, required: true, busy: true })).toEqual([
48
+ expect.objectContaining({ role: 'textbox', name: 'Email' }),
49
+ ]);
50
+ expect(findNodes(tree, { text: 'valid email address' })).toEqual([
51
+ expect.objectContaining({ role: 'textbox', name: 'Email' }),
52
+ ]);
39
53
  expect(findNodes(tree, { role: 'checkbox', checked: true })).toEqual([
40
54
  expect.objectContaining({ name: 'Notion Website' }),
41
55
  ]);
@@ -1,11 +1,12 @@
1
1
  import { describe, expect, it } from 'vitest';
2
- import { buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
2
+ import { buildA11yTree, buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
3
3
  function node(role, name, bounds, options) {
4
4
  return {
5
5
  role,
6
6
  ...(name ? { name } : {}),
7
7
  ...(options?.value ? { value: options.value } : {}),
8
8
  ...(options?.state ? { state: options.state } : {}),
9
+ ...(options?.validation ? { validation: options.validation } : {}),
9
10
  ...(options?.meta ? { meta: options.meta } : {}),
10
11
  bounds,
11
12
  path: options?.path ?? [],
@@ -87,10 +88,13 @@ describe('buildPageModel', () => {
87
88
  node('textbox', 'Full name*', { x: 60, y: 160, width: 300, height: 36 }, {
88
89
  path: [0, 0, 1],
89
90
  value: 'Taylor Applicant',
91
+ state: { required: true },
90
92
  }),
91
93
  node('textbox', 'Email:', { x: 60, y: 208, width: 300, height: 36 }, {
92
94
  path: [0, 0, 2],
93
95
  value: 'taylor@example.com',
96
+ state: { invalid: true, required: true },
97
+ validation: { error: 'Please enter a valid email address.' },
94
98
  }),
95
99
  node('button', 'Submit application', { x: 60, y: 264, width: 180, height: 40 }, {
96
100
  path: [0, 0, 3],
@@ -116,6 +120,9 @@ describe('buildPageModel', () => {
116
120
  });
117
121
  expect(detail?.fields.map(field => field.name)).toEqual(['Full name', 'Email']);
118
122
  expect(detail?.fields.map(field => field.value)).toEqual(['Taylor Applicant', 'taylor@example.com']);
123
+ expect(detail?.fields[0]?.state).toEqual({ required: true });
124
+ expect(detail?.fields[1]?.state).toEqual({ invalid: true, required: true });
125
+ expect(detail?.fields[1]?.validation).toEqual({ error: 'Please enter a valid email address.' });
119
126
  expect(detail?.actions.map(action => action.id)).toEqual(['n:0.0.3']);
120
127
  expect(detail?.fields[0]).not.toHaveProperty('bounds');
121
128
  });
@@ -324,3 +331,53 @@ describe('buildUiDelta', () => {
324
331
  expect(summarizeUiDelta(delta)).toContain('value "Austin" -> "Austin, Texas, United States"');
325
332
  });
326
333
  });
334
+ describe('buildA11yTree', () => {
335
+ it('maps required, invalid, busy, and validation text from raw semantic nodes', () => {
336
+ const tree = {
337
+ kind: 'box',
338
+ props: {},
339
+ semantic: {},
340
+ children: [
341
+ {
342
+ kind: 'box',
343
+ props: { value: '' },
344
+ semantic: {
345
+ role: 'textbox',
346
+ ariaLabel: 'Email',
347
+ ariaRequired: true,
348
+ ariaInvalid: true,
349
+ ariaBusy: true,
350
+ validationDescription: 'We will contact you about this role.',
351
+ validationError: 'Please enter a valid email address.',
352
+ },
353
+ handlers: { onClick: true, onKeyDown: true },
354
+ },
355
+ ],
356
+ };
357
+ const layout = {
358
+ x: 0,
359
+ y: 0,
360
+ width: 800,
361
+ height: 600,
362
+ children: [
363
+ {
364
+ x: 24,
365
+ y: 40,
366
+ width: 320,
367
+ height: 36,
368
+ children: [],
369
+ },
370
+ ],
371
+ };
372
+ const a11y = buildA11yTree(tree, layout);
373
+ expect(a11y.children[0]).toMatchObject({
374
+ role: 'textbox',
375
+ name: 'Email',
376
+ state: { required: true, invalid: true, busy: true },
377
+ validation: {
378
+ description: 'We will contact you about this role.',
379
+ error: 'Please enter a valid email address.',
380
+ },
381
+ });
382
+ });
383
+ });
package/dist/server.d.ts CHANGED
@@ -12,6 +12,9 @@ interface NodeFilter {
12
12
  focused?: boolean;
13
13
  selected?: boolean;
14
14
  expanded?: boolean;
15
+ invalid?: boolean;
16
+ required?: boolean;
17
+ busy?: boolean;
15
18
  }
16
19
  export declare function createServer(): McpServer;
17
20
  export declare function findNodes(node: A11yNode, filter: NodeFilter): A11yNode[];
package/dist/server.js CHANGED
@@ -1,15 +1,151 @@
1
1
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  import { z } from 'zod';
3
3
  import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
4
- import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendType, sendKey, sendFileUpload, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
4
+ import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
5
5
  function checkedStateInput() {
6
6
  return z
7
7
  .union([z.boolean(), z.literal('mixed')])
8
8
  .optional()
9
9
  .describe('Match checked state (`true`, `false`, or `mixed`)');
10
10
  }
11
+ function detailInput() {
12
+ return z
13
+ .enum(['minimal', 'verbose'])
14
+ .optional()
15
+ .default('minimal')
16
+ .describe('`minimal` (default) returns terse action summaries. Use `verbose` for a fuller current-UI fallback.');
17
+ }
18
+ function nodeFilterShape() {
19
+ return {
20
+ id: z.string().optional().describe('Stable node id from geometra_snapshot or geometra_expand_section'),
21
+ role: z.string().optional().describe('ARIA role to match'),
22
+ name: z.string().optional().describe('Accessible name to match (exact or substring)'),
23
+ text: z.string().optional().describe('Text content to search for (substring match)'),
24
+ value: z.string().optional().describe('Displayed / current field value to match (substring match)'),
25
+ checked: checkedStateInput(),
26
+ disabled: z.boolean().optional().describe('Match disabled state'),
27
+ focused: z.boolean().optional().describe('Match focused state'),
28
+ selected: z.boolean().optional().describe('Match selected state'),
29
+ expanded: z.boolean().optional().describe('Match expanded state'),
30
+ invalid: z.boolean().optional().describe('Match invalid / failed-validation state'),
31
+ required: z.boolean().optional().describe('Match required-field state'),
32
+ busy: z.boolean().optional().describe('Match busy / in-progress state'),
33
+ };
34
+ }
35
+ const timeoutMsInput = z.number().int().min(50).max(60_000).optional();
36
+ const fillFieldSchema = z.discriminatedUnion('kind', [
37
+ z.object({
38
+ kind: z.literal('text'),
39
+ fieldLabel: z.string().describe('Visible field label / accessible name'),
40
+ value: z.string().describe('Text value to set'),
41
+ exact: z.boolean().optional().describe('Exact label match'),
42
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
43
+ }),
44
+ z.object({
45
+ kind: z.literal('choice'),
46
+ fieldLabel: z.string().describe('Visible field label / accessible name'),
47
+ value: z.string().describe('Desired option value / answer label'),
48
+ query: z.string().optional().describe('Optional search text for searchable comboboxes'),
49
+ exact: z.boolean().optional().describe('Exact label match'),
50
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
51
+ }),
52
+ z.object({
53
+ kind: z.literal('toggle'),
54
+ label: z.string().describe('Visible checkbox/radio label to set'),
55
+ checked: z.boolean().optional().default(true).describe('Desired checked state (default true)'),
56
+ exact: z.boolean().optional().describe('Exact label match'),
57
+ controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
58
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
59
+ }),
60
+ z.object({
61
+ kind: z.literal('file'),
62
+ fieldLabel: z.string().describe('Visible file-field label / accessible name'),
63
+ paths: z.array(z.string()).min(1).describe('Absolute paths on the proxy machine'),
64
+ exact: z.boolean().optional().describe('Exact label match'),
65
+ timeoutMs: timeoutMsInput.describe('Optional action wait timeout'),
66
+ }),
67
+ ]);
68
+ const batchActionSchema = z.discriminatedUnion('type', [
69
+ z.object({
70
+ type: z.literal('click'),
71
+ x: z.number(),
72
+ y: z.number(),
73
+ timeoutMs: timeoutMsInput,
74
+ }),
75
+ z.object({
76
+ type: z.literal('type'),
77
+ text: z.string(),
78
+ timeoutMs: timeoutMsInput,
79
+ }),
80
+ z.object({
81
+ type: z.literal('key'),
82
+ key: z.string(),
83
+ shift: z.boolean().optional(),
84
+ ctrl: z.boolean().optional(),
85
+ meta: z.boolean().optional(),
86
+ alt: z.boolean().optional(),
87
+ timeoutMs: timeoutMsInput,
88
+ }),
89
+ z.object({
90
+ type: z.literal('upload_files'),
91
+ paths: z.array(z.string()).min(1),
92
+ x: z.number().optional(),
93
+ y: z.number().optional(),
94
+ fieldLabel: z.string().optional(),
95
+ exact: z.boolean().optional(),
96
+ strategy: z.enum(['auto', 'chooser', 'hidden', 'drop']).optional(),
97
+ dropX: z.number().optional(),
98
+ dropY: z.number().optional(),
99
+ timeoutMs: timeoutMsInput,
100
+ }),
101
+ z.object({
102
+ type: z.literal('pick_listbox_option'),
103
+ label: z.string(),
104
+ exact: z.boolean().optional(),
105
+ openX: z.number().optional(),
106
+ openY: z.number().optional(),
107
+ fieldLabel: z.string().optional(),
108
+ query: z.string().optional(),
109
+ timeoutMs: timeoutMsInput,
110
+ }),
111
+ z.object({
112
+ type: z.literal('select_option'),
113
+ x: z.number(),
114
+ y: z.number(),
115
+ value: z.string().optional(),
116
+ label: z.string().optional(),
117
+ index: z.number().int().min(0).optional(),
118
+ timeoutMs: timeoutMsInput,
119
+ }),
120
+ z.object({
121
+ type: z.literal('set_checked'),
122
+ label: z.string(),
123
+ checked: z.boolean().optional(),
124
+ exact: z.boolean().optional(),
125
+ controlType: z.enum(['checkbox', 'radio']).optional(),
126
+ timeoutMs: timeoutMsInput,
127
+ }),
128
+ z.object({
129
+ type: z.literal('wheel'),
130
+ deltaY: z.number(),
131
+ deltaX: z.number().optional(),
132
+ x: z.number().optional(),
133
+ y: z.number().optional(),
134
+ timeoutMs: timeoutMsInput,
135
+ }),
136
+ z.object({
137
+ type: z.literal('wait_for'),
138
+ ...nodeFilterShape(),
139
+ present: z.boolean().optional(),
140
+ timeoutMs: timeoutMsInput,
141
+ }),
142
+ z.object({
143
+ type: z.literal('fill_fields'),
144
+ fields: z.array(fillFieldSchema).min(1).max(80),
145
+ }),
146
+ ]);
11
147
  export function createServer() {
12
- const server = new McpServer({ name: 'geometra', version: '1.19.9' }, { capabilities: { tools: {} } });
148
+ const server = new McpServer({ name: 'geometra', version: '1.19.10' }, { capabilities: { tools: {} } });
13
149
  // ── connect ──────────────────────────────────────────────────
14
150
  server.tool('geometra_connect', `Connect to a Geometra WebSocket peer, or start \`geometra-proxy\` automatically for a normal web page.
15
151
 
@@ -77,23 +213,26 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
77
213
  // ── query ────────────────────────────────────────────────────
78
214
  server.tool('geometra_query', `Find elements in the current Geometra UI by stable id, role, name, text content, current value, or semantic state. Returns matching elements with their exact pixel bounds {x, y, width, height}, visible in-viewport bounds, an on-screen center point, visibility / scroll-reveal hints, role, name, value, state, and tree path.
79
215
 
80
- This is the Geometra equivalent of Playwright's locator — but instant, structured, and with no browser. Use the returned bounds to click elements or assert on layout.`, {
81
- id: z.string().optional().describe('Stable node id from geometra_snapshot or geometra_expand_section'),
82
- role: z.string().optional().describe('ARIA role to match (e.g. "button", "textbox", "text", "heading", "listitem")'),
83
- name: z.string().optional().describe('Accessible name to match (exact or substring)'),
84
- text: z.string().optional().describe('Text content to search for (substring match)'),
85
- value: z.string().optional().describe('Displayed / current field value to match (substring match)'),
86
- checked: checkedStateInput(),
87
- disabled: z.boolean().optional().describe('Match disabled state'),
88
- focused: z.boolean().optional().describe('Match focused state'),
89
- selected: z.boolean().optional().describe('Match selected state'),
90
- expanded: z.boolean().optional().describe('Match expanded state'),
91
- }, async ({ id, role, name, text, value, checked, disabled, focused, selected, expanded }) => {
216
+ This is the Geometra equivalent of Playwright's locator — but instant, structured, and with no browser. Use the returned bounds to click elements or assert on layout.`, nodeFilterShape(), async ({ id, role, name, text, value, checked, disabled, focused, selected, expanded, invalid, required, busy }) => {
92
217
  const session = getSession();
93
218
  if (!session?.tree || !session?.layout)
94
219
  return err('Not connected. Call geometra_connect first.');
95
220
  const a11y = buildA11yTree(session.tree, session.layout);
96
- const filter = { id, role, name, text, value, checked, disabled, focused, selected, expanded };
221
+ const filter = {
222
+ id,
223
+ role,
224
+ name,
225
+ text,
226
+ value,
227
+ checked,
228
+ disabled,
229
+ focused,
230
+ selected,
231
+ expanded,
232
+ invalid,
233
+ required,
234
+ busy,
235
+ };
97
236
  if (!hasNodeFilter(filter))
98
237
  return err('Provide at least one query filter (id, role, name, text, value, or state)');
99
238
  const matches = findNodes(a11y, filter);
@@ -106,16 +245,7 @@ This is the Geometra equivalent of Playwright's locator — but instant, structu
106
245
  server.tool('geometra_wait_for', `Wait for a semantic UI condition without guessing sleep durations. Use this for slow SPA transitions, resume parsing, custom validation alerts, disabled submit buttons, and value/state confirmation before submit.
107
246
 
108
247
  The filter matches the same fields as geometra_query. Set \`present: false\` to wait for something to disappear (for example an alert or a "Parsing…" status).`, {
109
- id: z.string().optional().describe('Stable node id from geometra_snapshot or geometra_expand_section'),
110
- role: z.string().optional().describe('ARIA role to match'),
111
- name: z.string().optional().describe('Accessible name to match (exact or substring)'),
112
- text: z.string().optional().describe('Text content to search for (substring match)'),
113
- value: z.string().optional().describe('Displayed / current field value to match (substring match)'),
114
- checked: checkedStateInput(),
115
- disabled: z.boolean().optional().describe('Match disabled state'),
116
- focused: z.boolean().optional().describe('Match focused state'),
117
- selected: z.boolean().optional().describe('Match selected state'),
118
- expanded: z.boolean().optional().describe('Match expanded state'),
248
+ ...nodeFilterShape(),
119
249
  present: z.boolean().optional().default(true).describe('Wait for a matching node to exist (default true) or disappear'),
120
250
  timeoutMs: z
121
251
  .number()
@@ -125,11 +255,25 @@ The filter matches the same fields as geometra_query. Set \`present: false\` to
125
255
  .optional()
126
256
  .default(10_000)
127
257
  .describe('Maximum time to wait before returning an error (default 10000ms)'),
128
- }, async ({ id, role, name, text, value, checked, disabled, focused, selected, expanded, present, timeoutMs }) => {
258
+ }, async ({ id, role, name, text, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs }) => {
129
259
  const session = getSession();
130
260
  if (!session?.tree || !session?.layout)
131
261
  return err('Not connected. Call geometra_connect first.');
132
- const filter = { id, role, name, text, value, checked, disabled, focused, selected, expanded };
262
+ const filter = {
263
+ id,
264
+ role,
265
+ name,
266
+ text,
267
+ value,
268
+ checked,
269
+ disabled,
270
+ focused,
271
+ selected,
272
+ expanded,
273
+ invalid,
274
+ required,
275
+ busy,
276
+ };
133
277
  if (!hasNodeFilter(filter))
134
278
  return err('Provide at least one wait filter (id, role, name, text, value, or state)');
135
279
  const matchesCondition = () => {
@@ -155,6 +299,90 @@ The filter matches the same fields as geometra_query. Set \`present: false\` to
155
299
  const result = matches.slice(0, 8).map(node => formatNode(node, after.bounds));
156
300
  return ok(JSON.stringify(result, null, 2));
157
301
  });
302
+ server.tool('geometra_fill_fields', `Fill several labeled form fields in one MCP call. This is the preferred high-level primitive for long forms.
303
+
304
+ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / comboboxes / radio-style questions addressed by field label + answer, \`"toggle"\` for individually labeled checkboxes or radios, and \`"file"\` for labeled uploads.`, {
305
+ fields: z.array(fillFieldSchema).min(1).max(80).describe('Ordered labeled field operations to apply'),
306
+ stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing field (default true)'),
307
+ failOnInvalid: z
308
+ .boolean()
309
+ .optional()
310
+ .default(false)
311
+ .describe('Return an error if invalid fields remain after filling'),
312
+ detail: detailInput(),
313
+ }, async ({ fields, stopOnError, failOnInvalid, detail }) => {
314
+ const session = getSession();
315
+ if (!session)
316
+ return err('Not connected. Call geometra_connect first.');
317
+ const steps = [];
318
+ let stoppedAt;
319
+ for (let index = 0; index < fields.length; index++) {
320
+ const field = fields[index];
321
+ try {
322
+ const summary = await executeFillField(session, field, detail);
323
+ steps.push({ index, kind: field.kind, ok: true, summary });
324
+ }
325
+ catch (e) {
326
+ const message = e instanceof Error ? e.message : String(e);
327
+ steps.push({ index, kind: field.kind, ok: false, error: message });
328
+ if (stopOnError) {
329
+ stoppedAt = index;
330
+ break;
331
+ }
332
+ }
333
+ }
334
+ const after = sessionA11y(session);
335
+ const signals = after ? collectSessionSignals(after) : undefined;
336
+ const invalidRemaining = signals?.invalidFields.length ?? 0;
337
+ const payload = {
338
+ completed: stoppedAt === undefined && steps.length === fields.length,
339
+ fieldCount: fields.length,
340
+ steps,
341
+ ...(stoppedAt !== undefined ? { stoppedAt } : {}),
342
+ ...(signals ? { final: sessionSignalsPayload(signals) } : {}),
343
+ };
344
+ if (failOnInvalid && invalidRemaining > 0) {
345
+ return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
346
+ }
347
+ return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
348
+ });
349
+ server.tool('geometra_run_actions', `Execute several Geometra actions in one MCP round trip and return one consolidated result. This is the preferred path for long, multi-step form fills where one-tool-per-field would otherwise create too much chatter.
350
+
351
+ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_listbox_option\`, \`select_option\`, \`set_checked\`, \`wheel\`, \`wait_for\`, and \`fill_fields\`.`, {
352
+ actions: z.array(batchActionSchema).min(1).max(80).describe('Ordered high-level action steps to run sequentially'),
353
+ stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing step (default true)'),
354
+ detail: detailInput(),
355
+ }, async ({ actions, stopOnError, detail }) => {
356
+ const session = getSession();
357
+ if (!session)
358
+ return err('Not connected. Call geometra_connect first.');
359
+ const steps = [];
360
+ let stoppedAt;
361
+ for (let index = 0; index < actions.length; index++) {
362
+ const action = actions[index];
363
+ try {
364
+ const summary = await executeBatchAction(session, action, detail);
365
+ steps.push({ index, type: action.type, ok: true, summary });
366
+ }
367
+ catch (e) {
368
+ const message = e instanceof Error ? e.message : String(e);
369
+ steps.push({ index, type: action.type, ok: false, error: message });
370
+ if (stopOnError) {
371
+ stoppedAt = index;
372
+ break;
373
+ }
374
+ }
375
+ }
376
+ const after = sessionA11y(session);
377
+ const payload = {
378
+ completed: stoppedAt === undefined && steps.length === actions.length,
379
+ stepCount: actions.length,
380
+ steps,
381
+ ...(stoppedAt !== undefined ? { stoppedAt } : {}),
382
+ ...(after ? { final: sessionSignalsPayload(collectSessionSignals(after)) } : {}),
383
+ };
384
+ return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
385
+ });
158
386
  // ── page model ────────────────────────────────────────────────
159
387
  server.tool('geometra_page_model', `Get a higher-level webpage summary instead of a raw node dump. Returns stable section ids, page archetypes, summary counts, top-level landmarks/forms/dialogs/lists, and a few primary actions.
160
388
 
@@ -225,13 +453,14 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
225
453
  .max(60_000)
226
454
  .optional()
227
455
  .describe('Optional action wait timeout (use a longer value for slow submits or route transitions)'),
228
- }, async ({ x, y, timeoutMs }) => {
456
+ detail: detailInput(),
457
+ }, async ({ x, y, timeoutMs, detail }) => {
229
458
  const session = getSession();
230
459
  if (!session)
231
460
  return err('Not connected. Call geometra_connect first.');
232
461
  const before = sessionA11y(session);
233
462
  const wait = await sendClick(session, x, y, timeoutMs);
234
- const summary = postActionSummary(session, before, wait);
463
+ const summary = postActionSummary(session, before, wait, detail);
235
464
  return ok(`Clicked at (${x}, ${y}).\n${summary}`);
236
465
  });
237
466
  // ── type ─────────────────────────────────────────────────────
@@ -246,13 +475,14 @@ Each character is sent as a key event through the geometry protocol. Returns a c
246
475
  .max(60_000)
247
476
  .optional()
248
477
  .describe('Optional action wait timeout'),
249
- }, async ({ text, timeoutMs }) => {
478
+ detail: detailInput(),
479
+ }, async ({ text, timeoutMs, detail }) => {
250
480
  const session = getSession();
251
481
  if (!session)
252
482
  return err('Not connected. Call geometra_connect first.');
253
483
  const before = sessionA11y(session);
254
484
  const wait = await sendType(session, text, timeoutMs);
255
- const summary = postActionSummary(session, before, wait);
485
+ const summary = postActionSummary(session, before, wait, detail);
256
486
  return ok(`Typed "${text}".\n${summary}`);
257
487
  });
258
488
  // ── key ──────────────────────────────────────────────────────
@@ -269,22 +499,25 @@ Each character is sent as a key event through the geometry protocol. Returns a c
269
499
  .max(60_000)
270
500
  .optional()
271
501
  .describe('Optional action wait timeout'),
272
- }, async ({ key, shift, ctrl, meta, alt, timeoutMs }) => {
502
+ detail: detailInput(),
503
+ }, async ({ key, shift, ctrl, meta, alt, timeoutMs, detail }) => {
273
504
  const session = getSession();
274
505
  if (!session)
275
506
  return err('Not connected. Call geometra_connect first.');
276
507
  const before = sessionA11y(session);
277
508
  const wait = await sendKey(session, key, { shift, ctrl, meta, alt }, timeoutMs);
278
- const summary = postActionSummary(session, before, wait);
509
+ const summary = postActionSummary(session, before, wait, detail);
279
510
  return ok(`Pressed ${formatKeyCombo(key, { shift, ctrl, meta, alt })}.\n${summary}`);
280
511
  });
281
512
  // ── upload files (proxy) ───────────────────────────────────────
282
513
  server.tool('geometra_upload_files', `Attach local files to a file input. Requires \`@geometra/proxy\` (paths exist on the proxy host).
283
514
 
284
- Strategies: **auto** (default) tries chooser click if x,y given, else hidden \`input[type=file]\`, else first visible file input. **hidden** targets hidden inputs directly. **drop** needs dropX,dropY for drag-target zones. **chooser** requires x,y.`, {
515
+ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled file input when \`fieldLabel\` is provided, else hidden \`input[type=file]\`, else first visible file input. **hidden** targets hidden inputs directly. **drop** needs dropX,dropY for drag-target zones. **chooser** requires x,y.`, {
285
516
  paths: z.array(z.string()).min(1).describe('Absolute paths on the proxy machine, e.g. /Users/you/resume.pdf'),
286
517
  x: z.number().optional().describe('Click X to trigger native file chooser'),
287
518
  y: z.number().optional().describe('Click Y to trigger native file chooser'),
519
+ fieldLabel: z.string().optional().describe('Prefer a specific labeled file field (for example "Resume" or "Cover letter")'),
520
+ exact: z.boolean().optional().describe('Exact match when using fieldLabel'),
288
521
  strategy: z
289
522
  .enum(['auto', 'chooser', 'hidden', 'drop'])
290
523
  .optional()
@@ -298,7 +531,8 @@ Strategies: **auto** (default) tries chooser click if x,y given, else hidden \`i
298
531
  .max(60_000)
299
532
  .optional()
300
533
  .describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
301
- }, async ({ paths, x, y, strategy, dropX, dropY, timeoutMs }) => {
534
+ detail: detailInput(),
535
+ }, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, timeoutMs, detail }) => {
302
536
  const session = getSession();
303
537
  if (!session)
304
538
  return err('Not connected. Call geometra_connect first.');
@@ -306,10 +540,12 @@ Strategies: **auto** (default) tries chooser click if x,y given, else hidden \`i
306
540
  try {
307
541
  const wait = await sendFileUpload(session, paths, {
308
542
  click: x !== undefined && y !== undefined ? { x, y } : undefined,
543
+ fieldLabel,
544
+ exact,
309
545
  strategy,
310
546
  drop: dropX !== undefined && dropY !== undefined ? { x: dropX, y: dropY } : undefined,
311
547
  }, timeoutMs ?? 8_000);
312
- const summary = postActionSummary(session, before, wait);
548
+ const summary = postActionSummary(session, before, wait, detail);
313
549
  return ok(`Uploaded ${paths.length} file(s).\n${summary}`);
314
550
  }
315
551
  catch (e) {
@@ -332,7 +568,8 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
332
568
  .max(60_000)
333
569
  .optional()
334
570
  .describe('Optional action wait timeout for slow dropdowns / remote search results'),
335
- }, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs }) => {
571
+ detail: detailInput(),
572
+ }, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs, detail }) => {
336
573
  const session = getSession();
337
574
  if (!session)
338
575
  return err('Not connected. Call geometra_connect first.');
@@ -344,7 +581,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
344
581
  fieldLabel,
345
582
  query,
346
583
  }, timeoutMs);
347
- const summary = postActionSummary(session, before, wait);
584
+ const summary = postActionSummary(session, before, wait, detail);
348
585
  const fieldSummary = fieldLabel ? summarizeFieldLabelState(session, fieldLabel) : undefined;
349
586
  return ok([
350
587
  `Picked listbox option "${label}".`,
@@ -372,7 +609,8 @@ Custom React/Vue dropdowns are not supported — open them with geometra_click a
372
609
  .max(60_000)
373
610
  .optional()
374
611
  .describe('Optional action wait timeout'),
375
- }, async ({ x, y, value, label, index, timeoutMs }) => {
612
+ detail: detailInput(),
613
+ }, async ({ x, y, value, label, index, timeoutMs, detail }) => {
376
614
  const session = getSession();
377
615
  if (!session)
378
616
  return err('Not connected. Call geometra_connect first.');
@@ -382,7 +620,7 @@ Custom React/Vue dropdowns are not supported — open them with geometra_click a
382
620
  const before = sessionA11y(session);
383
621
  try {
384
622
  const wait = await sendSelectOption(session, x, y, { value, label, index }, timeoutMs);
385
- const summary = postActionSummary(session, before, wait);
623
+ const summary = postActionSummary(session, before, wait, detail);
386
624
  return ok(`Selected option.\n${summary}`);
387
625
  }
388
626
  catch (e) {
@@ -403,14 +641,15 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
403
641
  .max(60_000)
404
642
  .optional()
405
643
  .describe('Optional action wait timeout'),
406
- }, async ({ label, checked, exact, controlType, timeoutMs }) => {
644
+ detail: detailInput(),
645
+ }, async ({ label, checked, exact, controlType, timeoutMs, detail }) => {
407
646
  const session = getSession();
408
647
  if (!session)
409
648
  return err('Not connected. Call geometra_connect first.');
410
649
  const before = sessionA11y(session);
411
650
  try {
412
651
  const wait = await sendSetChecked(session, label, { checked, exact, controlType }, timeoutMs);
413
- const summary = postActionSummary(session, before, wait);
652
+ const summary = postActionSummary(session, before, wait, detail);
414
653
  return ok(`Set ${controlType ?? 'checkbox/radio'} "${label}" to ${String(checked ?? true)}.\n${summary}`);
415
654
  }
416
655
  catch (e) {
@@ -430,14 +669,15 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
430
669
  .max(60_000)
431
670
  .optional()
432
671
  .describe('Optional action wait timeout'),
433
- }, async ({ deltaY, deltaX, x, y, timeoutMs }) => {
672
+ detail: detailInput(),
673
+ }, async ({ deltaY, deltaX, x, y, timeoutMs, detail }) => {
434
674
  const session = getSession();
435
675
  if (!session)
436
676
  return err('Not connected. Call geometra_connect first.');
437
677
  const before = sessionA11y(session);
438
678
  try {
439
679
  const wait = await sendWheel(session, deltaY, { deltaX, x, y }, timeoutMs);
440
- const summary = postActionSummary(session, before, wait);
680
+ const summary = postActionSummary(session, before, wait, detail);
441
681
  return ok(`Wheel delta (${deltaX ?? 0}, ${deltaY}).\n${summary}`);
442
682
  }
443
683
  catch (e) {
@@ -514,24 +754,37 @@ function sessionOverviewFromA11y(a11y) {
514
754
  const keyNodes = nodes.length > 0 ? `Key nodes:\n${summarizeCompactIndex(nodes, 18)}` : '';
515
755
  return [pageSummary, contextSummary, keyNodes].filter(Boolean).join('\n');
516
756
  }
517
- function postActionSummary(session, before, wait) {
757
+ function postActionSummary(session, before, wait, detail = 'minimal') {
518
758
  const after = sessionA11y(session);
519
759
  const notes = [];
520
760
  if (wait?.status === 'acknowledged') {
521
- notes.push('The peer acknowledged the action quickly; waiting logic did not need to rely on a full frame/patch round-trip.');
761
+ notes.push(detail === 'verbose'
762
+ ? 'The peer acknowledged the action quickly; waiting logic did not need to rely on a full frame/patch round-trip.'
763
+ : 'Peer acknowledged the action quickly.');
522
764
  }
523
765
  if (wait?.status === 'timed_out') {
524
- notes.push(`No frame or patch arrived within ${wait.timeoutMs}ms after the action. The action may still have succeeded if it did not change geometry or semantics.`);
766
+ notes.push(detail === 'verbose'
767
+ ? `No frame or patch arrived within ${wait.timeoutMs}ms after the action. The action may still have succeeded if it did not change geometry or semantics.`
768
+ : `No update arrived within ${wait.timeoutMs}ms; the action may still have succeeded.`);
525
769
  }
526
770
  if (!after)
527
771
  return [...notes, 'No UI update received'].filter(Boolean).join('\n');
772
+ const signals = collectSessionSignals(after);
773
+ const validationSummary = summarizeValidationSignals(signals);
528
774
  if (before) {
529
775
  const delta = buildUiDelta(before, after);
530
776
  if (hasUiDelta(delta)) {
531
- return [...notes, `Changes:\n${summarizeUiDelta(delta)}`].filter(Boolean).join('\n');
777
+ return [
778
+ ...notes,
779
+ `Changes:\n${summarizeUiDelta(delta, detail === 'verbose' ? 14 : 8)}`,
780
+ ...(detail === 'minimal' ? validationSummary : []),
781
+ ].filter(Boolean).join('\n');
532
782
  }
533
783
  }
534
- return [...notes, `Current UI:\n${sessionOverviewFromA11y(after)}`].filter(Boolean).join('\n');
784
+ if (detail === 'verbose') {
785
+ return [...notes, `Current UI:\n${sessionOverviewFromA11y(after)}`].filter(Boolean).join('\n');
786
+ }
787
+ return [...notes, summarizeSessionSignals(signals), ...validationSummary].filter(Boolean).join('\n');
535
788
  }
536
789
  function summarizeCompactContext(context) {
537
790
  const parts = [];
@@ -546,6 +799,283 @@ function summarizeCompactContext(context) {
546
799
  }
547
800
  return parts.length > 0 ? `Context: ${parts.join(' | ')}` : '';
548
801
  }
802
+ function collectSessionSignals(root) {
803
+ const signals = {
804
+ ...(root.meta?.pageUrl ? { pageUrl: root.meta.pageUrl } : {}),
805
+ ...(typeof root.meta?.scrollX === 'number' ? { scrollX: root.meta.scrollX } : {}),
806
+ ...(typeof root.meta?.scrollY === 'number' ? { scrollY: root.meta.scrollY } : {}),
807
+ dialogCount: 0,
808
+ busyCount: 0,
809
+ alerts: [],
810
+ invalidFields: [],
811
+ };
812
+ const seenAlerts = new Set();
813
+ const seenInvalidIds = new Set();
814
+ function walk(node) {
815
+ if (!signals.focus && node.state?.focused) {
816
+ signals.focus = {
817
+ id: nodeIdForPath(node.path),
818
+ role: node.role,
819
+ ...(node.name ? { name: node.name } : {}),
820
+ ...(node.value ? { value: node.value } : {}),
821
+ };
822
+ }
823
+ if (node.role === 'dialog' || node.role === 'alertdialog')
824
+ signals.dialogCount++;
825
+ if (node.state?.busy)
826
+ signals.busyCount++;
827
+ if (node.role === 'alert' || node.role === 'alertdialog') {
828
+ const text = truncateInlineText(node.name ?? node.validation?.error, 120);
829
+ if (text && !seenAlerts.has(text)) {
830
+ seenAlerts.add(text);
831
+ signals.alerts.push(text);
832
+ }
833
+ }
834
+ if ((node.role === 'textbox' || node.role === 'combobox' || node.role === 'checkbox' || node.role === 'radio') && node.state?.invalid) {
835
+ const id = nodeIdForPath(node.path);
836
+ if (!seenInvalidIds.has(id)) {
837
+ seenInvalidIds.add(id);
838
+ signals.invalidFields.push({
839
+ id,
840
+ role: node.role,
841
+ ...(node.name ? { name: truncateInlineText(node.name, 80) } : {}),
842
+ ...(node.validation?.error ? { error: truncateInlineText(node.validation.error, 120) } : {}),
843
+ });
844
+ }
845
+ }
846
+ for (const child of node.children)
847
+ walk(child);
848
+ }
849
+ walk(root);
850
+ return signals;
851
+ }
852
+ function summarizeSessionSignals(signals) {
853
+ const contextParts = [];
854
+ if (signals.pageUrl)
855
+ contextParts.push(`url=${signals.pageUrl}`);
856
+ if (signals.scrollX !== undefined || signals.scrollY !== undefined) {
857
+ contextParts.push(`scroll=(${signals.scrollX ?? 0},${signals.scrollY ?? 0})`);
858
+ }
859
+ if (signals.focus) {
860
+ const focusName = signals.focus.name ? ` "${truncateInlineText(signals.focus.name, 48)}"` : '';
861
+ const focusValue = signals.focus.value ? ` value=${JSON.stringify(truncateInlineText(signals.focus.value, 40))}` : '';
862
+ contextParts.push(`focus=${signals.focus.role}${focusName}${focusValue}`);
863
+ }
864
+ const statusParts = [
865
+ signals.dialogCount > 0 ? `dialogs=${signals.dialogCount}` : undefined,
866
+ signals.alerts.length > 0 ? `alerts=${signals.alerts.length}` : undefined,
867
+ signals.invalidFields.length > 0 ? `invalid=${signals.invalidFields.length}` : undefined,
868
+ signals.busyCount > 0 ? `busy=${signals.busyCount}` : undefined,
869
+ ].filter(Boolean);
870
+ return [
871
+ contextParts.length > 0 ? `Context: ${contextParts.join(' | ')}` : undefined,
872
+ statusParts.length > 0 ? `Status: ${statusParts.join(' | ')}` : 'Status: no semantic changes detected.',
873
+ ].filter(Boolean).join('\n');
874
+ }
875
+ function summarizeValidationSignals(signals) {
876
+ const lines = [];
877
+ if (signals.alerts.length > 0) {
878
+ lines.push(`Alerts: ${signals.alerts.slice(0, 2).map(text => JSON.stringify(text)).join(' | ')}`);
879
+ }
880
+ if (signals.invalidFields.length > 0) {
881
+ const invalidSummary = signals.invalidFields
882
+ .slice(0, 4)
883
+ .map(field => {
884
+ const label = field.name ? `"${field.name}"` : field.id;
885
+ return field.error ? `${label}: ${JSON.stringify(field.error)}` : label;
886
+ })
887
+ .join(' | ');
888
+ lines.push(`Validation: ${invalidSummary}`);
889
+ }
890
+ return lines;
891
+ }
892
+ function truncateInlineText(text, max) {
893
+ if (!text)
894
+ return undefined;
895
+ const normalized = text.replace(/\s+/g, ' ').trim();
896
+ if (!normalized)
897
+ return undefined;
898
+ return normalized.length > max ? `${normalized.slice(0, max - 1)}…` : normalized;
899
+ }
900
+ function sessionSignalsPayload(signals) {
901
+ return {
902
+ ...(signals.pageUrl ? { pageUrl: signals.pageUrl } : {}),
903
+ ...(signals.scrollX !== undefined || signals.scrollY !== undefined
904
+ ? { scroll: { x: signals.scrollX ?? 0, y: signals.scrollY ?? 0 } }
905
+ : {}),
906
+ ...(signals.focus ? { focus: signals.focus } : {}),
907
+ dialogCount: signals.dialogCount,
908
+ busyCount: signals.busyCount,
909
+ alerts: signals.alerts,
910
+ invalidFields: signals.invalidFields,
911
+ };
912
+ }
913
+ async function executeBatchAction(session, action, detail) {
914
+ switch (action.type) {
915
+ case 'click': {
916
+ const before = sessionA11y(session);
917
+ const wait = await sendClick(session, action.x, action.y, action.timeoutMs);
918
+ return `Clicked at (${action.x}, ${action.y}).\n${postActionSummary(session, before, wait, detail)}`;
919
+ }
920
+ case 'type': {
921
+ const before = sessionA11y(session);
922
+ const wait = await sendType(session, action.text, action.timeoutMs);
923
+ return `Typed "${action.text}".\n${postActionSummary(session, before, wait, detail)}`;
924
+ }
925
+ case 'key': {
926
+ const before = sessionA11y(session);
927
+ const wait = await sendKey(session, action.key, { shift: action.shift, ctrl: action.ctrl, meta: action.meta, alt: action.alt }, action.timeoutMs);
928
+ return `Pressed ${formatKeyCombo(action.key, action)}.\n${postActionSummary(session, before, wait, detail)}`;
929
+ }
930
+ case 'upload_files': {
931
+ const before = sessionA11y(session);
932
+ const wait = await sendFileUpload(session, action.paths, {
933
+ click: action.x !== undefined && action.y !== undefined ? { x: action.x, y: action.y } : undefined,
934
+ fieldLabel: action.fieldLabel,
935
+ exact: action.exact,
936
+ strategy: action.strategy,
937
+ drop: action.dropX !== undefined && action.dropY !== undefined ? { x: action.dropX, y: action.dropY } : undefined,
938
+ }, action.timeoutMs ?? 8_000);
939
+ return `Uploaded ${action.paths.length} file(s).\n${postActionSummary(session, before, wait, detail)}`;
940
+ }
941
+ case 'pick_listbox_option': {
942
+ const before = sessionA11y(session);
943
+ const wait = await sendListboxPick(session, action.label, {
944
+ exact: action.exact,
945
+ open: action.openX !== undefined && action.openY !== undefined ? { x: action.openX, y: action.openY } : undefined,
946
+ fieldLabel: action.fieldLabel,
947
+ query: action.query,
948
+ }, action.timeoutMs);
949
+ const summary = postActionSummary(session, before, wait, detail);
950
+ const fieldSummary = action.fieldLabel ? summarizeFieldLabelState(session, action.fieldLabel) : undefined;
951
+ return [`Picked listbox option "${action.label}".`, fieldSummary, summary].filter(Boolean).join('\n');
952
+ }
953
+ case 'select_option': {
954
+ if (action.value === undefined && action.label === undefined && action.index === undefined) {
955
+ throw new Error('select_option step requires at least one of value, label, or index');
956
+ }
957
+ const before = sessionA11y(session);
958
+ const wait = await sendSelectOption(session, action.x, action.y, {
959
+ value: action.value,
960
+ label: action.label,
961
+ index: action.index,
962
+ }, action.timeoutMs);
963
+ return `Selected option.\n${postActionSummary(session, before, wait, detail)}`;
964
+ }
965
+ case 'set_checked': {
966
+ const before = sessionA11y(session);
967
+ const wait = await sendSetChecked(session, action.label, {
968
+ checked: action.checked,
969
+ exact: action.exact,
970
+ controlType: action.controlType,
971
+ }, action.timeoutMs);
972
+ return `Set ${action.controlType ?? 'checkbox/radio'} "${action.label}" to ${String(action.checked ?? true)}.\n${postActionSummary(session, before, wait, detail)}`;
973
+ }
974
+ case 'wheel': {
975
+ const before = sessionA11y(session);
976
+ const wait = await sendWheel(session, action.deltaY, {
977
+ deltaX: action.deltaX,
978
+ x: action.x,
979
+ y: action.y,
980
+ }, action.timeoutMs);
981
+ return `Wheel delta (${action.deltaX ?? 0}, ${action.deltaY}).\n${postActionSummary(session, before, wait, detail)}`;
982
+ }
983
+ case 'wait_for': {
984
+ if (!session.tree || !session.layout)
985
+ throw new Error('Not connected. Call geometra_connect first.');
986
+ const filter = {
987
+ id: action.id,
988
+ role: action.role,
989
+ name: action.name,
990
+ text: action.text,
991
+ value: action.value,
992
+ checked: action.checked,
993
+ disabled: action.disabled,
994
+ focused: action.focused,
995
+ selected: action.selected,
996
+ expanded: action.expanded,
997
+ invalid: action.invalid,
998
+ required: action.required,
999
+ busy: action.busy,
1000
+ };
1001
+ if (!hasNodeFilter(filter)) {
1002
+ throw new Error('wait_for step requires at least one filter');
1003
+ }
1004
+ const present = action.present ?? true;
1005
+ const timeoutMs = action.timeoutMs ?? 10_000;
1006
+ const startedAt = Date.now();
1007
+ const matched = await waitForUiCondition(session, () => {
1008
+ if (!session.tree || !session.layout)
1009
+ return false;
1010
+ const a11y = buildA11yTree(session.tree, session.layout);
1011
+ const matches = findNodes(a11y, filter);
1012
+ return present ? matches.length > 0 : matches.length === 0;
1013
+ }, timeoutMs);
1014
+ const elapsedMs = Date.now() - startedAt;
1015
+ if (!matched) {
1016
+ throw new Error(`Timed out after ${timeoutMs}ms waiting for ${present ? 'presence' : 'absence'} of ${JSON.stringify(filter)}`);
1017
+ }
1018
+ if (!present) {
1019
+ return `Condition satisfied after ${elapsedMs}ms: no nodes matched ${JSON.stringify(filter)}.`;
1020
+ }
1021
+ const after = sessionA11y(session);
1022
+ if (!after) {
1023
+ return `Condition satisfied after ${elapsedMs}ms for ${JSON.stringify(filter)}.`;
1024
+ }
1025
+ const matches = findNodes(after, filter);
1026
+ if (detail === 'verbose') {
1027
+ return JSON.stringify(matches.slice(0, 8).map(node => formatNode(node, after.bounds)), null, 2);
1028
+ }
1029
+ return `Condition satisfied after ${elapsedMs}ms with ${matches.length} matching node(s).`;
1030
+ }
1031
+ case 'fill_fields': {
1032
+ const lines = [];
1033
+ for (const field of action.fields) {
1034
+ lines.push(await executeFillField(session, field, detail));
1035
+ }
1036
+ return lines.join('\n');
1037
+ }
1038
+ }
1039
+ }
1040
+ async function executeFillField(session, field, detail) {
1041
+ switch (field.kind) {
1042
+ case 'text': {
1043
+ const before = sessionA11y(session);
1044
+ const wait = await sendFieldText(session, field.fieldLabel, field.value, { exact: field.exact }, field.timeoutMs);
1045
+ const fieldSummary = summarizeFieldLabelState(session, field.fieldLabel);
1046
+ return [
1047
+ `Filled text field "${field.fieldLabel}".`,
1048
+ fieldSummary,
1049
+ postActionSummary(session, before, wait, detail),
1050
+ ].filter(Boolean).join('\n');
1051
+ }
1052
+ case 'choice': {
1053
+ const before = sessionA11y(session);
1054
+ const wait = await sendFieldChoice(session, field.fieldLabel, field.value, { exact: field.exact, query: field.query }, field.timeoutMs);
1055
+ const fieldSummary = summarizeFieldLabelState(session, field.fieldLabel);
1056
+ return [
1057
+ `Set choice field "${field.fieldLabel}" to "${field.value}".`,
1058
+ fieldSummary,
1059
+ postActionSummary(session, before, wait, detail),
1060
+ ].filter(Boolean).join('\n');
1061
+ }
1062
+ case 'toggle': {
1063
+ const before = sessionA11y(session);
1064
+ const wait = await sendSetChecked(session, field.label, { checked: field.checked, exact: field.exact, controlType: field.controlType }, field.timeoutMs);
1065
+ return `Set ${field.controlType ?? 'checkbox/radio'} "${field.label}" to ${String(field.checked ?? true)}.\n${postActionSummary(session, before, wait, detail)}`;
1066
+ }
1067
+ case 'file': {
1068
+ const before = sessionA11y(session);
1069
+ const wait = await sendFileUpload(session, field.paths, { fieldLabel: field.fieldLabel, exact: field.exact }, field.timeoutMs ?? 8_000);
1070
+ const fieldSummary = summarizeFieldLabelState(session, field.fieldLabel);
1071
+ return [
1072
+ `Uploaded ${field.paths.length} file(s) to "${field.fieldLabel}".`,
1073
+ fieldSummary,
1074
+ postActionSummary(session, before, wait, detail),
1075
+ ].filter(Boolean).join('\n');
1076
+ }
1077
+ }
1078
+ }
549
1079
  function ok(text) {
550
1080
  return { content: [{ type: 'text', text }] };
551
1081
  }
@@ -571,7 +1101,8 @@ function nodeMatchesFilter(node, filter) {
571
1101
  return false;
572
1102
  if (!textMatches(node.value, filter.value))
573
1103
  return false;
574
- if (filter.text && !textMatches(`${node.name ?? ''} ${node.value ?? ''}`.trim(), filter.text))
1104
+ if (filter.text &&
1105
+ !textMatches(`${node.name ?? ''} ${node.value ?? ''} ${node.validation?.error ?? ''} ${node.validation?.description ?? ''}`.trim(), filter.text))
575
1106
  return false;
576
1107
  if (filter.checked !== undefined && node.state?.checked !== filter.checked)
577
1108
  return false;
@@ -583,6 +1114,12 @@ function nodeMatchesFilter(node, filter) {
583
1114
  return false;
584
1115
  if (filter.expanded !== undefined && (node.state?.expanded ?? false) !== filter.expanded)
585
1116
  return false;
1117
+ if (filter.invalid !== undefined && (node.state?.invalid ?? false) !== filter.invalid)
1118
+ return false;
1119
+ if (filter.required !== undefined && (node.state?.required ?? false) !== filter.required)
1120
+ return false;
1121
+ if (filter.busy !== undefined && (node.state?.busy ?? false) !== filter.busy)
1122
+ return false;
586
1123
  return true;
587
1124
  }
588
1125
  export function findNodes(node, filter) {
@@ -618,6 +1155,8 @@ function summarizeFieldLabelState(session, fieldLabel) {
618
1155
  parts.push(`value=${JSON.stringify(match.value)}`);
619
1156
  if (match.state && Object.keys(match.state).length > 0)
620
1157
  parts.push(`state=${JSON.stringify(match.state)}`);
1158
+ if (match.validation?.error)
1159
+ parts.push(`error=${JSON.stringify(match.validation.error)}`);
621
1160
  return parts.join(' ');
622
1161
  }
623
1162
  function formatNode(node, viewport) {
@@ -669,6 +1208,7 @@ function formatNode(node, viewport) {
669
1208
  },
670
1209
  focusable: node.focusable,
671
1210
  ...(node.state && Object.keys(node.state).length > 0 ? { state: node.state } : {}),
1211
+ ...(node.validation && Object.keys(node.validation).length > 0 ? { validation: node.validation } : {}),
672
1212
  path: node.path,
673
1213
  };
674
1214
  }
package/dist/session.d.ts CHANGED
@@ -15,6 +15,13 @@ export interface A11yNode {
15
15
  selected?: boolean;
16
16
  checked?: boolean | 'mixed';
17
17
  focused?: boolean;
18
+ invalid?: boolean;
19
+ required?: boolean;
20
+ busy?: boolean;
21
+ };
22
+ validation?: {
23
+ description?: string;
24
+ error?: string;
18
25
  };
19
26
  meta?: {
20
27
  pageUrl?: string;
@@ -128,6 +135,7 @@ export interface PageFieldModel {
128
135
  name?: string;
129
136
  value?: string;
130
137
  state?: A11yNode['state'];
138
+ validation?: A11yNode['validation'];
131
139
  bounds?: {
132
140
  x: number;
133
141
  y: number;
@@ -280,12 +288,23 @@ export declare function sendFileUpload(session: Session, paths: string[], opts?:
280
288
  x: number;
281
289
  y: number;
282
290
  };
291
+ fieldLabel?: string;
292
+ exact?: boolean;
283
293
  strategy?: 'auto' | 'chooser' | 'hidden' | 'drop';
284
294
  drop?: {
285
295
  x: number;
286
296
  y: number;
287
297
  };
288
298
  }, timeoutMs?: number): Promise<UpdateWaitResult>;
299
+ /** Set a labeled text-like field (`input`, `textarea`, contenteditable, ARIA textbox) semantically. */
300
+ export declare function sendFieldText(session: Session, fieldLabel: string, value: string, opts?: {
301
+ exact?: boolean;
302
+ }, timeoutMs?: number): Promise<UpdateWaitResult>;
303
+ /** Choose a value for a labeled choice field (select, custom combobox, or radio-style group). */
304
+ export declare function sendFieldChoice(session: Session, fieldLabel: string, value: string, opts?: {
305
+ exact?: boolean;
306
+ query?: string;
307
+ }, timeoutMs?: number): Promise<UpdateWaitResult>;
289
308
  /** ARIA `role=option` listbox (e.g. React Select). Optional click opens the list. */
290
309
  export declare function sendListboxPick(session: Session, label: string, opts?: {
291
310
  exact?: boolean;
package/dist/session.js CHANGED
@@ -227,6 +227,10 @@ export function sendFileUpload(session, paths, opts, timeoutMs) {
227
227
  payload.x = opts.click.x;
228
228
  payload.y = opts.click.y;
229
229
  }
230
+ if (opts?.fieldLabel)
231
+ payload.fieldLabel = opts.fieldLabel;
232
+ if (opts?.exact !== undefined)
233
+ payload.exact = opts.exact;
230
234
  if (opts?.strategy)
231
235
  payload.strategy = opts.strategy;
232
236
  if (opts?.drop) {
@@ -235,6 +239,30 @@ export function sendFileUpload(session, paths, opts, timeoutMs) {
235
239
  }
236
240
  return sendAndWaitForUpdate(session, payload, timeoutMs);
237
241
  }
242
+ /** Set a labeled text-like field (`input`, `textarea`, contenteditable, ARIA textbox) semantically. */
243
+ export function sendFieldText(session, fieldLabel, value, opts, timeoutMs) {
244
+ const payload = {
245
+ type: 'setFieldText',
246
+ fieldLabel,
247
+ value,
248
+ };
249
+ if (opts?.exact !== undefined)
250
+ payload.exact = opts.exact;
251
+ return sendAndWaitForUpdate(session, payload, timeoutMs);
252
+ }
253
+ /** Choose a value for a labeled choice field (select, custom combobox, or radio-style group). */
254
+ export function sendFieldChoice(session, fieldLabel, value, opts, timeoutMs = LISTBOX_UPDATE_TIMEOUT_MS) {
255
+ const payload = {
256
+ type: 'setFieldChoice',
257
+ fieldLabel,
258
+ value,
259
+ };
260
+ if (opts?.exact !== undefined)
261
+ payload.exact = opts.exact;
262
+ if (opts?.query)
263
+ payload.query = opts.query;
264
+ return sendAndWaitForUpdate(session, payload, timeoutMs);
265
+ }
238
266
  /** ARIA `role=option` listbox (e.g. React Select). Optional click opens the list. */
239
267
  export function sendListboxPick(session, label, opts, timeoutMs = LISTBOX_UPDATE_TIMEOUT_MS) {
240
268
  const payload = { type: 'listboxPick', label };
@@ -588,6 +616,22 @@ function cloneState(state) {
588
616
  next.checked = state.checked;
589
617
  if (state.focused !== undefined)
590
618
  next.focused = state.focused;
619
+ if (state.invalid !== undefined)
620
+ next.invalid = state.invalid;
621
+ if (state.required !== undefined)
622
+ next.required = state.required;
623
+ if (state.busy !== undefined)
624
+ next.busy = state.busy;
625
+ return Object.keys(next).length > 0 ? next : undefined;
626
+ }
627
+ function cloneValidation(validation) {
628
+ if (!validation)
629
+ return undefined;
630
+ const next = {};
631
+ if (validation.description)
632
+ next.description = validation.description;
633
+ if (validation.error)
634
+ next.error = validation.error;
591
635
  return Object.keys(next).length > 0 ? next : undefined;
592
636
  }
593
637
  function clonePath(path) {
@@ -707,6 +751,7 @@ function toFieldModel(node, includeBounds = true) {
707
751
  ...(fieldLabel(node) ? { name: fieldLabel(node) } : {}),
708
752
  ...(value ? { value } : {}),
709
753
  ...(cloneState(node.state) ? { state: cloneState(node.state) } : {}),
754
+ ...(cloneValidation(node.validation) ? { validation: cloneValidation(node.validation) } : {}),
710
755
  ...(includeBounds ? { bounds: cloneBounds(node.bounds) } : {}),
711
756
  };
712
757
  }
@@ -982,7 +1027,7 @@ function diffCompactNodes(before, after) {
982
1027
  }
983
1028
  const beforeState = before.state ?? {};
984
1029
  const afterState = after.state ?? {};
985
- for (const key of ['disabled', 'expanded', 'selected', 'checked', 'focused']) {
1030
+ for (const key of ['disabled', 'expanded', 'selected', 'checked', 'focused', 'invalid', 'required', 'busy']) {
986
1031
  if (beforeState[key] !== afterState[key]) {
987
1032
  changes.push(`${key} ${formatStateValue(beforeState[key])} -> ${formatStateValue(afterState[key])}`);
988
1033
  }
@@ -1216,6 +1261,19 @@ function walkNode(element, layout, path) {
1216
1261
  state.checked = checked;
1217
1262
  if (semantic?.focused !== undefined)
1218
1263
  state.focused = !!semantic.focused;
1264
+ if (semantic?.ariaInvalid !== undefined)
1265
+ state.invalid = !!semantic.ariaInvalid;
1266
+ if (semantic?.ariaRequired !== undefined)
1267
+ state.required = !!semantic.ariaRequired;
1268
+ if (semantic?.ariaBusy !== undefined)
1269
+ state.busy = !!semantic.ariaBusy;
1270
+ const validation = {};
1271
+ if (typeof semantic?.validationDescription === 'string' && semantic.validationDescription.trim().length > 0) {
1272
+ validation.description = semantic.validationDescription;
1273
+ }
1274
+ if (typeof semantic?.validationError === 'string' && semantic.validationError.trim().length > 0) {
1275
+ validation.error = semantic.validationError;
1276
+ }
1219
1277
  const meta = {};
1220
1278
  if (typeof semantic?.pageUrl === 'string')
1221
1279
  meta.pageUrl = semantic.pageUrl;
@@ -1238,6 +1296,7 @@ function walkNode(element, layout, path) {
1238
1296
  ...(name ? { name } : {}),
1239
1297
  ...(value ? { value } : {}),
1240
1298
  ...(Object.keys(state).length > 0 ? { state } : {}),
1299
+ ...(Object.keys(validation).length > 0 ? { validation } : {}),
1241
1300
  ...(Object.keys(meta).length > 0 ? { meta } : {}),
1242
1301
  bounds,
1243
1302
  path,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@geometra/mcp",
3
- "version": "1.19.9",
3
+ "version": "1.19.10",
4
4
  "description": "MCP server for Geometra — interact with running Geometra apps via the geometry protocol, no browser needed",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,7 +30,7 @@
30
30
  "ui-testing"
31
31
  ],
32
32
  "dependencies": {
33
- "@geometra/proxy": "^1.19.9",
33
+ "@geometra/proxy": "^1.19.10",
34
34
  "@modelcontextprotocol/sdk": "^1.12.1",
35
35
  "ws": "^8.18.0",
36
36
  "zod": "^3.23.0"