@geometra/mcp 1.19.8 → 1.19.10

package/README.md

@@ -19,13 +19,16 @@ Geometra proxy:       Chromium → DOM geometry → same WebSocket as native →
 | Tool | Description |
 |---|---|
 | `geometra_connect` | Connect with `url` (ws://…) **or** `pageUrl` (https://…) to auto-start geometra-proxy; `url: "https://…"` is auto-coerced onto the proxy path |
-| `geometra_query` | Find elements by stable id, role, name, or text content |
+| `geometra_query` | Find elements by stable id, role, name, text content, current value, or semantic state such as `invalid`, `required`, or `busy` |
+| `geometra_wait_for` | Wait for a semantic condition instead of guessing sleeps (`busy`, `disabled`, alerts, values, etc.) |
+| `geometra_fill_fields` | Fill labeled text/choice/toggle/file fields in one MCP call |
+| `geometra_run_actions` | Execute a batch of high-level actions in one MCP round trip and get one consolidated result |
 | `geometra_page_model` | Summary-first webpage model: archetypes, stable section ids, counts, top-level sections, primary actions |
 | `geometra_expand_section` | Expand one form/dialog/list/landmark from `geometra_page_model` on demand |
 | `geometra_click` | Click an element by coordinates |
 | `geometra_type` | Type text into the focused element |
 | `geometra_key` | Send special keys (Enter, Tab, Escape, arrows) |
-| `geometra_upload_files` | Attach files: auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
+| `geometra_upload_files` | Attach files: labeled field / auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
 | `geometra_pick_listbox_option` | Pick an option from a custom dropdown/searchable combobox; can open by field label (`@geometra/proxy` only) |
 | `geometra_select_option` | Choose an option on a native `<select>` (`@geometra/proxy` only) |
 | `geometra_set_checked` | Set a checkbox or radio by label instead of coordinate clicks (`@geometra/proxy` only) |
@@ -173,4 +176,43 @@ Agent:  geometra_query({ role: "button", name: "Save" })
 8. Tools expose query, click, type, snapshot, page-model, and section-expansion operations over this structured data.
 9. After each interaction, the peer sends updated geometry (full `frame` or `patch`) — the MCP tools interpret that into compact summaries.
 
+## Long Forms
+
+For long application flows, prefer one of these patterns:
+
+1. `geometra_page_model`
+2. `geometra_expand_section`
+3. `geometra_fill_fields` for obvious field entry
+4. `geometra_run_actions` when you need mixed navigation + waits + field entry
+
+Typical batch:
+
+```json
+{
+  "actions": [
+    { "type": "click", "x": 412, "y": 228 },
+    { "type": "type", "text": "Taylor Applicant" },
+    { "type": "upload_files", "paths": ["/Users/you/resume.pdf"], "fieldLabel": "Resume" },
+    { "type": "wait_for", "text": "Parsing your resume", "present": false, "timeoutMs": 10000 }
+  ]
+}
+```
+
+Single action tools now default to terse summaries. Pass `detail: "verbose"` when you need a fuller current-UI fallback for debugging.
+
+Typical field fill:
+
+```json
+{
+  "fields": [
+    { "kind": "text", "fieldLabel": "Full name", "value": "Taylor Applicant" },
+    { "kind": "text", "fieldLabel": "Email", "value": "taylor@example.com" },
+    { "kind": "choice", "fieldLabel": "Country", "value": "Germany" },
+    { "kind": "choice", "fieldLabel": "Will you require sponsorship?", "value": "No" },
+    { "kind": "file", "fieldLabel": "Resume", "paths": ["/Users/you/resume.pdf"] }
+  ],
+  "failOnInvalid": true
+}
+```
+
 With a **native** Geometra server, layout comes from Textura/Yoga. With **`@geometra/proxy`**, layout comes from the browser’s computed DOM geometry; the MCP layer is the same.

package/dist/__tests__/proxy-session-actions.test.js

@@ -1,6 +1,6 @@
 import { afterAll, describe, expect, it } from 'vitest';
 import { WebSocketServer } from 'ws';
-import { connect, disconnect, sendListboxPick } from '../session.js';
+import { connect, disconnect, sendClick, sendListboxPick } from '../session.js';
 describe('proxy-backed MCP actions', () => {
     afterAll(() => {
         disconnect();
@@ -56,4 +56,100 @@ describe('proxy-backed MCP actions', () => {
             await new Promise((resolve, reject) => wss.close(err => (err ? reject(err) : resolve())));
         }
     });
+    it('falls back to the latest observed update when a legacy peer does not send request-scoped ack', async () => {
+        const wss = new WebSocketServer({ port: 0 });
+        wss.on('connection', ws => {
+            ws.on('message', raw => {
+                const msg = JSON.parse(String(raw));
+                if (msg.type === 'resize') {
+                    ws.send(JSON.stringify({
+                        type: 'frame',
+                        layout: { x: 0, y: 0, width: 1024, height: 768, children: [] },
+                        tree: { kind: 'box', props: {}, semantic: { tag: 'body', role: 'group' }, children: [] },
+                    }));
+                    return;
+                }
+                if (msg.type === 'event') {
+                    ws.send(JSON.stringify({
+                        type: 'frame',
+                        layout: { x: 0, y: 0, width: 1024, height: 768, children: [] },
+                        tree: { kind: 'box', props: {}, semantic: { tag: 'body', role: 'group', ariaLabel: 'Updated' }, children: [] },
+                    }));
+                }
+            });
+        });
+        const port = await new Promise((resolve, reject) => {
+            wss.once('listening', () => {
+                const address = wss.address();
+                if (typeof address === 'object' && address)
+                    resolve(address.port);
+                else
+                    reject(new Error('Failed to resolve ephemeral WebSocket port'));
+            });
+            wss.once('error', reject);
+        });
+        try {
+            const session = await connect(`ws://127.0.0.1:${port}`);
+            await expect(sendClick(session, 5, 5, 60)).resolves.toMatchObject({ status: 'updated', timeoutMs: 60 });
+        }
+        finally {
+            disconnect();
+            await new Promise((resolve, reject) => wss.close(err => (err ? reject(err) : resolve())));
+        }
+    });
+    it('ignores invalid patch paths instead of mutating ancestor layout nodes', async () => {
+        const wss = new WebSocketServer({ port: 0 });
+        wss.on('connection', ws => {
+            ws.on('message', raw => {
+                const msg = JSON.parse(String(raw));
+                if (msg.type === 'resize') {
+                    ws.send(JSON.stringify({
+                        type: 'frame',
+                        layout: {
+                            x: 0,
+                            y: 0,
+                            width: 200,
+                            height: 100,
+                            children: [{ x: 10, y: 20, width: 30, height: 40, children: [] }],
+                        },
+                        tree: {
+                            kind: 'box',
+                            props: {},
+                            semantic: { tag: 'body', role: 'group' },
+                            children: [{ kind: 'box', props: {}, semantic: { tag: 'div', role: 'group' }, children: [] }],
+                        },
+                    }));
+                    setTimeout(() => {
+                        ws.send(JSON.stringify({
+                            type: 'patch',
+                            patches: [{ path: [9], x: 999, y: 999 }],
+                        }));
+                    }, 10);
+                }
+            });
+        });
+        const port = await new Promise((resolve, reject) => {
+            wss.once('listening', () => {
+                const address = wss.address();
+                if (typeof address === 'object' && address)
+                    resolve(address.port);
+                else
+                    reject(new Error('Failed to resolve ephemeral WebSocket port'));
+            });
+            wss.once('error', reject);
+        });
+        try {
+            const session = await connect(`ws://127.0.0.1:${port}`);
+            await new Promise(resolve => setTimeout(resolve, 30));
+            expect(session.layout).toMatchObject({
+                x: 0,
+                y: 0,
+                children: [{ x: 10, y: 20 }],
+            });
+        }
+        finally {
+            disconnect();
+            await new Promise((resolve, reject) => wss.close(err => (err ? reject(err) : resolve())));
+        }
+    });
 });

package/dist/__tests__/server-filters.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/server-filters.test.js

@@ -0,0 +1,57 @@
+import { describe, expect, it } from 'vitest';
+import { findNodes } from '../server.js';
+function node(role, bounds, options) {
+    return {
+        role,
+        ...(options?.name ? { name: options.name } : {}),
+        ...(options?.value ? { value: options.value } : {}),
+        ...(options?.state ? { state: options.state } : {}),
+        ...(options?.validation ? { validation: options.validation } : {}),
+        bounds,
+        path: options?.path ?? [],
+        children: options?.children ?? [],
+        focusable: options?.focusable ?? false,
+    };
+}
+describe('findNodes', () => {
+    it('matches value and checked-state filters generically', () => {
+        const tree = node('group', { x: 0, y: 0, width: 800, height: 600 }, {
+            children: [
+                node('combobox', { x: 20, y: 20, width: 260, height: 36 }, {
+                    path: [0],
+                    name: 'Location',
+                    value: 'Austin, Texas, United States',
+                    focusable: true,
+                }),
+                node('textbox', { x: 20, y: 120, width: 260, height: 36 }, {
+                    path: [1],
+                    name: 'Email',
+                    focusable: true,
+                    state: { invalid: true, required: true, busy: true },
+                    validation: { error: 'Please enter a valid email address.' },
+                }),
+                node('checkbox', { x: 20, y: 72, width: 24, height: 24 }, {
+                    path: [2],
+                    name: 'Notion Website',
+                    state: { checked: true },
+                    focusable: true,
+                }),
+            ],
+        });
+        expect(findNodes(tree, { value: 'Austin, Texas' })).toEqual([
+            expect.objectContaining({ role: 'combobox', name: 'Location' }),
+        ]);
+        expect(findNodes(tree, { text: 'United States' })).toEqual([
+            expect.objectContaining({ role: 'combobox', value: 'Austin, Texas, United States' }),
+        ]);
+        expect(findNodes(tree, { invalid: true, required: true, busy: true })).toEqual([
+            expect.objectContaining({ role: 'textbox', name: 'Email' }),
+        ]);
+        expect(findNodes(tree, { text: 'valid email address' })).toEqual([
+            expect.objectContaining({ role: 'textbox', name: 'Email' }),
+        ]);
+        expect(findNodes(tree, { role: 'checkbox', checked: true })).toEqual([
+            expect.objectContaining({ name: 'Notion Website' }),
+        ]);
+    });
+});

package/dist/__tests__/session-model.test.js

@@ -1,10 +1,12 @@
 import { describe, expect, it } from 'vitest';
-import { buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
+import { buildA11yTree, buildCompactUiIndex, buildPageModel, expandPageSection, buildUiDelta, hasUiDelta, summarizeUiDelta, } from '../session.js';
 function node(role, name, bounds, options) {
     return {
         role,
         ...(name ? { name } : {}),
+        ...(options?.value ? { value: options.value } : {}),
         ...(options?.state ? { state: options.state } : {}),
+        ...(options?.validation ? { validation: options.validation } : {}),
         ...(options?.meta ? { meta: options.meta } : {}),
         bounds,
         path: options?.path ?? [],
@@ -83,8 +85,17 @@ describe('buildPageModel', () => {
                             path: [0, 0],
                             children: [
                                 node('heading', 'Application', { x: 60, y: 132, width: 200, height: 24 }, { path: [0, 0, 0] }),
-                                node('textbox', 'Full name*', { x: 60, y: 160, width: 300, height: 36 }, { path: [0, 0, 1] }),
-                                node('textbox', 'Email:', { x: 60, y: 208, width: 300, height: 36 }, { path: [0, 0, 2] }),
+                                node('textbox', 'Full name*', { x: 60, y: 160, width: 300, height: 36 }, {
+                                    path: [0, 0, 1],
+                                    value: 'Taylor Applicant',
+                                    state: { required: true },
+                                }),
+                                node('textbox', 'Email:', { x: 60, y: 208, width: 300, height: 36 }, {
+                                    path: [0, 0, 2],
+                                    value: 'taylor@example.com',
+                                    state: { invalid: true, required: true },
+                                    validation: { error: 'Please enter a valid email address.' },
+                                }),
                                 node('button', 'Submit application', { x: 60, y: 264, width: 180, height: 40 }, {
                                     path: [0, 0, 3],
                                     focusable: true,
@@ -108,6 +119,10 @@ describe('buildPageModel', () => {
             },
         });
         expect(detail?.fields.map(field => field.name)).toEqual(['Full name', 'Email']);
+        expect(detail?.fields.map(field => field.value)).toEqual(['Taylor Applicant', 'taylor@example.com']);
+        expect(detail?.fields[0]?.state).toEqual({ required: true });
+        expect(detail?.fields[1]?.state).toEqual({ invalid: true, required: true });
+        expect(detail?.fields[1]?.validation).toEqual({ error: 'Please enter a valid email address.' });
         expect(detail?.actions.map(action => action.id)).toEqual(['n:0.0.3']);
         expect(detail?.fields[0]).not.toHaveProperty('bounds');
     });
@@ -280,4 +295,89 @@ describe('buildUiDelta', () => {
         expect(summary).toContain('~ focus n:1.0 textbox "Full name" -> n:1.1 textbox "Country"');
         expect(summary).toContain('~ navigation "https://jobs.example.com/apply" -> "https://jobs.example.com/apply?step=details"');
     });
+    it('includes control values in compact indexes and semantic deltas', () => {
+        const before = node('group', undefined, { x: 0, y: 0, width: 640, height: 480 }, {
+            children: [
+                node('textbox', 'Location', { x: 20, y: 40, width: 280, height: 36 }, {
+                    path: [0],
+                    focusable: true,
+                    value: 'Austin',
+                }),
+            ],
+        });
+        const after = node('group', undefined, { x: 0, y: 0, width: 640, height: 480 }, {
+            children: [
+                node('textbox', 'Location', { x: 20, y: 40, width: 280, height: 36 }, {
+                    path: [0],
+                    focusable: true,
+                    value: 'Austin, Texas, United States',
+                }),
+            ],
+        });
+        const compact = buildCompactUiIndex(after, { maxNodes: 10 });
+        expect(compact.nodes[0]).toMatchObject({
+            role: 'textbox',
+            name: 'Location',
+            value: 'Austin, Texas, United States',
+        });
+        const delta = buildUiDelta(before, after);
+        expect(delta.updated).toEqual([
+            expect.objectContaining({
+                changes: [
+                    'value "Austin" -> "Austin, Texas, United States"',
+                ],
+            }),
+        ]);
+        expect(summarizeUiDelta(delta)).toContain('value "Austin" -> "Austin, Texas, United States"');
+    });
+});
+describe('buildA11yTree', () => {
+    it('maps required, invalid, busy, and validation text from raw semantic nodes', () => {
+        const tree = {
+            kind: 'box',
+            props: {},
+            semantic: {},
+            children: [
+                {
+                    kind: 'box',
+                    props: { value: '' },
+                    semantic: {
+                        role: 'textbox',
+                        ariaLabel: 'Email',
+                        ariaRequired: true,
+                        ariaInvalid: true,
+                        ariaBusy: true,
+                        validationDescription: 'We will contact you about this role.',
+                        validationError: 'Please enter a valid email address.',
+                    },
+                    handlers: { onClick: true, onKeyDown: true },
+                },
+            ],
+        };
+        const layout = {
+            x: 0,
+            y: 0,
+            width: 800,
+            height: 600,
+            children: [
+                {
+                    x: 24,
+                    y: 40,
+                    width: 320,
+                    height: 36,
+                    children: [],
+                },
+            ],
+        };
+        const a11y = buildA11yTree(tree, layout);
+        expect(a11y.children[0]).toMatchObject({
+            role: 'textbox',
+            name: 'Email',
+            state: { required: true, invalid: true, busy: true },
+            validation: {
+                description: 'We will contact you about this role.',
+                error: 'Please enter a valid email address.',
+            },
+        });
+    });
 });

package/dist/server.d.ts

@@ -1,2 +1,21 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { A11yNode } from './session.js';
+type NodeStateFilterValue = boolean | 'mixed';
+interface NodeFilter {
+    id?: string;
+    role?: string;
+    name?: string;
+    text?: string;
+    value?: string;
+    checked?: NodeStateFilterValue;
+    disabled?: boolean;
+    focused?: boolean;
+    selected?: boolean;
+    expanded?: boolean;
+    invalid?: boolean;
+    required?: boolean;
+    busy?: boolean;
+}
 export declare function createServer(): McpServer;
+export declare function findNodes(node: A11yNode, filter: NodeFilter): A11yNode[];
+export {};