@geometra/mcp 1.19.17 → 1.19.19

package/README.md

@@ -14,11 +14,17 @@ Native Geometra:      WebSocket → JSON geometry (no browser on the agent path)
 Geometra proxy:       Chromium → DOM geometry → same WebSocket as native → MCP tools unchanged (often started via `pageUrl`, no manual CLI)
 ```
 
+## When to choose this
+
+Use Geometra MCP when an LLM needs to explore, interpret, and operate a real UI with compact semantic state instead of repeatedly consuming large browser snapshots. Keep Playwright-style tooling for deterministic scripts, DOM-oriented test automation, and compatibility fallback paths while Geometra closes remaining live-site gaps.
+
+Proxy-backed sessions stay warm by default on disconnect, and MCP now keeps a small warm pool so compatible headed and headless workflows do not immediately evict each other.
+
 ## Tools
 
 | Tool | Description |
 |---|---|
-| `geometra_connect` | Connect with `url` (ws://…) **or** `pageUrl` (https://…) to auto-start geometra-proxy; `url: "https://…"` is auto-coerced onto the proxy path |
+| `geometra_connect` | Connect with `url` (ws://…) **or** `pageUrl` (https://…) to auto-start geometra-proxy; can inline `formSchema` and/or `pageModel` for lower-turn starts |
 | `geometra_query` | Find elements by stable id, role, name, text content, ancestor/prompt context, current value, or semantic state such as `invalid`, `required`, or `busy` |
 | `geometra_wait_for` | Wait for a semantic condition instead of guessing sleeps (`busy`, `disabled`, alerts, values, etc.) |
 | `geometra_form_schema` | Compact, fill-oriented form schema with stable field ids and collapsed radio/button groups; can auto-connect from `pageUrl` / `url` |
@@ -28,7 +34,7 @@ Geometra proxy:       Chromium → DOM geometry → same WebSocket as native →
 | `geometra_page_model` | Summary-first webpage model: archetypes, stable section ids, counts, top-level sections, primary actions |
 | `geometra_expand_section` | Expand one form/dialog/list/landmark from `geometra_page_model` on demand, with paging/filtering for long sections |
 | `geometra_reveal` | Scroll until a matching node is visible instead of guessing wheel deltas |
-| `geometra_click` | Click an element by coordinates |
+| `geometra_click` | Click by coordinates or semantic target, optionally waiting for a post-click semantic condition |
 | `geometra_type` | Type text into the focused element |
 | `geometra_key` | Send special keys (Enter, Tab, Escape, arrows) |
 | `geometra_upload_files` | Attach files: labeled field / auto / hidden input / native chooser / synthetic drop (`@geometra/proxy` only) |
@@ -251,19 +257,17 @@ Then in Claude Code (either backend):
 Agent:  geometra_connect({ url: "ws://localhost:3100" })
         → Connected. UI: button "Sign Up", textbox "Email", textbox "Password"
 
-Agent:  geometra_query({ role: "textbox", name: "Email" })
-        → [{ role: "textbox", name: "Email", bounds: {x:100, y:200, w:300, h:40}, center: {x:250, y:220} }]
-
-Agent:  geometra_click({ x: 250, y: 220 })
-        → Clicked. Email input focused.
+Agent:  geometra_click({ role: "textbox", name: "Email" })
+        → Clicked textbox "Email" (n:0) at (250, 220). Email input focused.
 
 Agent:  geometra_type({ text: "test@example.com" })
         → Typed. Email field updated.
 
-Agent:  geometra_query({ role: "button", name: "Sign Up" })
-        → [{ role: "button", name: "Sign Up", bounds: {x:100, y:350, w:120, h:44}, center: {x:160, y:372} }]
-
-Agent:  geometra_click({ x: 160, y: 372 })
+Agent:  geometra_click({
+          role: "button",
+          name: "Sign Up",
+          waitFor: { role: "dialog", name: "Success" }
+        })
         → Clicked. Success message visible.
 ```
 
@@ -312,18 +316,20 @@ For long application flows, prefer one of these patterns:
 2. otherwise `geometra_form_schema`
 3. then `geometra_fill_form`
 3. `geometra_reveal` for far-below-fold targets such as submit buttons
-4. `geometra_run_actions` when you need mixed navigation + waits + field entry
-5. `geometra_page_model` + `geometra_expand_section` when you are still exploring the page rather than filling it
+4. `geometra_click({ ..., waitFor: ... })` when one action should also wait for the next semantic state
+5. `geometra_run_actions` when you need mixed navigation + waits + field entry
+6. `geometra_connect({ pageUrl, returnPageModel: true })` when you want connect + summary-first exploration in one turn
+7. `geometra_page_model` + `geometra_expand_section` when you are still exploring the page rather than filling it
 
 Typical batch:
 
 ```json
 {
   "actions": [
-    { "type": "click", "x": 412, "y": 228 },
+    { "type": "click", "role": "textbox", "name": "Full name" },
     { "type": "type", "text": "Taylor Applicant" },
     { "type": "upload_files", "paths": ["/Users/you/resume.pdf"], "fieldLabel": "Resume" },
-    { "type": "wait_for", "text": "Parsing your resume", "present": false, "timeoutMs": 10000 }
+    { "type": "click", "role": "button", "name": "Submit", "waitFor": { "text": "Application submitted", "timeoutMs": 10000 } }
   ]
 }
 ```

package/dist/__tests__/proxy-session-recovery.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/proxy-session-recovery.test.js

@@ -0,0 +1,173 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { WebSocketServer } from 'ws';
+const mockState = vi.hoisted(() => ({
+    startEmbeddedGeometraProxy: vi.fn(),
+    spawnGeometraProxy: vi.fn(),
+}));
+vi.mock('../proxy-spawn.js', () => ({
+    startEmbeddedGeometraProxy: mockState.startEmbeddedGeometraProxy,
+    spawnGeometraProxy: mockState.spawnGeometraProxy,
+}));
+const { connectThroughProxy, disconnect } = await import('../session.js');
+function frame(pageUrl) {
+    return {
+        type: 'frame',
+        layout: { x: 0, y: 0, width: 1280, height: 720, children: [] },
+        tree: {
+            kind: 'box',
+            props: {},
+            semantic: {
+                tag: 'body',
+                role: 'group',
+                pageUrl,
+            },
+            children: [],
+        },
+    };
+}
+async function createProxyPeer(options) {
+    const wss = new WebSocketServer({ port: 0 });
+    wss.on('connection', ws => {
+        ws.send(JSON.stringify(frame(options?.pageUrl ?? 'https://jobs.example.com/original')));
+        ws.on('message', raw => {
+            const msg = JSON.parse(String(raw));
+            if (msg.type === 'navigate') {
+                options?.onNavigate?.(ws, msg);
+            }
+        });
+    });
+    const port = await new Promise((resolve, reject) => {
+        wss.once('listening', () => {
+            const address = wss.address();
+            if (typeof address === 'object' && address)
+                resolve(address.port);
+            else
+                reject(new Error('Failed to resolve ephemeral WebSocket port'));
+        });
+        wss.once('error', reject);
+    });
+    return {
+        wss,
+        wsUrl: `ws://127.0.0.1:${port}`,
+    };
+}
+afterEach(async () => {
+    disconnect({ closeProxy: true });
+    vi.clearAllMocks();
+});
+async function closePeer(wss) {
+    for (const client of wss.clients) {
+        client.close();
+    }
+    await new Promise((resolve, reject) => wss.close(err => (err ? reject(err) : resolve())));
+}
+describe('connectThroughProxy recovery', () => {
+    it('restarts from a fresh proxy when a reused browser session was already closed', async () => {
+        const stalePeer = await createProxyPeer({
+            pageUrl: 'https://jobs.example.com/original',
+            onNavigate(ws, msg) {
+                ws.send(JSON.stringify({
+                    type: 'error',
+                    requestId: msg.requestId,
+                    message: 'page.goto: Target page, context or browser has been closed',
+                }));
+            },
+        });
+        const freshPeer = await createProxyPeer({
+            pageUrl: 'https://jobs.example.com/recovered',
+        });
+        const staleRuntime = {
+            wsUrl: stalePeer.wsUrl,
+            closed: false,
+            close: vi.fn(async () => {
+                staleRuntime.closed = true;
+            }),
+        };
+        const freshRuntime = {
+            wsUrl: freshPeer.wsUrl,
+            closed: false,
+            close: vi.fn(async () => {
+                freshRuntime.closed = true;
+            }),
+        };
+        mockState.startEmbeddedGeometraProxy
+            .mockResolvedValueOnce({ runtime: staleRuntime, wsUrl: stalePeer.wsUrl })
+            .mockResolvedValueOnce({ runtime: freshRuntime, wsUrl: freshPeer.wsUrl });
+        mockState.spawnGeometraProxy.mockRejectedValue(new Error('spawn fallback should not be used'));
+        try {
+            const firstSession = await connectThroughProxy({
+                pageUrl: 'https://jobs.example.com/original',
+                headless: true,
+            });
+            expect(firstSession.proxyRuntime).toBe(staleRuntime);
+            const recoveredSession = await connectThroughProxy({
+                pageUrl: 'https://jobs.example.com/recovered',
+                headless: true,
+            });
+            expect(recoveredSession.proxyRuntime).toBe(freshRuntime);
+            expect(mockState.startEmbeddedGeometraProxy).toHaveBeenCalledTimes(2);
+            expect(staleRuntime.close).toHaveBeenCalledTimes(1);
+            expect(mockState.spawnGeometraProxy).not.toHaveBeenCalled();
+        }
+        finally {
+            disconnect({ closeProxy: true });
+            await closePeer(stalePeer.wss);
+            await closePeer(freshPeer.wss);
+        }
+    });
+    it('keeps separate warm proxies for compatible headed and headless reuse', async () => {
+        const headedPeer = await createProxyPeer({
+            pageUrl: 'https://jobs.example.com/headed',
+        });
+        const headlessPeer = await createProxyPeer({
+            pageUrl: 'https://jobs.example.com/headless',
+        });
+        const headedRuntime = {
+            wsUrl: headedPeer.wsUrl,
+            closed: false,
+            close: vi.fn(async () => {
+                headedRuntime.closed = true;
+            }),
+        };
+        const headlessRuntime = {
+            wsUrl: headlessPeer.wsUrl,
+            closed: false,
+            close: vi.fn(async () => {
+                headlessRuntime.closed = true;
+            }),
+        };
+        mockState.startEmbeddedGeometraProxy
+            .mockResolvedValueOnce({ runtime: headedRuntime, wsUrl: headedPeer.wsUrl })
+            .mockResolvedValueOnce({ runtime: headlessRuntime, wsUrl: headlessPeer.wsUrl });
+        mockState.spawnGeometraProxy.mockRejectedValue(new Error('spawn fallback should not be used'));
+        try {
+            const headedSession = await connectThroughProxy({
+                pageUrl: 'https://jobs.example.com/headed',
+                headless: false,
+            });
+            expect(headedSession.proxyRuntime).toBe(headedRuntime);
+            disconnect();
+            const headlessSession = await connectThroughProxy({
+                pageUrl: 'https://jobs.example.com/headless',
+                headless: true,
+            });
+            expect(headlessSession.proxyRuntime).toBe(headlessRuntime);
+            disconnect();
+            const reusedHeadedSession = await connectThroughProxy({
+                pageUrl: 'https://jobs.example.com/headed',
+                headless: false,
+            });
+            expect(reusedHeadedSession.proxyRuntime).toBe(headedRuntime);
+            expect(mockState.startEmbeddedGeometraProxy).toHaveBeenCalledTimes(2);
+            expect(headedRuntime.close).not.toHaveBeenCalled();
+            expect(headlessRuntime.close).not.toHaveBeenCalled();
+        }
+        finally {
+            disconnect({ closeProxy: true });
+            expect(headedRuntime.close).toHaveBeenCalledTimes(1);
+            expect(headlessRuntime.close).toHaveBeenCalledTimes(1);
+            await closePeer(headedPeer.wss);
+            await closePeer(headlessPeer.wss);
+        }
+    });
+});

package/dist/__tests__/server-batch-results.test.js

@@ -294,6 +294,28 @@ describe('batch MCP result shaping', () => {
             },
         });
     });
+    it('can inline the page model into connect for the low-turn exploration path', async () => {
+        const handler = getToolHandler('geometra_connect');
+        const result = await handler({
+            pageUrl: 'https://jobs.example.com/application',
+            headless: true,
+            returnPageModel: true,
+            maxPrimaryActions: 4,
+            maxSectionsPerKind: 3,
+        });
+        const payload = JSON.parse(result.content[0].text);
+        expect(payload).toMatchObject({
+            connected: true,
+            transport: 'proxy',
+            pageUrl: 'https://jobs.example.com/application',
+            pageModel: {
+                viewport: { width: 1280, height: 800 },
+                archetypes: ['form'],
+                summary: { formCount: 1 },
+            },
+        });
+        expect(payload).not.toHaveProperty('formSchema');
+    });
     it('returns compact form schemas without requiring section expansion', async () => {
         const handler = getToolHandler('geometra_form_schema');
         mockState.formSchemas = [
@@ -741,4 +763,222 @@ describe('query and reveal tools', () => {
             },
         });
     });
+    it('clicks an offscreen semantic target by revealing it first', async () => {
+        const handler = getToolHandler('geometra_click');
+        mockState.currentA11yRoot = node('group', undefined, {
+            bounds: { x: 0, y: 0, width: 1280, height: 800 },
+            meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+            children: [
+                node('form', 'Application', {
+                    bounds: { x: 20, y: -200, width: 760, height: 1900 },
+                    path: [0],
+                    children: [
+                        node('button', 'Submit application', {
+                            bounds: { x: 60, y: 1540, width: 180, height: 40 },
+                            path: [0, 0],
+                        }),
+                    ],
+                }),
+            ],
+        });
+        mockState.sendWheel.mockImplementationOnce(async () => {
+            mockState.currentA11yRoot = node('group', undefined, {
+                bounds: { x: 0, y: 0, width: 1280, height: 800 },
+                meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 1220 },
+                children: [
+                    node('form', 'Application', {
+                        bounds: { x: 20, y: -1420, width: 760, height: 1900 },
+                        path: [0],
+                        children: [
+                            node('button', 'Submit application', {
+                                bounds: { x: 60, y: 320, width: 180, height: 40 },
+                                path: [0, 0],
+                            }),
+                        ],
+                    }),
+                ],
+            });
+            bumpMockUiRevision();
+            return { status: 'updated', timeoutMs: 2500 };
+        });
+        const result = await handler({
+            id: 'n:0.0',
+            maxRevealSteps: 3,
+            revealTimeoutMs: 2500,
+            detail: 'minimal',
+        });
+        expect(mockState.sendWheel).toHaveBeenCalledTimes(1);
+        expect(mockState.sendClick).toHaveBeenCalledWith(mockState.session, 150, 340, undefined);
+        expect(result.content[0].text).toContain('Clicked button "Submit application" (n:0.0) at (150, 340) after 1 reveal step.');
+    });
+    it('can wait for a semantic post-click condition in the same click call', async () => {
+        const handler = getToolHandler('geometra_click');
+        mockState.currentA11yRoot = node('group', undefined, {
+            bounds: { x: 0, y: 0, width: 1280, height: 800 },
+            meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+            children: [
+                node('button', 'Submit application', {
+                    bounds: { x: 60, y: 320, width: 180, height: 40 },
+                    path: [0],
+                }),
+            ],
+        });
+        mockState.sendClick.mockImplementationOnce(async () => {
+            mockState.currentA11yRoot = node('group', undefined, {
+                bounds: { x: 0, y: 0, width: 1280, height: 800 },
+                meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+                children: [
+                    node('button', 'Submit application', {
+                        bounds: { x: 60, y: 320, width: 180, height: 40 },
+                        path: [0],
+                    }),
+                    node('dialog', 'Application submitted', {
+                        bounds: { x: 240, y: 140, width: 420, height: 260 },
+                        path: [1],
+                    }),
+                ],
+            });
+            bumpMockUiRevision();
+            return { status: 'updated', timeoutMs: 2000 };
+        });
+        const result = await handler({
+            role: 'button',
+            name: 'Submit application',
+            waitFor: {
+                role: 'dialog',
+                name: 'Application submitted',
+                timeoutMs: 5000,
+            },
+            detail: 'minimal',
+        });
+        expect(mockState.waitForUiCondition).toHaveBeenCalledWith(mockState.session, expect.any(Function), 5000);
+        expect(result.content[0].text).toContain('Post-click condition satisfied after');
+        expect(result.content[0].text).toContain('1 matching node(s).');
+    });
+    it('lets run_actions click a semantic target without manual coordinates', async () => {
+        const handler = getToolHandler('geometra_run_actions');
+        mockState.currentA11yRoot = node('group', undefined, {
+            bounds: { x: 0, y: 0, width: 1280, height: 800 },
+            meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+            children: [
+                node('form', 'Application', {
+                    bounds: { x: 20, y: -200, width: 760, height: 1900 },
+                    path: [0],
+                    children: [
+                        node('button', 'Submit application', {
+                            bounds: { x: 60, y: 1540, width: 180, height: 40 },
+                            path: [0, 0],
+                        }),
+                    ],
+                }),
+            ],
+        });
+        mockState.sendWheel.mockImplementationOnce(async () => {
+            mockState.currentA11yRoot = node('group', undefined, {
+                bounds: { x: 0, y: 0, width: 1280, height: 800 },
+                meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 1220 },
+                children: [
+                    node('form', 'Application', {
+                        bounds: { x: 20, y: -1420, width: 760, height: 1900 },
+                        path: [0],
+                        children: [
+                            node('button', 'Submit application', {
+                                bounds: { x: 60, y: 320, width: 180, height: 40 },
+                                path: [0, 0],
+                            }),
+                        ],
+                    }),
+                ],
+            });
+            bumpMockUiRevision();
+            return { status: 'updated', timeoutMs: 2500 };
+        });
+        const result = await handler({
+            actions: [
+                {
+                    type: 'click',
+                    role: 'button',
+                    name: 'Submit application',
+                    maxRevealSteps: 3,
+                    revealTimeoutMs: 2500,
+                },
+            ],
+            stopOnError: true,
+            includeSteps: true,
+            detail: 'minimal',
+        });
+        const payload = JSON.parse(result.content[0].text);
+        const steps = payload.steps;
+        expect(mockState.sendWheel).toHaveBeenCalledTimes(1);
+        expect(mockState.sendClick).toHaveBeenCalledWith(mockState.session, 150, 340, undefined);
+        expect(steps[0]).toMatchObject({
+            index: 0,
+            type: 'click',
+            ok: true,
+            at: { x: 150, y: 340 },
+            revealSteps: 1,
+            target: { id: 'n:0.0', role: 'button', name: 'Submit application' },
+            wait: 'updated',
+        });
+    });
+    it('lets run_actions click and wait for a semantic post-condition in one step', async () => {
+        const handler = getToolHandler('geometra_run_actions');
+        mockState.currentA11yRoot = node('group', undefined, {
+            bounds: { x: 0, y: 0, width: 1280, height: 800 },
+            meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+            children: [
+                node('button', 'Submit application', {
+                    bounds: { x: 60, y: 320, width: 180, height: 40 },
+                    path: [0],
+                }),
+            ],
+        });
+        mockState.sendClick.mockImplementationOnce(async () => {
+            mockState.currentA11yRoot = node('group', undefined, {
+                bounds: { x: 0, y: 0, width: 1280, height: 800 },
+                meta: { pageUrl: 'https://jobs.example.com/application', scrollX: 0, scrollY: 0 },
+                children: [
+                    node('button', 'Submit application', {
+                        bounds: { x: 60, y: 320, width: 180, height: 40 },
+                        path: [0],
+                    }),
+                    node('dialog', 'Application submitted', {
+                        bounds: { x: 240, y: 140, width: 420, height: 260 },
+                        path: [1],
+                    }),
+                ],
+            });
+            bumpMockUiRevision();
+            return { status: 'updated', timeoutMs: 2000 };
+        });
+        const result = await handler({
+            actions: [
+                {
+                    type: 'click',
+                    role: 'button',
+                    name: 'Submit application',
+                    waitFor: {
+                        role: 'dialog',
+                        name: 'Application submitted',
+                        timeoutMs: 5000,
+                    },
+                },
+            ],
+            stopOnError: true,
+            includeSteps: true,
+            detail: 'minimal',
+        });
+        const payload = JSON.parse(result.content[0].text);
+        const steps = payload.steps;
+        expect(steps[0]).toMatchObject({
+            index: 0,
+            type: 'click',
+            ok: true,
+            postWait: {
+                present: true,
+                matchCount: 1,
+                filter: { role: 'dialog', name: 'Application submitted' },
+            },
+        });
+    });
 });