@geometra/mcp 1.19.23 → 1.21.0

package/dist/__tests__/server-batch-results.test.js

@@ -281,7 +281,7 @@ describe('batch MCP result shaping', () => {
             alertCount: 1,
             invalidCount: 5,
         });
-        expect(final.invalidFields.length).toBe(4);
+        expect(final.invalidFields.length).toBe(5);
         expect(final.alerts.length).toBe(1);
     });
     it('uses the proxy batch path for fill_fields when step output is omitted', async () => {

package/dist/server.js

@@ -3,7 +3,7 @@ import { performance } from 'node:perf_hooks';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
 import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
-import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
+import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, sendScreenshot, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
 function checkedStateInput() {
     return z
         .union([z.boolean(), z.literal('mixed')])
@@ -726,8 +726,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
             .optional()
             .default(false)
             .describe('Include per-field step results in the JSON payload (default false for the smallest response)'),
+        resumeFromIndex: z
+            .number()
+            .int()
+            .min(0)
+            .optional()
+            .describe('Resume a partial fill from this field index (from a previous stoppedAt + 1). Skips already-filled fields.'),
         detail: detailInput(),
-    }, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, detail }) => {
+    }, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, resumeFromIndex, detail }) => {
         const directFields = !includeSteps && !formId && Object.keys(valuesById ?? {}).length === 0
             ? directLabelBatchFields(valuesByLabel)
             : null;
@@ -755,6 +761,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                 const wait = await sendFillFields(session, directFields);
                 const ackResult = parseProxyFillAckResult(wait.result);
                 if (ackResult && ackResult.invalidCount === 0) {
+                    recordWorkflowFill(session, undefined, undefined, valuesById, valuesByLabel, 0, directFields.length);
                     return ok(JSON.stringify({
                         ...connection,
                         completed: true,
@@ -771,6 +778,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                 const afterDirect = sessionA11y(session);
                 const directSignals = afterDirect ? collectSessionSignals(afterDirect) : undefined;
                 if (directSignals && directSignals.invalidFields.length === 0) {
+                    recordWorkflowFill(session, undefined, undefined, valuesById, valuesByLabel, 0, directFields.length);
                     return ok(JSON.stringify({
                         ...connection,
                         completed: true,
@@ -866,8 +874,12 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                     fieldCount: planned.fields.length,
                     successCount: planned.fields.length,
                     errorCount: 0,
+                    minConfidence: planned.planned.length > 0
+                        ? Number(Math.min(...planned.planned.map(p => p.confidence)).toFixed(2))
+                        : undefined,
                     ...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
                 };
+                recordWorkflowFill(session, schema.formId, schema.name, valuesById, valuesByLabel, invalidRemaining, planned.fields.length);
                 if (failOnInvalid && invalidRemaining > 0) {
                     return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
                 }
@@ -876,17 +888,21 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
         }
         const steps = [];
         let stoppedAt;
-        for (let index = 0; index < planned.fields.length; index++) {
+        const startIndex = resumeFromIndex ?? 0;
+        for (let index = startIndex; index < planned.fields.length; index++) {
             const field = planned.fields[index];
+            const plan = planned.planned[index];
+            const confidence = plan?.confidence;
+            const matchMethod = plan?.matchMethod;
             try {
                 const result = await executeFillField(session, field, detail);
                 steps.push(detail === 'verbose'
-                    ? { index, kind: field.kind, ok: true, summary: result.summary }
-                    : { index, kind: field.kind, ok: true, ...result.compact });
+                    ? { index, kind: field.kind, ok: true, ...(confidence !== undefined ? { confidence, matchMethod } : {}), summary: result.summary }
+                    : { index, kind: field.kind, ok: true, ...(confidence !== undefined ? { confidence, matchMethod } : {}), ...result.compact });
             }
             catch (e) {
                 const message = e instanceof Error ? e.message : String(e);
-                steps.push({ index, kind: field.kind, ok: false, error: message });
+                steps.push({ index, kind: field.kind, ok: false, ...(confidence !== undefined ? { confidence, matchMethod } : {}), error: message });
                 if (stopOnError) {
                     stoppedAt = index;
                     break;
@@ -900,17 +916,22 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
         const errorCount = steps.length - successCount;
         const payload = {
             ...connection,
-            completed: stoppedAt === undefined && steps.length === planned.fields.length,
+            completed: stoppedAt === undefined && (startIndex + steps.length) === planned.fields.length,
             execution: 'sequential',
             formId: schema.formId,
             requestedValueCount: entryCount,
             fieldCount: planned.fields.length,
             successCount,
             errorCount,
+            minConfidence: planned.planned.length > 0
+                ? Number(Math.min(...planned.planned.map(p => p.confidence)).toFixed(2))
+                : undefined,
+            ...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
             ...(includeSteps ? { steps } : {}),
-            ...(stoppedAt !== undefined ? { stoppedAt } : {}),
+            ...(stoppedAt !== undefined ? { stoppedAt, resumeFromIndex: stoppedAt + 1 } : {}),
             ...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
         };
+        recordWorkflowFill(session, schema.formId, schema.name, valuesById, valuesByLabel, invalidRemaining, planned.fields.length);
         if (failOnInvalid && invalidRemaining > 0) {
             return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
         }
@@ -952,6 +973,7 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
         const connection = autoConnectionPayload(resolved);
         const steps = [];
         let stoppedAt;
+        const batchStartedAt = performance.now();
         for (let index = 0; index < actions.length; index++) {
             const action = actions[index];
             const startedAt = performance.now();
@@ -964,32 +986,46 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
                 }
                 const result = await executeBatchAction(session, action, detail, includeSteps);
                 const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
+                const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
+                const stepSignals = includeSteps ? (() => {
+                    const a = sessionA11y(session);
+                    if (!a)
+                        return undefined;
+                    const s = collectSessionSignals(a);
+                    return { invalidCount: s.invalidFields.length, alertCount: s.alerts.length, dialogCount: s.dialogCount, busyCount: s.busyCount };
+                })() : undefined;
                 steps.push(detail === 'verbose'
                     ? {
                         index,
                         type: action.type,
                         ok: true,
                         elapsedMs,
+                        cumulativeMs,
                         ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
                         summary: result.summary,
+                        ...(stepSignals ? { signals: stepSignals } : {}),
                     }
                     : {
                         index,
                         type: action.type,
                         ok: true,
                         elapsedMs,
+                        cumulativeMs,
                         ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
                         ...result.compact,
+                        ...(stepSignals ? { signals: stepSignals } : {}),
                     });
             }
             catch (e) {
                 const message = e instanceof Error ? e.message : String(e);
                 const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
+                const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
                 steps.push({
                     index,
                     type: action.type,
                     ok: false,
                     elapsedMs,
+                    cumulativeMs,
                     ...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
                     error: message,
                 });
@@ -1041,7 +1077,12 @@ Use this first on normal HTML pages when you want to understand the page shape w
             .optional()
             .default(8)
             .describe('Cap returned landmarks/forms/dialogs/lists per kind (default 8).'),
-    }, async ({ maxPrimaryActions, maxSectionsPerKind }) => {
+        includeScreenshot: z
+            .boolean()
+            .optional()
+            .default(false)
+            .describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy. Use when geometry alone is ambiguous (icon-only buttons, visual styling cues).'),
+    }, async ({ maxPrimaryActions, maxSectionsPerKind, includeScreenshot }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
@@ -1049,7 +1090,8 @@ Use this first on normal HTML pages when you want to understand the page shape w
         if (!a11y)
             return err('No UI tree available');
         const model = buildPageModel(a11y, { maxPrimaryActions, maxSectionsPerKind });
-        return ok(JSON.stringify(model));
+        const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
+        return ok(JSON.stringify(model), screenshot);
     });
     server.tool('geometra_form_schema', `Get a compact, fill-oriented schema for forms on the page. This is the preferred discovery step before geometra_fill_form.
 
@@ -1369,6 +1411,8 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
             .describe('Upload strategy (default auto)'),
         dropX: z.number().optional().describe('Drop target X (viewport) for strategy drop'),
         dropY: z.number().optional().describe('Drop target Y (viewport) for strategy drop'),
+        contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated file inputs'),
+        sectionText: z.string().optional().describe('Containing section text to disambiguate repeated file inputs'),
         timeoutMs: z
             .number()
             .int()
@@ -1377,7 +1421,7 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
             .optional()
             .describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
         detail: detailInput(),
-    }, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, timeoutMs, detail }) => {
+    }, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
@@ -1411,6 +1455,8 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
         openX: z.number().optional().describe('Click to open dropdown'),
         openY: z.number().optional().describe('Click to open dropdown'),
         fieldLabel: z.string().optional().describe('Field label of the dropdown/combobox to open semantically (e.g. "Location")'),
+        contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated dropdowns with the same label'),
+        sectionText: z.string().optional().describe('Containing section text to disambiguate repeated dropdowns'),
         query: z.string().optional().describe('Optional text to type into a searchable combobox before selecting'),
         timeoutMs: z
             .number()
@@ -1420,7 +1466,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
             .optional()
             .describe('Optional action wait timeout for slow dropdowns / remote search results'),
         detail: detailInput(),
-    }, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs, detail }) => {
+    }, async ({ label, exact, openX, openY, fieldLabel, contextText, sectionText, query, timeoutMs, detail }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
@@ -1459,6 +1505,8 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
         value: z.string().optional().describe('Option value= attribute'),
         label: z.string().optional().describe('Visible option label (substring match)'),
         index: z.number().int().min(0).optional().describe('Zero-based option index'),
+        contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated selects'),
+        sectionText: z.string().optional().describe('Containing section text to disambiguate repeated selects'),
         timeoutMs: z
             .number()
             .int()
@@ -1467,7 +1515,7 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
             .optional()
             .describe('Optional action wait timeout'),
         detail: detailInput(),
-    }, async ({ x, y, value, label, index, timeoutMs, detail }) => {
+    }, async ({ x, y, value, label, index, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
@@ -1497,6 +1545,8 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
         checked: z.boolean().optional().default(true).describe('Desired checked state (radios only support true)'),
         exact: z.boolean().optional().describe('Exact label match'),
         controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
+        contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated checkboxes/radios'),
+        sectionText: z.string().optional().describe('Containing section text to disambiguate repeated checkboxes/radios'),
         timeoutMs: z
             .number()
             .int()
@@ -1505,7 +1555,7 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
             .optional()
             .describe('Optional action wait timeout'),
         detail: detailInput(),
-    }, async ({ label, checked, exact, controlType, timeoutMs, detail }) => {
+    }, async ({ label, checked, exact, controlType, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
@@ -1557,6 +1607,57 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
             return err(e.message);
         }
     });
+    // ── list items (virtualized list pagination) ─────────────────
+    server.tool('geometra_list_items', `Auto-scroll a virtualized or long list and collect all visible items across scroll positions. Requires \`@geometra/proxy\`.
+
+Use this for dropdowns, location pickers, or any scrollable list where items are rendered on demand. Scrolls down in steps, collecting new items each time, until no new items appear or the cap is reached.`, {
+        listId: z.string().optional().describe('Stable section id from geometra_page_model (e.g. ls:2.1) to scope item collection'),
+        role: z.string().optional().describe('Role filter for list items (default: listitem)'),
+        scrollX: z.number().optional().describe('X coordinate to position mouse for scrolling (default: viewport center)'),
+        scrollY: z.number().optional().describe('Y coordinate to position mouse for scrolling (default: viewport center)'),
+        maxItems: z.number().int().min(1).max(500).optional().default(100).describe('Cap collected items (default 100)'),
+        maxScrollSteps: z.number().int().min(1).max(50).optional().default(20).describe('Max scroll steps before stopping (default 20)'),
+        scrollDelta: z.number().optional().default(300).describe('Vertical scroll delta per step (default 300)'),
+    }, async ({ listId: _listId, role, scrollX, scrollY, maxItems, maxScrollSteps, scrollDelta }) => {
+        const session = getSession();
+        if (!session)
+            return err('Not connected. Call geometra_connect first.');
+        const itemRole = role ?? 'listitem';
+        const collected = new Map();
+        const cx = scrollX ?? 400;
+        const cy = scrollY ?? 400;
+        for (let step = 0; step < maxScrollSteps; step++) {
+            const a11y = await sessionA11yWhenReady(session);
+            if (!a11y)
+                break;
+            const items = findNodes(a11y, { role: itemRole });
+            let newCount = 0;
+            for (const item of items) {
+                const id = nodeIdForPath(item.path);
+                if (!collected.has(id)) {
+                    collected.set(id, {
+                        ...(item.name ? { name: item.name } : {}),
+                        ...(item.value ? { value: item.value } : {}),
+                    });
+                    newCount++;
+                }
+            }
+            if (collected.size >= maxItems || newCount === 0)
+                break;
+            try {
+                await sendWheel(session, scrollDelta, { x: cx, y: cy }, 1_000);
+            }
+            catch {
+                break;
+            }
+        }
+        const items = [...collected.entries()].slice(0, maxItems).map(([id, data]) => ({ id, ...data }));
+        return ok(JSON.stringify({
+            itemCount: items.length,
+            items,
+            truncated: collected.size > maxItems,
+        }));
+    });
     // ── snapshot ─────────────────────────────────────────────────
     server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout. Use **form-required** to list required fields across forms, including offscreen ones, with bounds + visibility + scroll hints for long application flows.
 
@@ -1577,15 +1678,21 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
         formId: z.string().optional().describe('Optional form id from geometra_form_schema / geometra_page_model when view=form-required'),
         maxFields: z.number().int().min(1).max(200).optional().default(80).describe('Per-form field cap when view=form-required'),
         includeOptions: z.boolean().optional().default(false).describe('Include explicit choice option labels when view=form-required'),
-    }, async ({ view, maxNodes, formId, maxFields, includeOptions }) => {
+        includeScreenshot: z
+            .boolean()
+            .optional()
+            .default(false)
+            .describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy.'),
+    }, async ({ view, maxNodes, formId, maxFields, includeOptions, includeScreenshot }) => {
         const session = getSession();
         if (!session)
             return err('Not connected. Call geometra_connect first.');
         const a11y = await sessionA11yWhenReady(session);
         if (!a11y)
             return err('No UI tree available');
+        const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
         if (view === 'full') {
-            return ok(JSON.stringify(a11y, null, 2));
+            return ok(JSON.stringify(a11y, null, 2), screenshot);
         }
         if (view === 'form-required') {
             const payload = {
@@ -1598,7 +1705,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
                     includeContext: 'auto',
                 }),
             };
-            return ok(JSON.stringify(payload));
+            return ok(JSON.stringify(payload), screenshot);
         }
         const { nodes, truncated, context } = buildCompactUiIndex(a11y, { maxNodes });
         const payload = {
@@ -1608,7 +1715,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
             nodes,
             truncated,
         };
-        return ok(JSON.stringify(payload));
+        return ok(JSON.stringify(payload), screenshot);
     });
     // ── layout ───────────────────────────────────────────────────
     server.tool('geometra_layout', `Get the raw computed layout geometry — the exact {x, y, width, height} for every node in the UI tree. This is the lowest-level view, useful for pixel-precise assertions in tests.
@@ -1619,6 +1726,43 @@ For a token-efficient semantic view, use geometra_snapshot (default compact). Fo
             return err('Not connected. Call geometra_connect first.');
         return ok(JSON.stringify(session.layout, null, 2));
     });
+    // ── workflow state ───────────────────────────────────────────
+    server.tool('geometra_workflow_state', `Get the accumulated workflow state across page navigations. Shows which pages/forms have been filled, what values were submitted, and the fill status per page.
+
+Use this after navigating to a new page in a multi-step flow (e.g. job applications) to understand what has been completed so far. Pass \`clear: true\` to reset the workflow state.`, {
+        clear: z.boolean().optional().default(false).describe('Reset the workflow state'),
+    }, async ({ clear }) => {
+        const session = getSession();
+        if (!session)
+            return err('Not connected. Call geometra_connect first.');
+        if (clear) {
+            session.workflowState = undefined;
+            return ok(JSON.stringify({ cleared: true }));
+        }
+        if (!session.workflowState || session.workflowState.pages.length === 0) {
+            return ok(JSON.stringify({
+                pageCount: 0,
+                message: 'No workflow state recorded yet. Fill a form with geometra_fill_form to start tracking.',
+            }));
+        }
+        const state = session.workflowState;
+        const totalFields = state.pages.reduce((sum, p) => sum + p.fieldCount, 0);
+        const totalInvalid = state.pages.reduce((sum, p) => sum + p.invalidCount, 0);
+        return ok(JSON.stringify({
+            pageCount: state.pages.length,
+            totalFieldsFilled: totalFields,
+            totalInvalidRemaining: totalInvalid,
+            elapsedMs: Date.now() - state.startedAt,
+            pages: state.pages.map(p => ({
+                pageUrl: p.pageUrl,
+                ...(p.formId ? { formId: p.formId } : {}),
+                ...(p.formName ? { formName: p.formName } : {}),
+                fieldCount: p.fieldCount,
+                invalidCount: p.invalidCount,
+                filledValues: p.filledValues,
+            })),
+        }));
+    });
     // ── disconnect ───────────────────────────────────────────────
     server.tool('geometra_disconnect', `Disconnect from the Geometra server. Proxy-backed sessions keep compatible browsers alive by default so the next geometra_connect can reuse them quickly; pass closeBrowser=true to fully tear down the warm proxy/browser pool.`, {
         closeBrowser: z.boolean().optional().default(false).describe('Fully close the spawned proxy/browser instead of keeping it warm for reuse'),
@@ -1720,8 +1864,10 @@ function packedFormSchemas(forms) {
             ...(field.valueLength !== undefined ? { vl: field.valueLength } : {}),
             ...(field.checked !== undefined ? { c: field.checked ? 1 : 0 } : {}),
             ...(field.values && field.values.length > 0 ? { vs: field.values } : {}),
+            ...(field.aliases ? { al: field.aliases } : {}),
             ...(field.context ? { x: field.context } : {}),
         })),
+        ...(form.sections ? { s: form.sections.map(s => ({ n: s.name, fi: s.fieldIds })) } : {}),
     }));
 }
 function formSchemaResponsePayload(session, opts) {
@@ -1935,6 +2081,12 @@ function collectSessionSignals(root) {
     };
     const seenAlerts = new Set();
     const seenInvalidIds = new Set();
+    const captchaPattern = /recaptcha|g-recaptcha|hcaptcha|h-captcha|turnstile|cf-turnstile|captcha/i;
+    const captchaTypes = {
+        recaptcha: 'recaptcha', 'g-recaptcha': 'recaptcha',
+        hcaptcha: 'hcaptcha', 'h-captcha': 'hcaptcha',
+        turnstile: 'turnstile', 'cf-turnstile': 'turnstile',
+    };
     function walk(node) {
         if (!signals.focus && node.state?.focused) {
             signals.focus = {
@@ -1967,6 +2119,14 @@ function collectSessionSignals(root) {
                 });
             }
         }
+        if (!signals.captchaDetected) {
+            const text = [node.name, node.value].filter(Boolean).join(' ');
+            if (captchaPattern.test(text)) {
+                signals.captchaDetected = true;
+                const match = text.toLowerCase().match(/recaptcha|g-recaptcha|hcaptcha|h-captcha|turnstile|cf-turnstile/);
+                signals.captchaType = match ? (captchaTypes[match[0]] ?? 'unknown') : 'unknown';
+            }
+        }
         for (const child of node.children)
             walk(child);
     }
@@ -2028,6 +2188,7 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
             ? { scroll: { x: signals.scrollX ?? 0, y: signals.scrollY ?? 0 } }
             : {}),
         ...(signals.focus ? { focus: signals.focus } : {}),
+        ...(signals.captchaDetected ? { captchaDetected: true, captchaType: signals.captchaType ?? 'unknown' } : {}),
         dialogCount: signals.dialogCount,
         busyCount: signals.busyCount,
         alertCount: signals.alerts.length,
@@ -2037,12 +2198,10 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
                 alerts: signals.alerts,
                 invalidFields: signals.invalidFields,
             }
-            : detail === 'minimal'
-                ? {
-                    alerts: signals.alerts.slice(0, 2),
-                    invalidFields: signals.invalidFields.slice(0, 4),
-                }
-                : {}),
+            : {
+                alerts: signals.alerts.slice(0, 2),
+                invalidFields: signals.invalidFields.slice(0, 6),
+            }),
     };
 }
 function compactTextValue(value, inlineLimit = 48) {
@@ -2348,7 +2507,7 @@ function planFormFill(schema, opts) {
         else
             fieldsByLabel.set(key, [field]);
     }
-    const planned = [];
+    const allPlanned = [];
     const seenFieldIds = new Set();
     for (const [fieldId, value] of Object.entries(opts.valuesById ?? {})) {
         const field = fieldById.get(fieldId);
@@ -2357,7 +2516,8 @@ function planFormFill(schema, opts) {
         const next = plannedFillInputsForField(field, value);
         if ('error' in next)
             return { ok: false, error: next.error };
-        planned.push(...next);
+        for (const n of next)
+            allPlanned.push({ field: n, confidence: 1.0, matchMethod: 'id' });
         seenFieldIds.add(field.id);
     }
     for (const [label, value] of Object.entries(opts.valuesByLabel ?? {})) {
@@ -2374,10 +2534,14 @@ function planFormFill(schema, opts) {
         const next = plannedFillInputsForField(field, value);
         if ('error' in next)
             return { ok: false, error: next.error };
-        planned.push(...next);
+        const isExact = field.label === label;
+        const confidence = isExact ? 0.95 : 0.8;
+        const matchMethod = isExact ? 'label-exact' : 'label-normalized';
+        for (const n of next)
+            allPlanned.push({ field: n, confidence, matchMethod });
         seenFieldIds.add(field.id);
     }
-    return { ok: true, fields: planned };
+    return { ok: true, fields: allPlanned.map(p => p.field), planned: allPlanned };
 }
 function isResolvedFillFieldInput(field) {
     if (field.kind === 'toggle')
@@ -2502,12 +2666,14 @@ function parseProxyFillAckResult(value) {
         typeof candidate.busyCount !== 'number') {
         return undefined;
     }
+    const invalidFields = Array.isArray(candidate.invalidFields) ? candidate.invalidFields : undefined;
     return {
         ...(typeof candidate.pageUrl === 'string' ? { pageUrl: candidate.pageUrl } : {}),
         invalidCount: candidate.invalidCount,
         alertCount: candidate.alertCount,
         dialogCount: candidate.dialogCount,
         busyCount: candidate.busyCount,
+        ...(invalidFields && invalidFields.length > 0 ? { invalidFields } : {}),
     };
 }
 function directLabelBatchFields(valuesByLabel) {
@@ -2968,8 +3134,49 @@ async function executeFillField(session, field, detail) {
         }
     }
 }
-function ok(text) {
-    return { content: [{ type: 'text', text }] };
+function ok(text, screenshot) {
+    const content = [
+        { type: 'text', text },
+    ];
+    if (screenshot) {
+        content.push({ type: 'image', data: screenshot, mimeType: 'image/png' });
+    }
+    return { content };
+}
+function recordWorkflowFill(session, formId, formName, valuesById, valuesByLabel, invalidCount, fieldCount) {
+    if (!session.workflowState) {
+        session.workflowState = { pages: [], startedAt: Date.now() };
+    }
+    const filledValues = {};
+    for (const [k, v] of Object.entries(valuesById ?? {})) {
+        if (typeof v === 'string' || typeof v === 'boolean')
+            filledValues[k] = v;
+    }
+    for (const [k, v] of Object.entries(valuesByLabel ?? {})) {
+        if (typeof v === 'string' || typeof v === 'boolean')
+            filledValues[k] = v;
+    }
+    const a11y = sessionA11y(session);
+    const pageUrl = a11y?.meta?.pageUrl ?? session.url;
+    session.workflowState.pages.push({
+        pageUrl,
+        formId,
+        formName,
+        filledValues,
+        filledAt: Date.now(),
+        fieldCount,
+        invalidCount,
+    });
+}
+async function captureScreenshotBase64(session) {
+    try {
+        const wait = await sendScreenshot(session);
+        const result = wait.result;
+        return typeof result?.screenshot === 'string' ? result.screenshot : undefined;
+    }
+    catch {
+        return undefined;
+    }
 }
 function err(text) {
     return { content: [{ type: 'text', text }], isError: true };

package/dist/session.d.ts

@@ -121,6 +121,11 @@ export interface PageDialogModel extends PageSectionSummaryBase {
 export interface PageListModel extends PageSectionSummaryBase {
     itemCount: number;
 }
+export interface CaptchaDetection {
+    detected: boolean;
+    type?: 'recaptcha' | 'hcaptcha' | 'turnstile' | 'cloudflare-challenge' | 'unknown';
+    hint?: string;
+}
 export interface PageModel {
     viewport: {
         width: number;
@@ -134,6 +139,7 @@ export interface PageModel {
         listCount: number;
         focusableCount: number;
     };
+    captcha?: CaptchaDetection;
     primaryActions: PagePrimaryAction[];
     landmarks: PageLandmark[];
     forms: PageFormModel[];
@@ -263,8 +269,13 @@ export interface FormSchemaField {
     values?: string[];
     optionCount?: number;
     options?: string[];
+    aliases?: Record<string, string[]>;
     context?: NodeContextModel;
 }
+export interface FormSchemaSection {
+    name: string;
+    fieldIds: string[];
+}
 export interface FormSchemaModel {
     formId: string;
     name?: string;
@@ -272,6 +283,7 @@ export interface FormSchemaModel {
     requiredCount: number;
     invalidCount: number;
     fields: FormSchemaField[];
+    sections?: FormSchemaSection[];
 }
 export interface FormRequiredFieldSnapshot extends FormSchemaField {
     bounds: {
@@ -337,6 +349,19 @@ export interface UiDelta {
     viewport?: UiViewportChange;
     focus?: UiFocusChange;
 }
+export interface WorkflowPageEntry {
+    pageUrl: string;
+    formId?: string;
+    formName?: string;
+    filledValues: Record<string, string | boolean>;
+    filledAt: number;
+    fieldCount: number;
+    invalidCount: number;
+}
+export interface WorkflowState {
+    pages: WorkflowPageEntry[];
+    startedAt: number;
+}
 export interface Session {
     ws: WebSocket;
     layout: Record<string, unknown> | null;
@@ -354,6 +379,7 @@ export interface Session {
         revision: number;
         forms: FormSchemaModel[];
     }>;
+    workflowState?: WorkflowState;
 }
 export interface SessionConnectTrace {
     mode: 'direct-ws' | 'fresh-proxy' | 'reused-proxy';
@@ -531,6 +557,8 @@ export declare function sendWheel(session: Session, deltaY: number, opts?: {
     x?: number;
     y?: number;
 }, timeoutMs?: number): Promise<UpdateWaitResult>;
+/** Capture a viewport screenshot from the proxy (base64 PNG). */
+export declare function sendScreenshot(session: Session, timeoutMs?: number): Promise<UpdateWaitResult>;
 /** Navigate the proxy page to a new URL while keeping the browser process alive. */
 export declare function sendNavigate(session: Session, url: string, timeoutMs?: number): Promise<UpdateWaitResult>;
 /**
@@ -555,10 +583,6 @@ export declare function buildCompactUiIndex(root: A11yNode, options?: {
 };
 export declare function summarizeCompactIndex(nodes: CompactUiNode[], maxLines?: number): string;
 export declare function nodeContextForNode(root: A11yNode, node: A11yNode): NodeContextModel | undefined;
-/**
- * Build a summary-first, stable-ID webpage model from the accessibility tree.
- * Use {@link expandPageSection} to fetch details for a specific section on demand.
- */
 export declare function buildPageModel(root: A11yNode, options?: {
     maxPrimaryActions?: number;
     maxSectionsPerKind?: number;

package/dist/session.js

@@ -875,6 +875,10 @@ export function sendWheel(session, deltaY, opts, timeoutMs) {
         ...(opts?.y !== undefined ? { y: opts.y } : {}),
     }, timeoutMs);
 }
+/** Capture a viewport screenshot from the proxy (base64 PNG). */
+export function sendScreenshot(session, timeoutMs = 10_000) {
+    return sendAndWaitForUpdate(session, { type: 'screenshot' }, timeoutMs);
+}
 /** Navigate the proxy page to a new URL while keeping the browser process alive. */
 export function sendNavigate(session, url, timeoutMs = 15_000) {
     return sendAndWaitForUpdate(session, {
@@ -1566,6 +1570,33 @@ function groupedChoiceForNode(root, formNode, seed) {
     const controls = sortByBounds(collectDescendants(formNode, matchesPrompt));
     return controls.length >= 2 ? { container: formNode, prompt, controls } : null;
 }
+const SEMANTIC_ALIAS_GROUPS = [
+    { triggers: ['yes', 'true'], aliases: ['yes', 'true', 'agree', 'agreed', 'accept', 'accepted', 'consent', 'acknowledge', 'opt in'] },
+    { triggers: ['no', 'false'], aliases: ['no', 'false', 'decline', 'declined', 'disagree', 'deny', 'opt out', 'prefer not'] },
+    { triggers: ['decline'], aliases: ['decline', 'prefer not', 'opt out', 'do not'] },
+    { triggers: ['atx', 'austin'], aliases: ['atx', 'austin', 'austin tx', 'austin texas'] },
+    { triggers: ['nyc', 'new york'], aliases: ['nyc', 'new york', 'new york ny'] },
+    { triggers: ['sf', 'san francisco'], aliases: ['sf', 'san francisco', 'san francisco ca'] },
+    { triggers: ['la', 'los angeles'], aliases: ['la', 'los angeles', 'los angeles ca'] },
+    { triggers: ['dc', 'washington dc'], aliases: ['dc', 'washington dc', 'washington d c'] },
+    { triggers: ['us', 'usa', 'united states'], aliases: ['us', 'usa', 'united states'] },
+];
+function computeOptionAliases(options) {
+    const result = {};
+    for (const option of options) {
+        const normalized = option.toLowerCase().trim();
+        for (const group of SEMANTIC_ALIAS_GROUPS) {
+            if (group.triggers.some(t => normalized === t || normalized.includes(t))) {
+                const relevant = group.aliases.filter(a => a !== normalized);
+                if (relevant.length > 0) {
+                    result[option] = relevant;
+                    break;
+                }
+            }
+        }
+    }
+    return Object.keys(result).length > 0 ? result : undefined;
+}
 function simpleSchemaField(root, node) {
     const context = nodeContextForNode(root, node);
     const label = fieldLabel(node) ?? sanitizeInlineName(node.name, 80) ?? context?.prompt;
@@ -1617,6 +1648,7 @@ function groupedSchemaField(root, grouped) {
             }),
         optionCount: options.length,
         options,
+        ...(computeOptionAliases(options) ? { aliases: computeOptionAliases(options) } : {}),
         ...(compactSchemaContext(context, grouped.prompt) ? { context: compactSchemaContext(context, grouped.prompt) } : {}),
     };
 }
@@ -1637,6 +1669,41 @@ function toggleSchemaField(root, node) {
         ...(compactSchemaContext(context, label) ? { context: compactSchemaContext(context, label) } : {}),
     };
 }
+function detectFormSections(formNode, fields) {
+    const sectionRoles = new Set(['group', 'region']);
+    const sectionNodes = [];
+    function walk(node) {
+        if (sectionRoles.has(node.role) && node.name && node.path.length > formNode.path.length) {
+            sectionNodes.push({ name: node.name, path: node.path });
+        }
+        for (const child of node.children)
+            walk(child);
+    }
+    walk(formNode);
+    if (sectionNodes.length === 0)
+        return [];
+    const fieldIdToPath = new Map();
+    for (const field of fields) {
+        const parsed = parseFormFieldId(field.id);
+        if (parsed)
+            fieldIdToPath.set(field.id, parsed);
+    }
+    const sections = [];
+    for (const sec of sectionNodes) {
+        const fieldIds = fields
+            .filter(field => {
+            const fieldPath = fieldIdToPath.get(field.id);
+            if (!fieldPath || fieldPath.length <= sec.path.length)
+                return false;
+            return sec.path.every((v, i) => fieldPath[i] === v);
+        })
+            .map(field => field.id);
+        if (fieldIds.length > 0) {
+            sections.push({ name: sec.name, fieldIds });
+        }
+    }
+    return sections;
+}
 function buildFormSchemaForNode(root, formNode, options) {
     const candidates = sortByBounds(collectDescendants(formNode, candidate => candidate.role === 'textbox' ||
         candidate.role === 'combobox' ||
@@ -1690,6 +1757,10 @@ function buildFormSchemaForNode(root, formNode, options) {
         requiredCount: compactFields.filter(field => field.required).length,
         invalidCount: compactFields.filter(field => field.invalid).length,
         fields: pageFields,
+        ...(() => {
+            const sections = detectFormSections(formNode, pageFields);
+            return sections.length > 0 ? { sections } : {};
+        })(),
     };
 }
 function trimSchemaFieldContexts(fields) {
@@ -1712,8 +1783,10 @@ function presentFormSchemaFields(fields, options) {
         const next = { ...field };
         if (booleanChoice)
             next.booleanChoice = true;
-        if (!includeOptions)
+        if (!includeOptions) {
             delete next.options;
+            delete next.aliases;
+        }
         if (includeContext === 'none') {
             delete next.context;
             return next;
@@ -1772,6 +1845,47 @@ function inferPageArchetypes(model) {
  * Build a summary-first, stable-ID webpage model from the accessibility tree.
  * Use {@link expandPageSection} to fetch details for a specific section on demand.
  */
+const CAPTCHA_PATTERNS = [
+    { pattern: /recaptcha|g-recaptcha/i, type: 'recaptcha', hint: 'Google reCAPTCHA detected' },
+    { pattern: /hcaptcha|h-captcha/i, type: 'hcaptcha', hint: 'hCaptcha detected' },
+    { pattern: /turnstile|cf-turnstile/i, type: 'turnstile', hint: 'Cloudflare Turnstile detected' },
+    { pattern: /cloudflare.*challenge|challenge-platform|just a moment/i, type: 'cloudflare-challenge', hint: 'Cloudflare challenge page detected' },
+    { pattern: /captcha/i, type: 'unknown', hint: 'CAPTCHA element detected' },
+];
+function detectCaptcha(root) {
+    let found;
+    function walk(node) {
+        if (found)
+            return;
+        const text = [node.name, node.value, node.role].filter(Boolean).join(' ');
+        for (const { pattern, type, hint } of CAPTCHA_PATTERNS) {
+            if (pattern.test(text)) {
+                found = { detected: true, type, hint };
+                return;
+            }
+        }
+        // Check iframe placeholders (common for reCAPTCHA/hCaptcha/Turnstile)
+        if (node.meta && typeof node.meta.frameUrl === 'string') {
+            const frameUrl = node.meta.frameUrl;
+            for (const { pattern, type, hint } of CAPTCHA_PATTERNS) {
+                if (pattern.test(frameUrl)) {
+                    found = { detected: true, type, hint };
+                    return;
+                }
+            }
+        }
+        for (const child of node.children)
+            walk(child);
+    }
+    walk(root);
+    // Also check the page URL for Cloudflare challenge pages
+    if (!found && root.meta?.pageUrl) {
+        if (/challenge|cdn-cgi.*challenge/i.test(root.meta.pageUrl)) {
+            found = { detected: true, type: 'cloudflare-challenge', hint: 'Cloudflare challenge page URL detected' };
+        }
+    }
+    return found ?? { detected: false };
+}
 export function buildPageModel(root, options) {
     const maxPrimaryActions = options?.maxPrimaryActions ?? 6;
     const maxSectionsPerKind = options?.maxSectionsPerKind ?? 8;
@@ -1855,8 +1969,10 @@ export function buildPageModel(root, options) {
         dialogs: sortByBounds(dialogs).slice(0, maxSectionsPerKind),
         lists: sortByBounds(lists).slice(0, maxSectionsPerKind),
     };
+    const captcha = detectCaptcha(root);
     return {
         ...baseModel,
+        ...(captcha.detected ? { captcha } : {}),
         archetypes: inferPageArchetypes(baseModel),
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geometra/mcp",
-  "version": "1.19.23",
+  "version": "1.21.0",
   "description": "MCP server for Geometra — interact with running Geometra apps via the geometry protocol, no browser needed",
   "license": "MIT",
   "type": "module",