@geometra/mcp 1.20.0 → 1.21.0

package/dist/server.js

@@ -761,6 +761,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                 const wait = await sendFillFields(session, directFields);
                 const ackResult = parseProxyFillAckResult(wait.result);
                 if (ackResult && ackResult.invalidCount === 0) {
+                    recordWorkflowFill(session, undefined, undefined, valuesById, valuesByLabel, 0, directFields.length);
                     return ok(JSON.stringify({
                         ...connection,
                         completed: true,
@@ -777,6 +778,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                 const afterDirect = sessionA11y(session);
                 const directSignals = afterDirect ? collectSessionSignals(afterDirect) : undefined;
                 if (directSignals && directSignals.invalidFields.length === 0) {
+                    recordWorkflowFill(session, undefined, undefined, valuesById, valuesByLabel, 0, directFields.length);
                     return ok(JSON.stringify({
                         ...connection,
                         completed: true,
@@ -872,8 +874,12 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
                     fieldCount: planned.fields.length,
                     successCount: planned.fields.length,
                     errorCount: 0,
+                    minConfidence: planned.planned.length > 0
+                        ? Number(Math.min(...planned.planned.map(p => p.confidence)).toFixed(2))
+                        : undefined,
                     ...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
                 };
+                recordWorkflowFill(session, schema.formId, schema.name, valuesById, valuesByLabel, invalidRemaining, planned.fields.length);
                 if (failOnInvalid && invalidRemaining > 0) {
                     return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
                 }
@@ -885,15 +891,18 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
         const startIndex = resumeFromIndex ?? 0;
         for (let index = startIndex; index < planned.fields.length; index++) {
             const field = planned.fields[index];
+            const plan = planned.planned[index];
+            const confidence = plan?.confidence;
+            const matchMethod = plan?.matchMethod;
             try {
                 const result = await executeFillField(session, field, detail);
                 steps.push(detail === 'verbose'
-                    ? { index, kind: field.kind, ok: true, summary: result.summary }
-                    : { index, kind: field.kind, ok: true, ...result.compact });
+                    ? { index, kind: field.kind, ok: true, ...(confidence !== undefined ? { confidence, matchMethod } : {}), summary: result.summary }
+                    : { index, kind: field.kind, ok: true, ...(confidence !== undefined ? { confidence, matchMethod } : {}), ...result.compact });
             }
             catch (e) {
                 const message = e instanceof Error ? e.message : String(e);
-                steps.push({ index, kind: field.kind, ok: false, error: message });
+                steps.push({ index, kind: field.kind, ok: false, ...(confidence !== undefined ? { confidence, matchMethod } : {}), error: message });
                 if (stopOnError) {
                     stoppedAt = index;
                     break;
@@ -914,11 +923,15 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
             fieldCount: planned.fields.length,
             successCount,
             errorCount,
+            minConfidence: planned.planned.length > 0
+                ? Number(Math.min(...planned.planned.map(p => p.confidence)).toFixed(2))
+                : undefined,
             ...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
             ...(includeSteps ? { steps } : {}),
             ...(stoppedAt !== undefined ? { stoppedAt, resumeFromIndex: stoppedAt + 1 } : {}),
             ...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
         };
+        recordWorkflowFill(session, schema.formId, schema.name, valuesById, valuesByLabel, invalidRemaining, planned.fields.length);
         if (failOnInvalid && invalidRemaining > 0) {
             return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
         }
@@ -1713,6 +1726,43 @@ For a token-efficient semantic view, use geometra_snapshot (default compact). Fo
             return err('Not connected. Call geometra_connect first.');
         return ok(JSON.stringify(session.layout, null, 2));
     });
+    // ── workflow state ───────────────────────────────────────────
+    server.tool('geometra_workflow_state', `Get the accumulated workflow state across page navigations. Shows which pages/forms have been filled, what values were submitted, and the fill status per page.
+
+Use this after navigating to a new page in a multi-step flow (e.g. job applications) to understand what has been completed so far. Pass \`clear: true\` to reset the workflow state.`, {
+        clear: z.boolean().optional().default(false).describe('Reset the workflow state'),
+    }, async ({ clear }) => {
+        const session = getSession();
+        if (!session)
+            return err('Not connected. Call geometra_connect first.');
+        if (clear) {
+            session.workflowState = undefined;
+            return ok(JSON.stringify({ cleared: true }));
+        }
+        if (!session.workflowState || session.workflowState.pages.length === 0) {
+            return ok(JSON.stringify({
+                pageCount: 0,
+                message: 'No workflow state recorded yet. Fill a form with geometra_fill_form to start tracking.',
+            }));
+        }
+        const state = session.workflowState;
+        const totalFields = state.pages.reduce((sum, p) => sum + p.fieldCount, 0);
+        const totalInvalid = state.pages.reduce((sum, p) => sum + p.invalidCount, 0);
+        return ok(JSON.stringify({
+            pageCount: state.pages.length,
+            totalFieldsFilled: totalFields,
+            totalInvalidRemaining: totalInvalid,
+            elapsedMs: Date.now() - state.startedAt,
+            pages: state.pages.map(p => ({
+                pageUrl: p.pageUrl,
+                ...(p.formId ? { formId: p.formId } : {}),
+                ...(p.formName ? { formName: p.formName } : {}),
+                fieldCount: p.fieldCount,
+                invalidCount: p.invalidCount,
+                filledValues: p.filledValues,
+            })),
+        }));
+    });
     // ── disconnect ───────────────────────────────────────────────
     server.tool('geometra_disconnect', `Disconnect from the Geometra server. Proxy-backed sessions keep compatible browsers alive by default so the next geometra_connect can reuse them quickly; pass closeBrowser=true to fully tear down the warm proxy/browser pool.`, {
         closeBrowser: z.boolean().optional().default(false).describe('Fully close the spawned proxy/browser instead of keeping it warm for reuse'),
@@ -2031,6 +2081,12 @@ function collectSessionSignals(root) {
     };
     const seenAlerts = new Set();
     const seenInvalidIds = new Set();
+    const captchaPattern = /recaptcha|g-recaptcha|hcaptcha|h-captcha|turnstile|cf-turnstile|captcha/i;
+    const captchaTypes = {
+        recaptcha: 'recaptcha', 'g-recaptcha': 'recaptcha',
+        hcaptcha: 'hcaptcha', 'h-captcha': 'hcaptcha',
+        turnstile: 'turnstile', 'cf-turnstile': 'turnstile',
+    };
     function walk(node) {
         if (!signals.focus && node.state?.focused) {
             signals.focus = {
@@ -2063,6 +2119,14 @@ function collectSessionSignals(root) {
                 });
             }
         }
+        if (!signals.captchaDetected) {
+            const text = [node.name, node.value].filter(Boolean).join(' ');
+            if (captchaPattern.test(text)) {
+                signals.captchaDetected = true;
+                const match = text.toLowerCase().match(/recaptcha|g-recaptcha|hcaptcha|h-captcha|turnstile|cf-turnstile/);
+                signals.captchaType = match ? (captchaTypes[match[0]] ?? 'unknown') : 'unknown';
+            }
+        }
         for (const child of node.children)
             walk(child);
     }
@@ -2124,6 +2188,7 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
             ? { scroll: { x: signals.scrollX ?? 0, y: signals.scrollY ?? 0 } }
             : {}),
         ...(signals.focus ? { focus: signals.focus } : {}),
+        ...(signals.captchaDetected ? { captchaDetected: true, captchaType: signals.captchaType ?? 'unknown' } : {}),
         dialogCount: signals.dialogCount,
         busyCount: signals.busyCount,
         alertCount: signals.alerts.length,
@@ -2442,7 +2507,7 @@ function planFormFill(schema, opts) {
         else
             fieldsByLabel.set(key, [field]);
     }
-    const planned = [];
+    const allPlanned = [];
     const seenFieldIds = new Set();
     for (const [fieldId, value] of Object.entries(opts.valuesById ?? {})) {
         const field = fieldById.get(fieldId);
@@ -2451,7 +2516,8 @@ function planFormFill(schema, opts) {
         const next = plannedFillInputsForField(field, value);
         if ('error' in next)
             return { ok: false, error: next.error };
-        planned.push(...next);
+        for (const n of next)
+            allPlanned.push({ field: n, confidence: 1.0, matchMethod: 'id' });
         seenFieldIds.add(field.id);
     }
     for (const [label, value] of Object.entries(opts.valuesByLabel ?? {})) {
@@ -2468,10 +2534,14 @@ function planFormFill(schema, opts) {
         const next = plannedFillInputsForField(field, value);
         if ('error' in next)
             return { ok: false, error: next.error };
-        planned.push(...next);
+        const isExact = field.label === label;
+        const confidence = isExact ? 0.95 : 0.8;
+        const matchMethod = isExact ? 'label-exact' : 'label-normalized';
+        for (const n of next)
+            allPlanned.push({ field: n, confidence, matchMethod });
         seenFieldIds.add(field.id);
     }
-    return { ok: true, fields: planned };
+    return { ok: true, fields: allPlanned.map(p => p.field), planned: allPlanned };
 }
 function isResolvedFillFieldInput(field) {
     if (field.kind === 'toggle')
@@ -3073,6 +3143,31 @@ function ok(text, screenshot) {
     }
     return { content };
 }
+function recordWorkflowFill(session, formId, formName, valuesById, valuesByLabel, invalidCount, fieldCount) {
+    if (!session.workflowState) {
+        session.workflowState = { pages: [], startedAt: Date.now() };
+    }
+    const filledValues = {};
+    for (const [k, v] of Object.entries(valuesById ?? {})) {
+        if (typeof v === 'string' || typeof v === 'boolean')
+            filledValues[k] = v;
+    }
+    for (const [k, v] of Object.entries(valuesByLabel ?? {})) {
+        if (typeof v === 'string' || typeof v === 'boolean')
+            filledValues[k] = v;
+    }
+    const a11y = sessionA11y(session);
+    const pageUrl = a11y?.meta?.pageUrl ?? session.url;
+    session.workflowState.pages.push({
+        pageUrl,
+        formId,
+        formName,
+        filledValues,
+        filledAt: Date.now(),
+        fieldCount,
+        invalidCount,
+    });
+}
 async function captureScreenshotBase64(session) {
     try {
         const wait = await sendScreenshot(session);

package/dist/session.d.ts

@@ -121,6 +121,11 @@ export interface PageDialogModel extends PageSectionSummaryBase {
 export interface PageListModel extends PageSectionSummaryBase {
     itemCount: number;
 }
+export interface CaptchaDetection {
+    detected: boolean;
+    type?: 'recaptcha' | 'hcaptcha' | 'turnstile' | 'cloudflare-challenge' | 'unknown';
+    hint?: string;
+}
 export interface PageModel {
     viewport: {
         width: number;
@@ -134,6 +139,7 @@ export interface PageModel {
         listCount: number;
         focusableCount: number;
     };
+    captcha?: CaptchaDetection;
     primaryActions: PagePrimaryAction[];
     landmarks: PageLandmark[];
     forms: PageFormModel[];
@@ -343,6 +349,19 @@ export interface UiDelta {
     viewport?: UiViewportChange;
     focus?: UiFocusChange;
 }
+export interface WorkflowPageEntry {
+    pageUrl: string;
+    formId?: string;
+    formName?: string;
+    filledValues: Record<string, string | boolean>;
+    filledAt: number;
+    fieldCount: number;
+    invalidCount: number;
+}
+export interface WorkflowState {
+    pages: WorkflowPageEntry[];
+    startedAt: number;
+}
 export interface Session {
     ws: WebSocket;
     layout: Record<string, unknown> | null;
@@ -360,6 +379,7 @@ export interface Session {
         revision: number;
         forms: FormSchemaModel[];
     }>;
+    workflowState?: WorkflowState;
 }
 export interface SessionConnectTrace {
     mode: 'direct-ws' | 'fresh-proxy' | 'reused-proxy';
@@ -563,10 +583,6 @@ export declare function buildCompactUiIndex(root: A11yNode, options?: {
 };
 export declare function summarizeCompactIndex(nodes: CompactUiNode[], maxLines?: number): string;
 export declare function nodeContextForNode(root: A11yNode, node: A11yNode): NodeContextModel | undefined;
-/**
- * Build a summary-first, stable-ID webpage model from the accessibility tree.
- * Use {@link expandPageSection} to fetch details for a specific section on demand.
- */
 export declare function buildPageModel(root: A11yNode, options?: {
     maxPrimaryActions?: number;
     maxSectionsPerKind?: number;

package/dist/session.js

@@ -1845,6 +1845,47 @@ function inferPageArchetypes(model) {
  * Build a summary-first, stable-ID webpage model from the accessibility tree.
  * Use {@link expandPageSection} to fetch details for a specific section on demand.
  */
+const CAPTCHA_PATTERNS = [
+    { pattern: /recaptcha|g-recaptcha/i, type: 'recaptcha', hint: 'Google reCAPTCHA detected' },
+    { pattern: /hcaptcha|h-captcha/i, type: 'hcaptcha', hint: 'hCaptcha detected' },
+    { pattern: /turnstile|cf-turnstile/i, type: 'turnstile', hint: 'Cloudflare Turnstile detected' },
+    { pattern: /cloudflare.*challenge|challenge-platform|just a moment/i, type: 'cloudflare-challenge', hint: 'Cloudflare challenge page detected' },
+    { pattern: /captcha/i, type: 'unknown', hint: 'CAPTCHA element detected' },
+];
+function detectCaptcha(root) {
+    let found;
+    function walk(node) {
+        if (found)
+            return;
+        const text = [node.name, node.value, node.role].filter(Boolean).join(' ');
+        for (const { pattern, type, hint } of CAPTCHA_PATTERNS) {
+            if (pattern.test(text)) {
+                found = { detected: true, type, hint };
+                return;
+            }
+        }
+        // Check iframe placeholders (common for reCAPTCHA/hCaptcha/Turnstile)
+        if (node.meta && typeof node.meta.frameUrl === 'string') {
+            const frameUrl = node.meta.frameUrl;
+            for (const { pattern, type, hint } of CAPTCHA_PATTERNS) {
+                if (pattern.test(frameUrl)) {
+                    found = { detected: true, type, hint };
+                    return;
+                }
+            }
+        }
+        for (const child of node.children)
+            walk(child);
+    }
+    walk(root);
+    // Also check the page URL for Cloudflare challenge pages
+    if (!found && root.meta?.pageUrl) {
+        if (/challenge|cdn-cgi.*challenge/i.test(root.meta.pageUrl)) {
+            found = { detected: true, type: 'cloudflare-challenge', hint: 'Cloudflare challenge page URL detected' };
+        }
+    }
+    return found ?? { detected: false };
+}
 export function buildPageModel(root, options) {
     const maxPrimaryActions = options?.maxPrimaryActions ?? 6;
     const maxSectionsPerKind = options?.maxSectionsPerKind ?? 8;
@@ -1928,8 +1969,10 @@ export function buildPageModel(root, options) {
         dialogs: sortByBounds(dialogs).slice(0, maxSectionsPerKind),
         lists: sortByBounds(lists).slice(0, maxSectionsPerKind),
     };
+    const captcha = detectCaptcha(root);
     return {
         ...baseModel,
+        ...(captcha.detected ? { captcha } : {}),
         archetypes: inferPageArchetypes(baseModel),
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geometra/mcp",
-  "version": "1.20.0",
+  "version": "1.21.0",
   "description": "MCP server for Geometra — interact with running Geometra apps via the geometry protocol, no browser needed",
   "license": "MIT",
   "type": "module",