@geometra/mcp 1.64.0 → 1.65.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -7
- package/dist/index.js +2 -5
- package/dist/proxy-spawn.d.ts +29 -0
- package/dist/proxy-spawn.js +49 -17
- package/dist/server.js +849 -219
- package/dist/session-state.js +262 -80
- package/dist/session.d.ts +93 -13
- package/dist/session.js +1365 -321
- package/dist/state-privacy.d.ts +23 -0
- package/dist/state-privacy.js +171 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.js +32 -0
- package/package.json +4 -4
package/dist/server.js
CHANGED
|
@@ -3,7 +3,9 @@ import { performance } from 'node:perf_hooks';
|
|
|
3
3
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
|
|
6
|
-
import {
|
|
6
|
+
import { REDACTED_STATE_URL, sanitizeUrlToOrigin } from './state-privacy.js';
|
|
7
|
+
import { SERVER_IMPLEMENTATION } from './version.js';
|
|
8
|
+
import { connect, connectThroughProxy, ensureSessionConnected, disconnect, pruneDisconnectedSessions, resolveSession, listSessions, getDefaultSessionId, prewarmProxy, sendClick, sendFillFields, sendFillOtp, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, sendScreenshot, sendPdfGenerate, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, formSchemaToFormGraph, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, parseSectionId, findNodeByPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
|
|
7
9
|
function checkedStateInput() {
|
|
8
10
|
return z
|
|
9
11
|
.union([z.boolean(), z.literal('mixed')])
|
|
@@ -188,7 +190,7 @@ const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic fil
|
|
|
188
190
|
'(common for “Parsing…”, “Parsing your resume”, or similar). Passing only present/timeoutMs is not enough without a filter.';
|
|
189
191
|
/** Strict input so unknown keys (e.g. textGone) fail parse; empty-filter checks happen in handlers / waitForSemanticCondition. */
|
|
190
192
|
const sessionIdSchemaField = {
|
|
191
|
-
sessionId: z.string().optional().describe('Session identifier returned by geometra_connect. Omit
|
|
193
|
+
sessionId: z.string().optional().describe('Session identifier returned by geometra_connect or an auto-connected tool. Omit only when exactly one non-isolated session is active.'),
|
|
192
194
|
};
|
|
193
195
|
const geometraQueryInputSchema = z.object({
|
|
194
196
|
...nodeFilterShape(),
|
|
@@ -285,6 +287,79 @@ function capBatchActionTimeouts(action, capMs) {
|
|
|
285
287
|
const nonEmptyFieldIdSchema = z.string().trim().min(1, 'fieldId must not be empty');
|
|
286
288
|
const nonEmptyFieldLabelSchema = z.string().trim().min(1, 'fieldLabel must not be empty');
|
|
287
289
|
const nonEmptyToggleLabelSchema = z.string().trim().min(1, 'toggle label must not be empty');
|
|
290
|
+
function incompleteCoordinatePairError(input, first, second) {
|
|
291
|
+
const hasFirst = input[first] !== undefined;
|
|
292
|
+
const hasSecond = input[second] !== undefined;
|
|
293
|
+
return hasFirst === hasSecond
|
|
294
|
+
? undefined
|
|
295
|
+
: `${first} and ${second} must be provided together`;
|
|
296
|
+
}
|
|
297
|
+
function fileUploadContractError(input) {
|
|
298
|
+
const clickPairError = incompleteCoordinatePairError(input, 'x', 'y');
|
|
299
|
+
if (clickPairError)
|
|
300
|
+
return clickPairError;
|
|
301
|
+
const dropPairError = incompleteCoordinatePairError(input, 'dropX', 'dropY');
|
|
302
|
+
if (dropPairError)
|
|
303
|
+
return dropPairError;
|
|
304
|
+
const hasClick = input.x !== undefined && input.y !== undefined;
|
|
305
|
+
const hasDrop = input.dropX !== undefined && input.dropY !== undefined;
|
|
306
|
+
const hasFieldLabel = typeof input.fieldLabel === 'string' && input.fieldLabel.trim().length > 0;
|
|
307
|
+
const hasSemanticConstraint = hasFieldLabel || input.contextText !== undefined || input.sectionText !== undefined || input.exact !== undefined;
|
|
308
|
+
const strategy = input.strategy ?? 'auto';
|
|
309
|
+
if ((input.contextText !== undefined || input.sectionText !== undefined || input.exact !== undefined) && !hasFieldLabel) {
|
|
310
|
+
return 'contextText, sectionText, and exact require fieldLabel';
|
|
311
|
+
}
|
|
312
|
+
if (strategy === 'chooser') {
|
|
313
|
+
if (!hasClick)
|
|
314
|
+
return 'chooser strategy requires x and y';
|
|
315
|
+
if (hasDrop || hasSemanticConstraint)
|
|
316
|
+
return 'chooser strategy accepts only an x,y target; semantic and drop targets cannot be combined';
|
|
317
|
+
return undefined;
|
|
318
|
+
}
|
|
319
|
+
if (strategy === 'drop') {
|
|
320
|
+
if (!hasDrop)
|
|
321
|
+
return 'drop strategy requires dropX and dropY';
|
|
322
|
+
if (hasClick || hasSemanticConstraint)
|
|
323
|
+
return 'drop strategy accepts only a dropX,dropY target; chooser and semantic targets cannot be combined';
|
|
324
|
+
return undefined;
|
|
325
|
+
}
|
|
326
|
+
if (strategy === 'hidden') {
|
|
327
|
+
if (!hasFieldLabel)
|
|
328
|
+
return 'hidden strategy requires fieldLabel to prevent a global first-input fallback';
|
|
329
|
+
if (hasClick || hasDrop)
|
|
330
|
+
return 'hidden strategy does not accept chooser or drop coordinates';
|
|
331
|
+
return undefined;
|
|
332
|
+
}
|
|
333
|
+
if (hasDrop)
|
|
334
|
+
return 'dropX and dropY require strategy="drop"';
|
|
335
|
+
if (hasClick && hasFieldLabel)
|
|
336
|
+
return 'auto strategy requires exactly one target: either x,y or fieldLabel';
|
|
337
|
+
if (!hasClick && !hasFieldLabel)
|
|
338
|
+
return 'upload_files requires an explicit target: fieldLabel, x and y, or strategy="drop" with dropX and dropY';
|
|
339
|
+
return undefined;
|
|
340
|
+
}
|
|
341
|
+
function listboxPickContractError(input) {
|
|
342
|
+
if (typeof input.fieldLabel !== 'string' || input.fieldLabel.trim().length === 0) {
|
|
343
|
+
return 'pick_listbox_option requires fieldLabel so the committed field can be confirmed';
|
|
344
|
+
}
|
|
345
|
+
return undefined;
|
|
346
|
+
}
|
|
347
|
+
function selectOptionContractError(input) {
|
|
348
|
+
const coordinateError = incompleteCoordinatePairError(input, 'x', 'y');
|
|
349
|
+
if (coordinateError)
|
|
350
|
+
return coordinateError;
|
|
351
|
+
if (input.x === undefined || input.y === undefined)
|
|
352
|
+
return 'select_option requires x and y';
|
|
353
|
+
const selectorCount = Number(input.value !== undefined) + Number(input.label !== undefined) + Number(input.index !== undefined);
|
|
354
|
+
if (selectorCount === 0)
|
|
355
|
+
return 'select_option requires at least one of value, label, or index';
|
|
356
|
+
return undefined;
|
|
357
|
+
}
|
|
358
|
+
function addActionContractIssue(ctx, message) {
|
|
359
|
+
if (!message)
|
|
360
|
+
return;
|
|
361
|
+
ctx.addIssue({ code: z.ZodIssueCode.custom, message });
|
|
362
|
+
}
|
|
288
363
|
const fillFieldSchema = z.union([
|
|
289
364
|
z.object({
|
|
290
365
|
kind: z.literal('text'),
|
|
@@ -417,7 +492,7 @@ const batchActionSchema = z.discriminatedUnion('type', [
|
|
|
417
492
|
}).strict(),
|
|
418
493
|
z.object({
|
|
419
494
|
type: z.literal('type'),
|
|
420
|
-
text: z.string(),
|
|
495
|
+
text: z.string().max(65_536),
|
|
421
496
|
timeoutMs: timeoutMsInput,
|
|
422
497
|
}).strict(),
|
|
423
498
|
z.object({
|
|
@@ -431,25 +506,25 @@ const batchActionSchema = z.discriminatedUnion('type', [
|
|
|
431
506
|
}).strict(),
|
|
432
507
|
z.object({
|
|
433
508
|
type: z.literal('upload_files'),
|
|
434
|
-
paths: z.array(z.string()).min(1),
|
|
509
|
+
paths: z.array(z.string().trim().min(1)).min(1),
|
|
435
510
|
x: z.number().optional(),
|
|
436
511
|
y: z.number().optional(),
|
|
437
|
-
fieldLabel: z.string().optional(),
|
|
512
|
+
fieldLabel: z.string().trim().min(1).optional(),
|
|
438
513
|
exact: z.boolean().optional(),
|
|
439
514
|
strategy: z.enum(['auto', 'chooser', 'hidden', 'drop']).optional(),
|
|
440
515
|
dropX: z.number().optional(),
|
|
441
516
|
dropY: z.number().optional(),
|
|
517
|
+
contextText: z.string().trim().min(1).optional(),
|
|
518
|
+
sectionText: z.string().trim().min(1).optional(),
|
|
442
519
|
timeoutMs: timeoutMsInput,
|
|
443
520
|
}).strict(),
|
|
444
521
|
z.object({
|
|
445
522
|
type: z.literal('pick_listbox_option'),
|
|
446
|
-
label: z.string(),
|
|
523
|
+
label: z.string().trim().min(1),
|
|
447
524
|
exact: z.boolean().optional(),
|
|
448
|
-
openX: z.number().optional(),
|
|
449
|
-
openY: z.number().optional(),
|
|
450
525
|
fieldId: z.string().trim().min(1).optional(),
|
|
451
526
|
fieldKey: z.string().trim().min(1).optional(),
|
|
452
|
-
fieldLabel: z.string().
|
|
527
|
+
fieldLabel: z.string().trim().min(1),
|
|
453
528
|
query: z.string().optional(),
|
|
454
529
|
timeoutMs: timeoutMsInput,
|
|
455
530
|
}).strict(),
|
|
@@ -458,7 +533,7 @@ const batchActionSchema = z.discriminatedUnion('type', [
|
|
|
458
533
|
x: z.number(),
|
|
459
534
|
y: z.number(),
|
|
460
535
|
value: z.string().optional(),
|
|
461
|
-
label: z.string().optional(),
|
|
536
|
+
label: z.string().trim().min(1).optional(),
|
|
462
537
|
index: z.number().int().min(0).optional(),
|
|
463
538
|
timeoutMs: timeoutMsInput,
|
|
464
539
|
}).strict(),
|
|
@@ -512,10 +587,20 @@ const batchActionSchema = z.discriminatedUnion('type', [
|
|
|
512
587
|
maxTextPreview: z.number().int().min(0).max(20).optional(),
|
|
513
588
|
includeBounds: z.boolean().optional(),
|
|
514
589
|
}).strict(),
|
|
515
|
-
])
|
|
590
|
+
]).superRefine((action, ctx) => {
|
|
591
|
+
if (action.type === 'upload_files') {
|
|
592
|
+
addActionContractIssue(ctx, fileUploadContractError(action));
|
|
593
|
+
}
|
|
594
|
+
else if (action.type === 'pick_listbox_option') {
|
|
595
|
+
addActionContractIssue(ctx, listboxPickContractError(action));
|
|
596
|
+
}
|
|
597
|
+
else if (action.type === 'select_option') {
|
|
598
|
+
addActionContractIssue(ctx, selectOptionContractError(action));
|
|
599
|
+
}
|
|
600
|
+
});
|
|
516
601
|
export function createServer() {
|
|
517
|
-
const server = new McpServer(
|
|
518
|
-
const sessionIdInput = z.string().optional().describe('Session identifier returned by geometra_connect. Omit
|
|
602
|
+
const server = new McpServer(SERVER_IMPLEMENTATION, { capabilities: { tools: {} } });
|
|
603
|
+
const sessionIdInput = z.string().optional().describe('Session identifier returned by geometra_connect or an auto-connected tool. Omit only when exactly one non-isolated session is active.');
|
|
519
604
|
// ── connect ──────────────────────────────────────────────────
|
|
520
605
|
server.tool('geometra_connect', `Connect to a Geometra WebSocket peer, or start \`geometra-proxy\` automatically for a normal web page.
|
|
521
606
|
|
|
@@ -525,11 +610,16 @@ Use \`url\` (ws://…) only when a Geometra/native server or an already-running
|
|
|
525
610
|
|
|
526
611
|
Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth: true\` (or set \`GEOMETRA_STEALTH=1\`) to opt into CloakBrowser's Chromium for authorized testing instead of stock Playwright Chromium. File upload / wheel / native \`<select>\` need the proxy path (\`pageUrl\` or ws to proxy). Set \`returnForms: true\` and/or \`returnPageModel: true\` when you want a lower-turn startup response. When connect first-response latency matters more than inlining the page model, pair \`returnPageModel: true\` with \`pageModelMode: "deferred"\` and call \`geometra_page_model\` next.
|
|
527
612
|
|
|
528
|
-
**
|
|
613
|
+
**Session isolation:** browser sessions started from \`pageUrl\` (or an HTTP(S) \`url\`) are isolated by default: each gets a fresh Chromium that is destroyed on disconnect. Pass \`isolated: false\` explicitly only when you intentionally want sequential warm-browser reuse, including shared cookies, localStorage, and page state. Direct \`ws://\` connections attach to an externally managed endpoint and cannot create browser isolation.`, {
|
|
529
614
|
url: z
|
|
530
615
|
.string()
|
|
531
616
|
.optional()
|
|
532
617
|
.describe('WebSocket URL when a server is already running (e.g. ws://127.0.0.1:3200 or ws://localhost:3100). If you pass http(s) here by mistake, MCP will treat it as a page URL and start geometra-proxy.'),
|
|
618
|
+
authToken: z
|
|
619
|
+
.string()
|
|
620
|
+
.min(32)
|
|
621
|
+
.optional()
|
|
622
|
+
.describe('Bearer capability for an already-running authenticated geometra-proxy. Kept private and never returned in session metadata. Not needed when pageUrl starts the proxy automatically.'),
|
|
533
623
|
pageUrl: z
|
|
534
624
|
.string()
|
|
535
625
|
.url()
|
|
@@ -560,8 +650,7 @@ Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth
|
|
|
560
650
|
isolated: z
|
|
561
651
|
.boolean()
|
|
562
652
|
.optional()
|
|
563
|
-
.
|
|
564
|
-
.describe('When true, bypass the reusable proxy pool and spawn a brand-new Chromium for this session that is destroyed on disconnect. Required for safe parallel form submission — without this, two parallel sessions can land on the same pooled proxy and contaminate each other. Default false (use the pool for speed).'),
|
|
653
|
+
.describe('For pageUrl/HTTP(S) browser connects, omission uses a fresh Chromium that is destroyed on disconnect. Pass false explicitly to opt into sequential warm-browser reuse and shared browser state. Direct ws:// endpoints are externally managed.'),
|
|
565
654
|
proxy: z
|
|
566
655
|
.object({
|
|
567
656
|
server: z
|
|
@@ -610,6 +699,9 @@ Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth
|
|
|
610
699
|
if (!browser.ok)
|
|
611
700
|
return err(browser.error);
|
|
612
701
|
const target = normalized.value;
|
|
702
|
+
if (target.kind === 'ws' && input.isolated === true) {
|
|
703
|
+
return err('isolated:true is unsupported for direct ws:// endpoints because MCP does not own that browser/runtime. Use pageUrl/HTTP(S) url for an isolated browser, or omit isolated for the externally managed endpoint.');
|
|
704
|
+
}
|
|
613
705
|
const formSchema = {
|
|
614
706
|
formId: input.formId,
|
|
615
707
|
maxFields: input.maxFields,
|
|
@@ -639,7 +731,7 @@ Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth
|
|
|
639
731
|
height: input.height,
|
|
640
732
|
slowMo: input.slowMo,
|
|
641
733
|
...(browser.stealth !== undefined && { stealth: browser.stealth }),
|
|
642
|
-
isolated: input.isolated,
|
|
734
|
+
isolated: input.isolated !== false,
|
|
643
735
|
proxy: input.proxy,
|
|
644
736
|
awaitInitialFrame: deferInlinePageModel ? false : undefined,
|
|
645
737
|
eagerInitialExtract: deferInlinePageModel ? true : undefined,
|
|
@@ -671,6 +763,7 @@ Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth
|
|
|
671
763
|
width: input.width,
|
|
672
764
|
height: input.height,
|
|
673
765
|
awaitInitialFrame: deferInlinePageModel ? false : undefined,
|
|
766
|
+
authToken: input.authToken,
|
|
674
767
|
});
|
|
675
768
|
if (input.returnForms) {
|
|
676
769
|
await stabilizeInlineFormSchemas(session, formSchema);
|
|
@@ -702,7 +795,7 @@ Chromium runs **headless** by default unless \`headless: false\`. Pass \`stealth
|
|
|
702
795
|
// ── prepare browser ──────────────────────────────────────────
|
|
703
796
|
server.tool('geometra_prepare_browser', `Pre-launch and pre-navigate a reusable geometra-proxy browser for a normal web page without creating an active MCP session.
|
|
704
797
|
|
|
705
|
-
Use this when you can prepare ahead of the user-facing task so the next \`geometra_connect\` or one-call \`geometra_run_actions\` on the same \`pageUrl\` / viewport / headless settings skips the cold browser launch.`, {
|
|
798
|
+
Use this when you can prepare ahead of the user-facing task so the next \`geometra_connect\` or one-call \`geometra_run_actions\` on the same \`pageUrl\` / viewport / headless settings skips the cold browser launch. The consuming call must pass \`isolated: false\` explicitly; isolated browser connects intentionally ignore the warm pool.`, {
|
|
706
799
|
pageUrl: z
|
|
707
800
|
.string()
|
|
708
801
|
.url()
|
|
@@ -1005,7 +1098,7 @@ Captures the current URL, then polls until the URL changes and a stable UI tree
|
|
|
1005
1098
|
});
|
|
1006
1099
|
server.tool('geometra_fill_fields', `Fill several labeled form fields in one MCP call. This is the preferred high-level primitive for long forms.
|
|
1007
1100
|
|
|
1008
|
-
Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / comboboxes / radio-style questions addressed by field label + answer, \`"toggle"\` for individually labeled checkboxes or radios, and \`"file"\` for labeled uploads. When \`fieldId\` from \`geometra_form_schema\` is present, MCP can resolve the current label server-side so you do not need to duplicate \`fieldLabel\` / \`label\` for text, choice,
|
|
1101
|
+
Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / comboboxes / radio-style questions addressed by field label + answer, \`"toggle"\` for individually labeled checkboxes or radios, and \`"file"\` for labeled uploads. When \`fieldId\` from \`geometra_form_schema\` is present, MCP can resolve the current label server-side so you do not need to duplicate \`fieldLabel\` / \`label\` for text, choice, toggle, or file fields.`, {
|
|
1009
1102
|
fields: z.array(fillFieldSchema).min(1).max(80).describe('Ordered field operations to apply. Use fieldId from geometra_form_schema to omit duplicate fieldLabel/label on schema-backed fields.'),
|
|
1010
1103
|
stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing field (default true)'),
|
|
1011
1104
|
failOnInvalid: z
|
|
@@ -1053,12 +1146,21 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
|
|
|
1053
1146
|
fallbackFromBatch = { attempted: true, used: true, reason: 'batched-unavailable', attempts: 2 };
|
|
1054
1147
|
}
|
|
1055
1148
|
catch (e) {
|
|
1149
|
+
const ambiguity = ambiguousOutcomeDetails(e);
|
|
1150
|
+
if (ambiguity) {
|
|
1151
|
+
return err(JSON.stringify({
|
|
1152
|
+
completed: false,
|
|
1153
|
+
execution: 'batched',
|
|
1154
|
+
...ambiguousOutcomePayload(ambiguity),
|
|
1155
|
+
}, null, detail === 'verbose' ? 2 : undefined));
|
|
1156
|
+
}
|
|
1056
1157
|
const message = e instanceof Error ? e.message : String(e);
|
|
1057
1158
|
return err(message);
|
|
1058
1159
|
}
|
|
1059
1160
|
}
|
|
1060
1161
|
const steps = [];
|
|
1061
1162
|
let stoppedAt;
|
|
1163
|
+
let ambiguousAt;
|
|
1062
1164
|
for (let index = 0; index < resolvedFields.fields.length; index++) {
|
|
1063
1165
|
const field = resolvedFields.fields[index];
|
|
1064
1166
|
try {
|
|
@@ -1074,9 +1176,10 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
|
|
|
1074
1176
|
: { index, kind: field.kind, ok: true, ...result.compact });
|
|
1075
1177
|
}
|
|
1076
1178
|
catch (e) {
|
|
1077
|
-
|
|
1179
|
+
let handledError = e;
|
|
1180
|
+
let message = e instanceof Error ? e.message : String(e);
|
|
1078
1181
|
// Retry once for transient selection failures
|
|
1079
|
-
if (message.includes('selection_not_confirmed') || message.includes('still invalid after fill')) {
|
|
1182
|
+
if (!ambiguousOutcomeDetails(e) && (message.includes('selection_not_confirmed') || message.includes('still invalid after fill'))) {
|
|
1080
1183
|
try {
|
|
1081
1184
|
const retryResult = await executeFillField(session, field, detail);
|
|
1082
1185
|
steps.push(detail === 'verbose'
|
|
@@ -1084,12 +1187,25 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
|
|
|
1084
1187
|
: { index, kind: field.kind, ok: true, ...retryResult.compact, retried: true });
|
|
1085
1188
|
continue;
|
|
1086
1189
|
}
|
|
1087
|
-
catch {
|
|
1190
|
+
catch (retryError) {
|
|
1191
|
+
handledError = retryError;
|
|
1192
|
+
message = retryError instanceof Error ? retryError.message : String(retryError);
|
|
1193
|
+
}
|
|
1088
1194
|
}
|
|
1195
|
+
const ambiguity = ambiguousOutcomeDetails(handledError);
|
|
1089
1196
|
const suggestion = isResolvedFillFieldInput(field) ? suggestRecovery(field, message) : undefined;
|
|
1090
|
-
steps.push({
|
|
1091
|
-
|
|
1197
|
+
steps.push({
|
|
1198
|
+
index,
|
|
1199
|
+
kind: field.kind,
|
|
1200
|
+
ok: false,
|
|
1201
|
+
error: message,
|
|
1202
|
+
...(ambiguity ? ambiguousOutcomePayload(ambiguity) : {}),
|
|
1203
|
+
...(suggestion ? { suggestion } : {}),
|
|
1204
|
+
});
|
|
1205
|
+
if (ambiguity || stopOnError) {
|
|
1092
1206
|
stoppedAt = index;
|
|
1207
|
+
if (ambiguity)
|
|
1208
|
+
ambiguousAt = index;
|
|
1093
1209
|
break;
|
|
1094
1210
|
}
|
|
1095
1211
|
}
|
|
@@ -1106,6 +1222,13 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
|
|
|
1106
1222
|
errorCount,
|
|
1107
1223
|
...(includeSteps ? { steps } : {}),
|
|
1108
1224
|
...(stoppedAt !== undefined ? { stoppedAt } : {}),
|
|
1225
|
+
...(ambiguousAt !== undefined ? {
|
|
1226
|
+
ambiguous: true,
|
|
1227
|
+
ambiguousAt,
|
|
1228
|
+
resumeBlocked: true,
|
|
1229
|
+
resumeFromIndex: ambiguousAt,
|
|
1230
|
+
resumeInstruction: 'Inspect the current UI before resuming; do not skip the ambiguous field.',
|
|
1231
|
+
} : {}),
|
|
1109
1232
|
...(fallbackFromBatch ? { fallback: fallbackFromBatch } : {}),
|
|
1110
1233
|
...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
|
|
1111
1234
|
};
|
|
@@ -1159,8 +1282,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1159
1282
|
isolated: z
|
|
1160
1283
|
.boolean()
|
|
1161
1284
|
.optional()
|
|
1162
|
-
.default
|
|
1163
|
-
.describe('When auto-connecting via pageUrl/url, request an isolated proxy (own brand-new Chromium, destroyed on disconnect). Required for safe parallel form submission. See geometra_connect for details. Ignored when reusing an existing sessionId — set isolated on the original geometra_connect for that case.'),
|
|
1285
|
+
.describe('When auto-connecting via pageUrl/HTTP(S) url, use a fresh isolated Chromium by default. Pass false explicitly for sequential warm reuse. Ignored when using an existing sessionId.'),
|
|
1164
1286
|
detail: detailInput(),
|
|
1165
1287
|
sessionId: sessionIdInput,
|
|
1166
1288
|
}, async ({ url, pageUrl, port, headless, width, height, slowMo, stealth, browserMode, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, resumeFromIndex, verifyFills, skipPreFilled, isolated, detail, sessionId }) => {
|
|
@@ -1239,10 +1361,43 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1239
1361
|
let fallbackFromBatch;
|
|
1240
1362
|
if (!includeSteps) {
|
|
1241
1363
|
let usedBatch = false;
|
|
1364
|
+
let batchReadbackSnapshot;
|
|
1242
1365
|
try {
|
|
1243
1366
|
const startRevision = session.updateRevision;
|
|
1244
1367
|
const wait = await sendFillFields(session, toProxyFillFields(planned.fields));
|
|
1368
|
+
assertActionWaitConfirmed('Batched form fill', wait);
|
|
1245
1369
|
const ackResult = parseProxyFillAckResult(wait.result);
|
|
1370
|
+
const batchIncludesFile = planned.fields.some(field => field.kind === 'file');
|
|
1371
|
+
if (batchIncludesFile) {
|
|
1372
|
+
// Legacy proxies do not return a structured fillFields result.
|
|
1373
|
+
// Because a file chooser may already have mutated, replaying the
|
|
1374
|
+
// batch or falling back sequentially would be unsafe. Current
|
|
1375
|
+
// proxies only emit this result after attachFiles confirms the
|
|
1376
|
+
// exact input retained the requested files.
|
|
1377
|
+
if (!ackResult) {
|
|
1378
|
+
throw new AmbiguousActionOutcomeError('Batched file fill', wait, 'Batched file fill was acknowledged without structured proxy confirmation; the upload may already have happened. Do not retry blindly.');
|
|
1379
|
+
}
|
|
1380
|
+
await waitForDeferredBatchUpdate(session, startRevision, wait);
|
|
1381
|
+
const verification = verifyFills ? verifyFormFills(session, planned.planned) : undefined;
|
|
1382
|
+
const payload = {
|
|
1383
|
+
...connection,
|
|
1384
|
+
completed: true,
|
|
1385
|
+
execution: 'batched',
|
|
1386
|
+
finalSource: 'proxy',
|
|
1387
|
+
formId: schema.formId,
|
|
1388
|
+
requestedValueCount: entryCount,
|
|
1389
|
+
fieldCount: planned.fields.length,
|
|
1390
|
+
successCount: planned.fields.length,
|
|
1391
|
+
errorCount: 0,
|
|
1392
|
+
...(verification ? { verification } : {}),
|
|
1393
|
+
final: ackResult,
|
|
1394
|
+
};
|
|
1395
|
+
recordWorkflowFill(session, schema.formId, schema.name, valuesById, valuesByLabel, ackResult.invalidCount, planned.fields.length);
|
|
1396
|
+
if (failOnInvalid && ackResult.invalidCount > 0) {
|
|
1397
|
+
return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
1398
|
+
}
|
|
1399
|
+
return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
1400
|
+
}
|
|
1246
1401
|
if (ackResult && ackResult.invalidCount === 0) {
|
|
1247
1402
|
usedBatch = true;
|
|
1248
1403
|
if (!verifyFills) {
|
|
@@ -1262,10 +1417,29 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1262
1417
|
}
|
|
1263
1418
|
}
|
|
1264
1419
|
await waitForDeferredBatchUpdate(session, startRevision, wait);
|
|
1265
|
-
await waitForBatchFieldReadback(session, planned.fields);
|
|
1266
|
-
|
|
1420
|
+
batchReadbackSnapshot = await waitForBatchFieldReadback(session, planned.fields);
|
|
1421
|
+
if (!batchReadbackSnapshot) {
|
|
1422
|
+
if (!sessionA11y(session)) {
|
|
1423
|
+
throw new AmbiguousActionOutcomeError('Batched form fill', wait, 'Batched form fill was sent, but its read-back evidence became unavailable before confirmation. The fields may already have changed. Do not retry blindly.');
|
|
1424
|
+
}
|
|
1425
|
+
usedBatch = false;
|
|
1426
|
+
fallbackFromBatch = { attempted: true, used: true, reason: 'batched-invalid-readback', attempts: 2 };
|
|
1427
|
+
}
|
|
1428
|
+
else {
|
|
1429
|
+
usedBatch = true;
|
|
1430
|
+
}
|
|
1267
1431
|
}
|
|
1268
1432
|
catch (e) {
|
|
1433
|
+
const ambiguity = ambiguousOutcomeDetails(e);
|
|
1434
|
+
if (ambiguity) {
|
|
1435
|
+
return err(JSON.stringify({
|
|
1436
|
+
...connection,
|
|
1437
|
+
completed: false,
|
|
1438
|
+
execution: 'batched',
|
|
1439
|
+
formId: schema.formId,
|
|
1440
|
+
...ambiguousOutcomePayload(ambiguity),
|
|
1441
|
+
}, null, detail === 'verbose' ? 2 : undefined));
|
|
1442
|
+
}
|
|
1269
1443
|
if (!canFallbackToSequentialFill(e)) {
|
|
1270
1444
|
const message = e instanceof Error ? e.message : String(e);
|
|
1271
1445
|
return err(message);
|
|
@@ -1273,7 +1447,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1273
1447
|
fallbackFromBatch = { attempted: true, used: true, reason: 'batched-threw', attempts: 2 };
|
|
1274
1448
|
}
|
|
1275
1449
|
if (usedBatch) {
|
|
1276
|
-
const after =
|
|
1450
|
+
const after = batchReadbackSnapshot;
|
|
1277
1451
|
const signals = after
|
|
1278
1452
|
? scopeSessionSignalsToForm(after, targetFormPath ? formNodeAtPath(after, targetFormPath) : undefined, targetFormPath !== undefined)
|
|
1279
1453
|
: undefined;
|
|
@@ -1284,12 +1458,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1284
1458
|
}
|
|
1285
1459
|
}
|
|
1286
1460
|
if (usedBatch) {
|
|
1287
|
-
const after =
|
|
1461
|
+
const after = batchReadbackSnapshot;
|
|
1288
1462
|
const signals = after
|
|
1289
1463
|
? scopeSessionSignalsToForm(after, targetFormPath ? formNodeAtPath(after, targetFormPath) : undefined, targetFormPath !== undefined)
|
|
1290
1464
|
: undefined;
|
|
1291
1465
|
const invalidRemaining = signals?.invalidFields.length ?? 0;
|
|
1292
|
-
const verification = verifyFills
|
|
1466
|
+
const verification = verifyFills
|
|
1467
|
+
? verifyFormFills(session, planned.planned, batchReadbackSnapshot)
|
|
1468
|
+
: undefined;
|
|
1293
1469
|
const payload = {
|
|
1294
1470
|
...connection,
|
|
1295
1471
|
completed: true,
|
|
@@ -1315,6 +1491,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1315
1491
|
}
|
|
1316
1492
|
const steps = [];
|
|
1317
1493
|
let stoppedAt;
|
|
1494
|
+
let ambiguousAt;
|
|
1318
1495
|
const startIndex = resumeFromIndex ?? 0;
|
|
1319
1496
|
for (let index = startIndex; index < planned.fields.length; index++) {
|
|
1320
1497
|
const field = planned.fields[index];
|
|
@@ -1329,10 +1506,21 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1329
1506
|
}
|
|
1330
1507
|
catch (e) {
|
|
1331
1508
|
const message = e instanceof Error ? e.message : String(e);
|
|
1509
|
+
const ambiguity = ambiguousOutcomeDetails(e);
|
|
1332
1510
|
const suggestion = suggestRecovery(field, message);
|
|
1333
|
-
steps.push({
|
|
1334
|
-
|
|
1511
|
+
steps.push({
|
|
1512
|
+
index,
|
|
1513
|
+
kind: field.kind,
|
|
1514
|
+
ok: false,
|
|
1515
|
+
...(confidence !== undefined ? { confidence, matchMethod } : {}),
|
|
1516
|
+
error: message,
|
|
1517
|
+
...(ambiguity ? ambiguousOutcomePayload(ambiguity) : {}),
|
|
1518
|
+
...(suggestion ? { suggestion } : {}),
|
|
1519
|
+
});
|
|
1520
|
+
if (ambiguity || stopOnError) {
|
|
1335
1521
|
stoppedAt = index;
|
|
1522
|
+
if (ambiguity)
|
|
1523
|
+
ambiguousAt = index;
|
|
1336
1524
|
break;
|
|
1337
1525
|
}
|
|
1338
1526
|
}
|
|
@@ -1360,7 +1548,16 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1360
1548
|
: undefined,
|
|
1361
1549
|
...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
|
|
1362
1550
|
...(includeSteps ? { steps } : {}),
|
|
1363
|
-
...(stoppedAt !== undefined ? {
|
|
1551
|
+
...(stoppedAt !== undefined ? {
|
|
1552
|
+
stoppedAt,
|
|
1553
|
+
resumeFromIndex: ambiguousAt ?? (stoppedAt + 1),
|
|
1554
|
+
} : {}),
|
|
1555
|
+
...(ambiguousAt !== undefined ? {
|
|
1556
|
+
ambiguous: true,
|
|
1557
|
+
ambiguousAt,
|
|
1558
|
+
resumeBlocked: true,
|
|
1559
|
+
resumeInstruction: 'Inspect the current UI before resuming; do not skip the ambiguous field.',
|
|
1560
|
+
} : {}),
|
|
1364
1561
|
...(fallbackFromBatch ? { fallback: fallbackFromBatch } : {}),
|
|
1365
1562
|
...(verification ? { verification } : {}),
|
|
1366
1563
|
...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
|
|
@@ -1392,7 +1589,7 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
1392
1589
|
|
|
1393
1590
|
Pass \`valuesById\` or \`valuesByLabel\` to populate fields, \`submit\` to target the submit button (default: semantic \`{ role: 'button', name: 'Submit' }\`), and \`waitFor\` to block on the post-submit state (success banner, navigation, submit button gone, etc.). Navigation is detected automatically and surfaced as \`navigated: true\` with \`afterUrl\`.
|
|
1394
1591
|
|
|
1395
|
-
Pass \`pageUrl\`/\`url\` to auto-connect in the same call
|
|
1592
|
+
Pass \`pageUrl\`/\`url\` to auto-connect in the same call. These browser connects are isolated by default; pass \`isolated: false\` only for intentional sequential warm reuse.`, {
|
|
1396
1593
|
url: z.string().optional().describe('Optional target URL. Use a ws:// Geometra server URL or an http(s) page URL to auto-connect before submitting.'),
|
|
1397
1594
|
pageUrl: z.string().optional().describe('Optional http(s) page URL to auto-connect before submitting. Prefer this over url for browser pages.'),
|
|
1398
1595
|
port: z.number().int().min(0).max(65535).optional().describe('Preferred local port for an auto-spawned proxy (default: ephemeral OS-assigned port).'),
|
|
@@ -1402,7 +1599,7 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1402
1599
|
slowMo: z.number().int().nonnegative().optional().describe('Playwright slowMo (ms) when auto-spawning a proxy.'),
|
|
1403
1600
|
stealth: stealthInput(),
|
|
1404
1601
|
browserMode: browserModeInput(),
|
|
1405
|
-
isolated: z.boolean().optional().
|
|
1602
|
+
isolated: z.boolean().optional().describe('When auto-connecting via pageUrl/HTTP(S) url, use a fresh isolated Chromium by default. Pass false explicitly for sequential warm reuse.'),
|
|
1406
1603
|
formId: z.string().optional().describe('Optional form id from geometra_form_schema or geometra_page_model'),
|
|
1407
1604
|
valuesById: formValuesRecordSchema.optional().describe('Form values keyed by stable field id from geometra_form_schema'),
|
|
1408
1605
|
valuesByLabel: formValuesRecordSchema.optional().describe('Form values keyed by schema field label'),
|
|
@@ -1428,22 +1625,30 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1428
1625
|
const session = resolved.session;
|
|
1429
1626
|
const connection = autoConnectionPayload(resolved);
|
|
1430
1627
|
let targetFormId = formId;
|
|
1628
|
+
let targetFormIdentity;
|
|
1431
1629
|
let fillSummary;
|
|
1432
1630
|
let fillFallback;
|
|
1433
|
-
const pausedPayload = (phase, resumeHint, extra) =>
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1631
|
+
const pausedPayload = (phase, resumeHint, extra) => {
|
|
1632
|
+
const resumesSubmitForm = resumeHint?.tool === 'geometra_submit_form';
|
|
1633
|
+
const scopedResumeHint = {
|
|
1634
|
+
...(resumeHint ?? {}),
|
|
1635
|
+
...(resumesSubmitForm && targetFormId ? { formId: targetFormId } : {}),
|
|
1636
|
+
};
|
|
1637
|
+
return {
|
|
1638
|
+
...connection,
|
|
1639
|
+
completed: false,
|
|
1640
|
+
outcome: 'unconfirmed',
|
|
1641
|
+
paused: true,
|
|
1642
|
+
phase,
|
|
1643
|
+
pauseReason: 'soft-timeout',
|
|
1644
|
+
softTimeoutMs: effectiveSoftTimeoutMs,
|
|
1645
|
+
elapsedMs: Number((performance.now() - toolStartedAt).toFixed(1)),
|
|
1646
|
+
...(Object.keys(scopedResumeHint).length > 0 ? { resumeHint: scopedResumeHint } : {}),
|
|
1647
|
+
...(fillSummary ? { fill: fillSummary } : {}),
|
|
1648
|
+
...(fillFallback ? { fill_fallback: fillFallback } : {}),
|
|
1649
|
+
...(extra ?? {}),
|
|
1650
|
+
};
|
|
1651
|
+
};
|
|
1447
1652
|
if (!session.tree || !session.layout) {
|
|
1448
1653
|
await waitForUiCondition(session, () => Boolean(session.tree && session.layout), capTimeoutMs(2_000, timeoutCapFromDeadline(deadlineAt), 2_000));
|
|
1449
1654
|
}
|
|
@@ -1466,6 +1671,17 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1466
1671
|
return err(resolution.error);
|
|
1467
1672
|
const schema = resolution.schema;
|
|
1468
1673
|
targetFormId = schema.formId;
|
|
1674
|
+
const initialFormPath = parseSectionId(schema.formId);
|
|
1675
|
+
const initialFormNode = initialFormPath?.kind === 'form'
|
|
1676
|
+
? formScopeNodeAtPath(entryA11y, initialFormPath.path)
|
|
1677
|
+
: undefined;
|
|
1678
|
+
if (!initialFormNode) {
|
|
1679
|
+
return err(`Target form ${schema.formId} does not resolve to a maintainable form scope before fill`);
|
|
1680
|
+
}
|
|
1681
|
+
const capturedIdentity = captureFormScopeIdentity(initialFormNode, schema);
|
|
1682
|
+
if (!capturedIdentity.ok)
|
|
1683
|
+
return err(capturedIdentity.error);
|
|
1684
|
+
targetFormIdentity = capturedIdentity.identity;
|
|
1469
1685
|
const planned = planFormFill(schema, { valuesById, valuesByLabel }, schemas);
|
|
1470
1686
|
if (!planned.ok)
|
|
1471
1687
|
return err(planned.error);
|
|
@@ -1473,6 +1689,28 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1473
1689
|
try {
|
|
1474
1690
|
const startRevision = session.updateRevision;
|
|
1475
1691
|
const wait = await sendFillFields(session, toProxyFillFields(planned.fields), capTimeoutMs(undefined, timeoutCapFromDeadline(deadlineAt), HOST_SAFE_TOOL_TIMEOUT_MS));
|
|
1692
|
+
if (wait.status === 'timed_out') {
|
|
1693
|
+
fillSummary = {
|
|
1694
|
+
formId: schema.formId,
|
|
1695
|
+
execution: 'batched',
|
|
1696
|
+
fieldCount: planned.fields.length,
|
|
1697
|
+
...waitStatusPayload(wait),
|
|
1698
|
+
...(entryCount !== planned.fields.length ? { requestedValueCount: entryCount } : {}),
|
|
1699
|
+
};
|
|
1700
|
+
const guidance = ambiguousActionGuidance(wait);
|
|
1701
|
+
return ok(JSON.stringify(pausedPayload('fill', {
|
|
1702
|
+
tool: 'geometra_submit_form',
|
|
1703
|
+
retrySameCall: false,
|
|
1704
|
+
inspectCurrentUiFirst: true,
|
|
1705
|
+
skipFill: false,
|
|
1706
|
+
actionId: wait.actionId,
|
|
1707
|
+
requestId: wait.requestId,
|
|
1708
|
+
}, {
|
|
1709
|
+
pauseReason: 'action-outcome-ambiguous',
|
|
1710
|
+
retrySafety: 'inspect-first',
|
|
1711
|
+
guidance,
|
|
1712
|
+
}), null, detail === 'verbose' ? 2 : undefined));
|
|
1713
|
+
}
|
|
1476
1714
|
const ack = parseProxyFillAckResult(wait.result);
|
|
1477
1715
|
await waitForDeferredBatchUpdate(session, startRevision, wait);
|
|
1478
1716
|
fillSummary = {
|
|
@@ -1484,15 +1722,20 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1484
1722
|
...(ack?.invalidFields ? { invalidFields: ack.invalidFields } : {}),
|
|
1485
1723
|
...(entryCount !== planned.fields.length ? { requestedValueCount: entryCount } : {}),
|
|
1486
1724
|
};
|
|
1487
|
-
if (wait.status === 'timed_out') {
|
|
1488
|
-
return ok(JSON.stringify(pausedPayload('fill', {
|
|
1489
|
-
tool: 'geometra_submit_form',
|
|
1490
|
-
retrySameCall: true,
|
|
1491
|
-
skipFill: false,
|
|
1492
|
-
}), null, detail === 'verbose' ? 2 : undefined));
|
|
1493
|
-
}
|
|
1494
1725
|
}
|
|
1495
1726
|
catch (e) {
|
|
1727
|
+
const ambiguity = ambiguousOutcomeDetails(e);
|
|
1728
|
+
if (ambiguity) {
|
|
1729
|
+
return err(JSON.stringify({
|
|
1730
|
+
...connection,
|
|
1731
|
+
completed: false,
|
|
1732
|
+
paused: true,
|
|
1733
|
+
phase: 'fill',
|
|
1734
|
+
pauseReason: 'action-outcome-ambiguous',
|
|
1735
|
+
...ambiguousOutcomePayload(ambiguity),
|
|
1736
|
+
submit: { attempted: false, reason: 'Fill outcome must be inspected before submission.' },
|
|
1737
|
+
}, null, detail === 'verbose' ? 2 : undefined));
|
|
1738
|
+
}
|
|
1496
1739
|
if (!canFallbackToSequentialFill(e)) {
|
|
1497
1740
|
const message = e instanceof Error ? e.message : String(e);
|
|
1498
1741
|
return err(`Failed to fill form before submit: ${message}`);
|
|
@@ -1503,6 +1746,7 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1503
1746
|
if (!usedBatch) {
|
|
1504
1747
|
let successCount = 0;
|
|
1505
1748
|
let firstErr;
|
|
1749
|
+
let firstAmbiguity;
|
|
1506
1750
|
for (const field of planned.fields) {
|
|
1507
1751
|
if (!hasSoftBudget(deadlineAt)) {
|
|
1508
1752
|
fillSummary = {
|
|
@@ -1524,6 +1768,7 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1524
1768
|
}
|
|
1525
1769
|
catch (e) {
|
|
1526
1770
|
firstErr = e instanceof Error ? e.message : String(e);
|
|
1771
|
+
firstAmbiguity = ambiguousOutcomeDetails(e);
|
|
1527
1772
|
break;
|
|
1528
1773
|
}
|
|
1529
1774
|
}
|
|
@@ -1536,6 +1781,18 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1536
1781
|
...(entryCount !== planned.fields.length ? { requestedValueCount: entryCount } : {}),
|
|
1537
1782
|
};
|
|
1538
1783
|
if (firstErr !== undefined) {
|
|
1784
|
+
if (firstAmbiguity) {
|
|
1785
|
+
return err(JSON.stringify({
|
|
1786
|
+
...connection,
|
|
1787
|
+
completed: false,
|
|
1788
|
+
paused: true,
|
|
1789
|
+
phase: 'fill',
|
|
1790
|
+
pauseReason: 'action-outcome-ambiguous',
|
|
1791
|
+
fill: fillSummary,
|
|
1792
|
+
...ambiguousOutcomePayload(firstAmbiguity),
|
|
1793
|
+
submit: { attempted: false, reason: 'Fill outcome must be inspected before submission.' },
|
|
1794
|
+
}, null, detail === 'verbose' ? 2 : undefined));
|
|
1795
|
+
}
|
|
1539
1796
|
return err(JSON.stringify({
|
|
1540
1797
|
...connection,
|
|
1541
1798
|
completed: false,
|
|
@@ -1560,29 +1817,85 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1560
1817
|
return ok(JSON.stringify(pausedPayload('submit', submitResumeHint), null, detail === 'verbose' ? 2 : undefined));
|
|
1561
1818
|
}
|
|
1562
1819
|
const submitFilter = submit ?? { role: 'button', name: 'Submit' };
|
|
1820
|
+
const submitTargetFailurePayload = (error, fallback) => ({
|
|
1821
|
+
...connection,
|
|
1822
|
+
completed: false,
|
|
1823
|
+
outcome: 'unconfirmed',
|
|
1824
|
+
...(fillSummary ? { fill: fillSummary } : {}),
|
|
1825
|
+
...(fillFallback ? { fill_fallback: fillFallback } : {}),
|
|
1826
|
+
submit: { ok: false, error: `Submit target not found: ${error}` },
|
|
1827
|
+
...(fallback ? { submit_fallback: fallback } : {}),
|
|
1828
|
+
});
|
|
1829
|
+
let submitFormNode;
|
|
1830
|
+
let submitFormPath;
|
|
1831
|
+
let formScope;
|
|
1832
|
+
if (targetFormId) {
|
|
1833
|
+
const currentA11y = sessionA11y(session);
|
|
1834
|
+
const parsedForm = parseSectionId(targetFormId);
|
|
1835
|
+
const currentSchema = currentA11y
|
|
1836
|
+
? getSessionFormSchemas(session, { includeOptions: false, includeContext: 'auto' })
|
|
1837
|
+
.find(schema => schema.formId === targetFormId)
|
|
1838
|
+
: undefined;
|
|
1839
|
+
const currentForm = currentA11y && parsedForm?.kind === 'form'
|
|
1840
|
+
? formScopeNodeAtPath(currentA11y, parsedForm.path)
|
|
1841
|
+
: undefined;
|
|
1842
|
+
if (!currentA11y || parsedForm?.kind !== 'form' || !currentSchema || !currentForm) {
|
|
1843
|
+
return err(JSON.stringify(submitTargetFailurePayload(`Target form ${targetFormId} has an invalid or stale form path`), null, detail === 'verbose' ? 2 : undefined));
|
|
1844
|
+
}
|
|
1845
|
+
if (!targetFormIdentity) {
|
|
1846
|
+
const capturedIdentity = captureFormScopeIdentity(currentForm, currentSchema);
|
|
1847
|
+
if (!capturedIdentity.ok) {
|
|
1848
|
+
return err(JSON.stringify(submitTargetFailurePayload(capturedIdentity.error), null, detail === 'verbose' ? 2 : undefined));
|
|
1849
|
+
}
|
|
1850
|
+
targetFormIdentity = capturedIdentity.identity;
|
|
1851
|
+
}
|
|
1852
|
+
submitFormPath = [...parsedForm.path];
|
|
1853
|
+
formScope = { formId: targetFormId, path: submitFormPath, identity: targetFormIdentity };
|
|
1854
|
+
const scoped = currentFormScope(session, currentA11y, formScope);
|
|
1855
|
+
if (!scoped.ok) {
|
|
1856
|
+
return err(JSON.stringify(submitTargetFailurePayload(scoped.error), null, detail === 'verbose' ? 2 : undefined));
|
|
1857
|
+
}
|
|
1858
|
+
submitFormNode = scoped.node;
|
|
1859
|
+
}
|
|
1563
1860
|
const resolvedClick = await resolveClickLocationWithFallback(session, {
|
|
1564
1861
|
filter: submitFilter,
|
|
1565
1862
|
index: submitIndex,
|
|
1566
1863
|
fullyVisible: true,
|
|
1864
|
+
formScope,
|
|
1567
1865
|
revealTimeoutMs: capTimeoutMs(2_500, timeoutCapFromDeadline(deadlineAt), 2_500),
|
|
1568
1866
|
});
|
|
1569
1867
|
if (!resolvedClick.ok) {
|
|
1570
|
-
const payload =
|
|
1571
|
-
...connection,
|
|
1572
|
-
completed: false,
|
|
1573
|
-
outcome: 'unconfirmed',
|
|
1574
|
-
...(fillSummary ? { fill: fillSummary } : {}),
|
|
1575
|
-
...(fillFallback ? { fill_fallback: fillFallback } : {}),
|
|
1576
|
-
submit: { ok: false, error: `Submit target not found: ${resolvedClick.error}` },
|
|
1577
|
-
...(resolvedClick.fallback ? { submit_fallback: resolvedClick.fallback } : {}),
|
|
1578
|
-
};
|
|
1868
|
+
const payload = submitTargetFailurePayload(resolvedClick.error, resolvedClick.fallback);
|
|
1579
1869
|
return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
1580
1870
|
}
|
|
1581
1871
|
const preSubmitA11y = sessionA11y(session);
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1872
|
+
let submitPoint = { x: resolvedClick.value.x, y: resolvedClick.value.y };
|
|
1873
|
+
if (formScope) {
|
|
1874
|
+
if (!preSubmitA11y) {
|
|
1875
|
+
return err(JSON.stringify(submitTargetFailurePayload(`Target form ${formScope.formId} cannot be revalidated because the current UI tree is unavailable`), null, detail === 'verbose' ? 2 : undefined));
|
|
1876
|
+
}
|
|
1877
|
+
const scoped = currentFormScope(session, preSubmitA11y, formScope);
|
|
1878
|
+
const targetPath = resolvedClick.value.target?.path;
|
|
1879
|
+
const currentTarget = targetPath ? findNodeByPath(preSubmitA11y, targetPath) : undefined;
|
|
1880
|
+
const targetContext = currentTarget ? nodeContextForNode(preSubmitA11y, currentTarget) : undefined;
|
|
1881
|
+
if (!scoped.ok ||
|
|
1882
|
+
!currentTarget ||
|
|
1883
|
+
!targetPath ||
|
|
1884
|
+
!pathIsStrictDescendant(targetPath, formScope.path) ||
|
|
1885
|
+
!nodeMatchesFilter(currentTarget, submitFilter, targetContext)) {
|
|
1886
|
+
return err(JSON.stringify(submitTargetFailurePayload(`Target form ${formScope.formId} or its selected submit control became stale before click`), null, detail === 'verbose' ? 2 : undefined));
|
|
1887
|
+
}
|
|
1888
|
+
const currentCenter = currentVisibleClickCenter(preSubmitA11y, currentTarget);
|
|
1889
|
+
if (!currentCenter) {
|
|
1890
|
+
return err(JSON.stringify(submitTargetFailurePayload(`Target form ${formScope.formId} submit control is no longer visible at a usable click point`), null, detail === 'verbose' ? 2 : undefined));
|
|
1891
|
+
}
|
|
1892
|
+
submitPoint = currentCenter;
|
|
1893
|
+
submitFormNode = scoped.node;
|
|
1894
|
+
}
|
|
1895
|
+
else if (preSubmitA11y) {
|
|
1896
|
+
submitFormNode = resolveSubmitFormNode(preSubmitA11y, undefined, resolvedClick.value.target?.path);
|
|
1897
|
+
submitFormPath = submitFormNode?.path;
|
|
1898
|
+
}
|
|
1586
1899
|
const preSubmitSignals = preSubmitA11y
|
|
1587
1900
|
? scopeSessionSignalsToForm(preSubmitA11y, submitFormNode)
|
|
1588
1901
|
: undefined;
|
|
@@ -1630,18 +1943,38 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1630
1943
|
const beforeWaitEvidence = waitFor && beforeSubmit
|
|
1631
1944
|
? captureWaitEvidence(beforeSubmit, waitFilter)
|
|
1632
1945
|
: undefined;
|
|
1633
|
-
const clickWait = await sendClick(session,
|
|
1946
|
+
const clickWait = await sendClick(session, submitPoint.x, submitPoint.y, capTimeoutMs(submitTimeoutMs, timeoutCapFromDeadline(deadlineAt), 15_000));
|
|
1947
|
+
if (clickWait.status === 'timed_out') {
|
|
1948
|
+
const guidance = ambiguousActionGuidance(clickWait);
|
|
1949
|
+
return ok(JSON.stringify(pausedPayload('submit', {
|
|
1950
|
+
...submitResumeHint,
|
|
1951
|
+
retrySameCall: false,
|
|
1952
|
+
inspectCurrentUiFirst: true,
|
|
1953
|
+
actionId: clickWait.actionId,
|
|
1954
|
+
requestId: clickWait.requestId,
|
|
1955
|
+
}, {
|
|
1956
|
+
pauseReason: 'action-outcome-ambiguous',
|
|
1957
|
+
retrySafety: 'inspect-first',
|
|
1958
|
+
guidance,
|
|
1959
|
+
submit: {
|
|
1960
|
+
at: submitPoint,
|
|
1961
|
+
...(resolvedClick.value.target ? { target: compactNodeReference(resolvedClick.value.target) } : {}),
|
|
1962
|
+
...waitStatusPayload(clickWait),
|
|
1963
|
+
},
|
|
1964
|
+
}), null, detail === 'verbose' ? 2 : undefined));
|
|
1965
|
+
}
|
|
1634
1966
|
let waitResult;
|
|
1635
1967
|
let waitError;
|
|
1636
1968
|
if (waitFor) {
|
|
1637
1969
|
if (!hasSoftBudget(deadlineAt)) {
|
|
1638
1970
|
return ok(JSON.stringify(pausedPayload('wait_for', {
|
|
1639
1971
|
tool: 'geometra_wait_for',
|
|
1640
|
-
|
|
1972
|
+
...compactFilterPayload(waitFilter),
|
|
1641
1973
|
present: waitFor.present ?? true,
|
|
1974
|
+
...(waitFor.timeoutMs !== undefined ? { timeoutMs: waitFor.timeoutMs } : {}),
|
|
1642
1975
|
}, {
|
|
1643
1976
|
submit: {
|
|
1644
|
-
at:
|
|
1977
|
+
at: submitPoint,
|
|
1645
1978
|
...(resolvedClick.value.target ? { target: compactNodeReference(resolvedClick.value.target) } : {}),
|
|
1646
1979
|
...waitStatusPayload(clickWait),
|
|
1647
1980
|
},
|
|
@@ -1661,7 +1994,7 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1661
1994
|
}
|
|
1662
1995
|
const after = sessionA11y(session);
|
|
1663
1996
|
const signals = after
|
|
1664
|
-
? scopeSessionSignalsToForm(after, submitFormPath ?
|
|
1997
|
+
? scopeSessionSignalsToForm(after, submitFormPath ? formScopeNodeAtPath(after, submitFormPath) : undefined, submitFormPath !== undefined)
|
|
1665
1998
|
: undefined;
|
|
1666
1999
|
const clickResult = clickWait.result && typeof clickWait.result === 'object'
|
|
1667
2000
|
? clickWait.result
|
|
@@ -1681,11 +2014,11 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1681
2014
|
const waitEvidenceFresh = waitResult
|
|
1682
2015
|
? waitConditionHasFreshEvidence(beforeWaitEvidence, after, waitResult)
|
|
1683
2016
|
: false;
|
|
1684
|
-
const execution =
|
|
2017
|
+
const execution = 'completed';
|
|
1685
2018
|
const invalidCount = signals?.invalidFields.length ?? 0;
|
|
1686
2019
|
const outcome = invalidCount > 0
|
|
1687
2020
|
? 'validation_failed'
|
|
1688
|
-
:
|
|
2021
|
+
: waitError
|
|
1689
2022
|
? 'unconfirmed'
|
|
1690
2023
|
: waitFor
|
|
1691
2024
|
? waitEvidenceFresh
|
|
@@ -1712,7 +2045,7 @@ Pass \`pageUrl\`/\`url\` to auto-connect in the same call — use \`isolated: tr
|
|
|
1712
2045
|
outcome,
|
|
1713
2046
|
...(fillSummary ? { fill: fillSummary } : {}),
|
|
1714
2047
|
submit: {
|
|
1715
|
-
at:
|
|
2048
|
+
at: submitPoint,
|
|
1716
2049
|
...(resolvedClick.value.target ? { target: compactNodeReference(resolvedClick.value.target), revealSteps: resolvedClick.value.revealAttempts ?? 0 } : {}),
|
|
1717
2050
|
...waitStatusPayload(clickWait),
|
|
1718
2051
|
},
|
|
@@ -1755,8 +2088,7 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
1755
2088
|
isolated: z
|
|
1756
2089
|
.boolean()
|
|
1757
2090
|
.optional()
|
|
1758
|
-
.default
|
|
1759
|
-
.describe('When auto-connecting via pageUrl/url, request an isolated proxy. See geometra_connect for details.'),
|
|
2091
|
+
.describe('When auto-connecting via pageUrl/HTTP(S) url, use a fresh isolated Chromium by default. Pass false explicitly for sequential warm reuse.'),
|
|
1760
2092
|
actions: z.array(batchActionSchema).min(1).max(80).describe('Ordered high-level action steps to run sequentially'),
|
|
1761
2093
|
resumeFromIndex: z
|
|
1762
2094
|
.number()
|
|
@@ -1806,6 +2138,7 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
1806
2138
|
const steps = [];
|
|
1807
2139
|
let stoppedAt;
|
|
1808
2140
|
let pausedAt;
|
|
2141
|
+
let ambiguousAt;
|
|
1809
2142
|
const batchStartedAt = performance.now();
|
|
1810
2143
|
// Collect transparent-fallback signals from each step so run_actions
|
|
1811
2144
|
// surfaces them at top level regardless of `includeSteps` — otherwise
|
|
@@ -1878,6 +2211,7 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
1878
2211
|
}
|
|
1879
2212
|
catch (e) {
|
|
1880
2213
|
const message = e instanceof Error ? e.message : String(e);
|
|
2214
|
+
const ambiguity = ambiguousOutcomeDetails(e);
|
|
1881
2215
|
const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
|
|
1882
2216
|
const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
|
|
1883
2217
|
steps.push({
|
|
@@ -1888,7 +2222,15 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
1888
2222
|
cumulativeMs,
|
|
1889
2223
|
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
1890
2224
|
error: message,
|
|
2225
|
+
...(ambiguity ? ambiguousOutcomePayload(ambiguity) : {}),
|
|
1891
2226
|
});
|
|
2227
|
+
if (ambiguity) {
|
|
2228
|
+
// Continuing would silently skip a mutation that may still land.
|
|
2229
|
+
// Stop regardless of stopOnError and pin any resume to this step.
|
|
2230
|
+
ambiguousAt = index;
|
|
2231
|
+
stoppedAt = index;
|
|
2232
|
+
break;
|
|
2233
|
+
}
|
|
1892
2234
|
if (stopOnError) {
|
|
1893
2235
|
stoppedAt = index;
|
|
1894
2236
|
break;
|
|
@@ -1902,6 +2244,15 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
1902
2244
|
const completed = errorCount === 0 && stoppedAt === undefined && pausedAt === undefined && startIndex + steps.length >= actions.length;
|
|
1903
2245
|
const resumePayload = {
|
|
1904
2246
|
...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
|
|
2247
|
+
...(ambiguousAt !== undefined
|
|
2248
|
+
? {
|
|
2249
|
+
ambiguous: true,
|
|
2250
|
+
ambiguousAt,
|
|
2251
|
+
resumeBlocked: true,
|
|
2252
|
+
resumeFromIndex: ambiguousAt,
|
|
2253
|
+
resumeInstruction: 'Do not skip this ambiguous step. Inspect the current UI; an identical retry will reuse its action identity when deduplication is supported.',
|
|
2254
|
+
}
|
|
2255
|
+
: {}),
|
|
1905
2256
|
...(pausedAt !== undefined
|
|
1906
2257
|
? {
|
|
1907
2258
|
paused: true,
|
|
@@ -1999,8 +2350,7 @@ Unlike geometra_expand_section, this collapses repeated radio/button groups into
|
|
|
1999
2350
|
isolated: z
|
|
2000
2351
|
.boolean()
|
|
2001
2352
|
.optional()
|
|
2002
|
-
.default
|
|
2003
|
-
.describe('When auto-connecting via pageUrl/url, request an isolated proxy. See geometra_connect for details.'),
|
|
2353
|
+
.describe('When auto-connecting via pageUrl/HTTP(S) url, use a fresh isolated Chromium by default. Pass false explicitly for sequential warm reuse.'),
|
|
2004
2354
|
formId: z.string().optional().describe('Optional form id from geometra_page_model. If omitted, returns every form schema on the page.'),
|
|
2005
2355
|
maxFields: z.number().int().min(1).max(120).optional().default(80).describe('Cap returned fields per form'),
|
|
2006
2356
|
onlyRequiredFields: z.boolean().optional().default(false).describe('Only include required fields'),
|
|
@@ -2264,7 +2614,14 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
2264
2614
|
...(resolved.fallback ? { fallback: resolved.fallback } : {}),
|
|
2265
2615
|
};
|
|
2266
2616
|
if (wait.status === 'timed_out') {
|
|
2267
|
-
|
|
2617
|
+
const guidance = ambiguousActionGuidance(wait);
|
|
2618
|
+
return err(detailText([...lines.filter(Boolean), guidance].join('\n'), {
|
|
2619
|
+
completed: false,
|
|
2620
|
+
outcome: 'unconfirmed',
|
|
2621
|
+
retrySafety: 'inspect-first',
|
|
2622
|
+
guidance,
|
|
2623
|
+
...compact,
|
|
2624
|
+
}, detail));
|
|
2268
2625
|
}
|
|
2269
2626
|
return ok(detailText(lines.filter(Boolean).join('\n'), compact, detail));
|
|
2270
2627
|
});
|
|
@@ -2272,7 +2629,7 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
2272
2629
|
server.tool('geometra_type', `Type text into the currently focused element. First click a textbox/input with geometra_click to focus it, then use this to type.
|
|
2273
2630
|
|
|
2274
2631
|
Each character is sent as a key event through the geometry protocol. Returns a compact semantic delta when possible, otherwise a short current-UI overview.`, {
|
|
2275
|
-
text: z.string().describe('Text to type into the focused element'),
|
|
2632
|
+
text: z.string().max(65_536).describe('Text to type into the focused element (maximum 65,536 characters)'),
|
|
2276
2633
|
timeoutMs: z
|
|
2277
2634
|
.number()
|
|
2278
2635
|
.int()
|
|
@@ -2365,32 +2722,40 @@ Detection is fully generic (no site branding). It refuses to run if the detected
|
|
|
2365
2722
|
}
|
|
2366
2723
|
});
|
|
2367
2724
|
// ── upload files (proxy) ───────────────────────────────────────
|
|
2368
|
-
server.
|
|
2725
|
+
server.registerTool('geometra_upload_files', {
|
|
2726
|
+
description: `Attach local files to a file input. Requires \`@geometra/proxy\` (paths exist on the proxy host).
|
|
2369
2727
|
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2728
|
+
Every upload must name one explicit target. **auto** (default) accepts either a complete x,y chooser target or a labeled field. **hidden** requires fieldLabel. **chooser** requires x,y. **drop** requires dropX,dropY. Geometra rejects incomplete, conflicting, and global-first-input targets.`,
|
|
2729
|
+
inputSchema: z.object({
|
|
2730
|
+
paths: z.array(z.string().trim().min(1)).min(1).describe('Absolute paths on the proxy machine, e.g. /Users/you/resume.pdf'),
|
|
2731
|
+
x: z.number().optional().describe('Click X to trigger native file chooser; must be paired with y'),
|
|
2732
|
+
y: z.number().optional().describe('Click Y to trigger native file chooser; must be paired with x'),
|
|
2733
|
+
fieldLabel: z.string().trim().min(1).optional().describe('Specific labeled file field (for example "Resume" or "Cover letter")'),
|
|
2734
|
+
exact: z.boolean().optional().describe('Exact match when using fieldLabel'),
|
|
2735
|
+
strategy: z
|
|
2736
|
+
.enum(['auto', 'chooser', 'hidden', 'drop'])
|
|
2737
|
+
.optional()
|
|
2738
|
+
.describe('Upload strategy (default auto)'),
|
|
2739
|
+
dropX: z.number().optional().describe('Drop target X (viewport) for strategy drop; must be paired with dropY'),
|
|
2740
|
+
dropY: z.number().optional().describe('Drop target Y (viewport) for strategy drop; must be paired with dropX'),
|
|
2741
|
+
contextText: z.string().trim().min(1).optional().describe('Ancestor / prompt text to disambiguate repeated file inputs; requires fieldLabel'),
|
|
2742
|
+
sectionText: z.string().trim().min(1).optional().describe('Containing section text to disambiguate repeated file inputs; requires fieldLabel'),
|
|
2743
|
+
timeoutMs: z
|
|
2744
|
+
.number()
|
|
2745
|
+
.int()
|
|
2746
|
+
.min(50)
|
|
2747
|
+
.max(60_000)
|
|
2748
|
+
.optional()
|
|
2749
|
+
.describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
|
|
2750
|
+
detail: detailInput(),
|
|
2751
|
+
sessionId: sessionIdInput,
|
|
2752
|
+
}).strict().superRefine((input, ctx) => {
|
|
2753
|
+
addActionContractIssue(ctx, fileUploadContractError(input));
|
|
2754
|
+
}),
|
|
2755
|
+
}, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, contextText, sectionText, timeoutMs, detail, sessionId }) => {
|
|
2756
|
+
const contractError = fileUploadContractError({ x, y, fieldLabel, exact, strategy, dropX, dropY, contextText, sectionText });
|
|
2757
|
+
if (contractError)
|
|
2758
|
+
return err(contractError);
|
|
2394
2759
|
const sessionResult = resolveToolSession(sessionId);
|
|
2395
2760
|
if ('error' in sessionResult)
|
|
2396
2761
|
return sessionResult.error;
|
|
@@ -2403,12 +2768,16 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
2403
2768
|
exact,
|
|
2404
2769
|
strategy,
|
|
2405
2770
|
drop: dropX !== undefined && dropY !== undefined ? { x: dropX, y: dropY } : undefined,
|
|
2771
|
+
contextText,
|
|
2772
|
+
sectionText,
|
|
2406
2773
|
}, timeoutMs ?? 8_000);
|
|
2407
2774
|
const summary = postActionSummary(session, before, wait, detail);
|
|
2408
2775
|
return actionWaitResponse(wait, `Uploaded ${paths.length} file(s).\n${summary}`, {
|
|
2409
2776
|
fileCount: paths.length,
|
|
2410
2777
|
...(fieldLabel ? { fieldLabel } : {}),
|
|
2411
2778
|
...(strategy ? { strategy } : {}),
|
|
2779
|
+
...(contextText ? { contextText } : {}),
|
|
2780
|
+
...(sectionText ? { sectionText } : {}),
|
|
2412
2781
|
...waitStatusPayload(wait),
|
|
2413
2782
|
...(fieldLabel ? { readback: fieldStatePayload(session, fieldLabel) } : {}),
|
|
2414
2783
|
}, detail);
|
|
@@ -2417,41 +2786,41 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
2417
2786
|
return err(e.message);
|
|
2418
2787
|
}
|
|
2419
2788
|
});
|
|
2420
|
-
server.
|
|
2789
|
+
server.registerTool('geometra_pick_listbox_option', {
|
|
2790
|
+
description: `Pick an option from a custom dropdown / listbox / searchable combobox (Headless UI, React Select, Radix, Ashby-style custom selects, etc.). Requires \`@geometra/proxy\`.
|
|
2421
2791
|
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
}, async ({ label, exact,
|
|
2792
|
+
fieldLabel is required so Geometra can confirm which control committed the selection. fieldId/fieldKey are supplemental exact identity and are only meaningful alongside fieldLabel. Coordinate-only opening is intentionally unsupported because it cannot provide field readback. If the opened control is editable, MCP types query (or the option label by default) before selecting.`,
|
|
2793
|
+
inputSchema: z.object({
|
|
2794
|
+
label: z.string().trim().min(1).describe('Accessible name of the option (visible text or aria-label)'),
|
|
2795
|
+
exact: z.boolean().optional().describe('Exact name match'),
|
|
2796
|
+
fieldId: z.string().trim().min(1).optional().describe('Stable field id from geometra_form_schema; used with fieldLabel'),
|
|
2797
|
+
fieldKey: z.string().trim().min(1).optional().describe('Authored field key from geometra_form_schema for exact resolution; used with fieldLabel'),
|
|
2798
|
+
fieldLabel: z.string().trim().min(1).describe('Field label of the dropdown/combobox to open and verify (e.g. "Location")'),
|
|
2799
|
+
query: z.string().optional().describe('Optional text to type into a searchable combobox before selecting'),
|
|
2800
|
+
timeoutMs: z
|
|
2801
|
+
.number()
|
|
2802
|
+
.int()
|
|
2803
|
+
.min(50)
|
|
2804
|
+
.max(60_000)
|
|
2805
|
+
.optional()
|
|
2806
|
+
.describe('Optional action wait timeout for slow dropdowns / remote search results'),
|
|
2807
|
+
detail: detailInput(),
|
|
2808
|
+
sessionId: sessionIdInput,
|
|
2809
|
+
}).strict().superRefine((input, ctx) => {
|
|
2810
|
+
addActionContractIssue(ctx, listboxPickContractError(input));
|
|
2811
|
+
}),
|
|
2812
|
+
}, async ({ label, exact, fieldId, fieldKey, fieldLabel, query, timeoutMs, detail, sessionId }) => {
|
|
2813
|
+
const contractError = listboxPickContractError({ fieldId, fieldKey, fieldLabel });
|
|
2814
|
+
if (contractError)
|
|
2815
|
+
return err(contractError);
|
|
2443
2816
|
const sessionResult = resolveToolSession(sessionId);
|
|
2444
2817
|
if ('error' in sessionResult)
|
|
2445
2818
|
return sessionResult.error;
|
|
2446
2819
|
const session = sessionResult.session;
|
|
2447
2820
|
const before = sessionA11y(session);
|
|
2448
|
-
if (contextText || sectionText) {
|
|
2449
|
-
return err('contextText/sectionText are not supported for listbox resolution; pass fieldKey from geometra_form_schema instead.');
|
|
2450
|
-
}
|
|
2451
2821
|
try {
|
|
2452
2822
|
const wait = await sendListboxPick(session, label, {
|
|
2453
2823
|
exact,
|
|
2454
|
-
open: openX !== undefined && openY !== undefined ? { x: openX, y: openY } : undefined,
|
|
2455
2824
|
fieldId,
|
|
2456
2825
|
fieldKey,
|
|
2457
2826
|
fieldLabel,
|
|
@@ -2478,33 +2847,36 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
2478
2847
|
}
|
|
2479
2848
|
});
|
|
2480
2849
|
// ── select option (proxy, native <select>) ─────────────────────
|
|
2481
|
-
server.
|
|
2850
|
+
server.registerTool('geometra_select_option', {
|
|
2851
|
+
description: `Set a native HTML \`<select>\` after clicking its center (x,y from geometra_query). Requires \`@geometra/proxy\`.
|
|
2482
2852
|
|
|
2483
|
-
Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbox_option\` for custom dropdowns / searchable comboboxes.`,
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2853
|
+
Provide one or more of value, label, and index. Every supplied selector is conjunctive identity and must describe the same enabled option. Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbox_option\` for custom dropdowns / searchable comboboxes.`,
|
|
2854
|
+
inputSchema: z.object({
|
|
2855
|
+
x: z.number().describe('X coordinate (e.g. center of the select from geometra_query)'),
|
|
2856
|
+
y: z.number().describe('Y coordinate'),
|
|
2857
|
+
value: z.string().optional().describe('Exact option value= attribute'),
|
|
2858
|
+
label: z.string().trim().min(1).optional().describe('Exact normalized visible option label'),
|
|
2859
|
+
index: z.number().int().min(0).optional().describe('Zero-based option index'),
|
|
2860
|
+
timeoutMs: z
|
|
2861
|
+
.number()
|
|
2862
|
+
.int()
|
|
2863
|
+
.min(50)
|
|
2864
|
+
.max(60_000)
|
|
2865
|
+
.optional()
|
|
2866
|
+
.describe('Optional action wait timeout'),
|
|
2867
|
+
detail: detailInput(),
|
|
2868
|
+
sessionId: sessionIdInput,
|
|
2869
|
+
}).strict().superRefine((input, ctx) => {
|
|
2870
|
+
addActionContractIssue(ctx, selectOptionContractError(input));
|
|
2871
|
+
}),
|
|
2872
|
+
}, async ({ x, y, value, label, index, timeoutMs, detail, sessionId }) => {
|
|
2873
|
+
const contractError = selectOptionContractError({ x, y, value, label, index });
|
|
2874
|
+
if (contractError)
|
|
2875
|
+
return err(contractError);
|
|
2501
2876
|
const sessionResult = resolveToolSession(sessionId);
|
|
2502
2877
|
if ('error' in sessionResult)
|
|
2503
2878
|
return sessionResult.error;
|
|
2504
2879
|
const session = sessionResult.session;
|
|
2505
|
-
if (value === undefined && label === undefined && index === undefined) {
|
|
2506
|
-
return err('Provide at least one of value, label, or index');
|
|
2507
|
-
}
|
|
2508
2880
|
const before = sessionA11y(session);
|
|
2509
2881
|
try {
|
|
2510
2882
|
const wait = await sendSelectOption(session, x, y, { value, label, index }, timeoutMs);
|
|
@@ -2747,7 +3119,7 @@ For a token-efficient semantic view, use geometra_snapshot (default compact). Fo
|
|
|
2747
3119
|
return ok(JSON.stringify(session.layout, null, 2));
|
|
2748
3120
|
});
|
|
2749
3121
|
// ── workflow state ───────────────────────────────────────────
|
|
2750
|
-
server.tool('geometra_workflow_state', `Get the accumulated workflow state across page navigations. Shows which pages/forms have been filled,
|
|
3122
|
+
server.tool('geometra_workflow_state', `Get the accumulated workflow state across page navigations. Shows which pages/forms and field identities have been filled, with the fill status per page. Submitted values and local file paths are never stored or returned; every recorded field is represented by a redaction marker.
|
|
2751
3123
|
|
|
2752
3124
|
Use this after navigating to a new page in a multi-step flow (e.g. job applications) to understand what has been completed so far. Pass \`clear: true\` to reset the workflow state.`, {
|
|
2753
3125
|
clear: z.boolean().optional().default(false).describe('Reset the workflow state'),
|
|
@@ -2764,6 +3136,8 @@ Use this after navigating to a new page in a multi-step flow (e.g. job applicati
|
|
|
2764
3136
|
if (!session.workflowState || session.workflowState.pages.length === 0) {
|
|
2765
3137
|
return ok(JSON.stringify({
|
|
2766
3138
|
pageCount: 0,
|
|
3139
|
+
valuesRedacted: true,
|
|
3140
|
+
redactionNotice: 'Submitted values and local file paths are not stored.',
|
|
2767
3141
|
message: 'No workflow state recorded yet. Fill a form with geometra_fill_form to start tracking.',
|
|
2768
3142
|
}));
|
|
2769
3143
|
}
|
|
@@ -2775,13 +3149,18 @@ Use this after navigating to a new page in a multi-step flow (e.g. job applicati
|
|
|
2775
3149
|
totalFieldsFilled: totalFields,
|
|
2776
3150
|
totalInvalidRemaining: totalInvalid,
|
|
2777
3151
|
elapsedMs: Date.now() - state.startedAt,
|
|
3152
|
+
valuesRedacted: true,
|
|
3153
|
+
redactionNotice: 'Submitted values and local file paths are not stored; filledFields lists identities and filledValues contains redaction markers only.',
|
|
2778
3154
|
pages: state.pages.map(p => ({
|
|
2779
3155
|
pageUrl: p.pageUrl,
|
|
2780
3156
|
...(p.formId ? { formId: p.formId } : {}),
|
|
2781
3157
|
...(p.formName ? { formName: p.formName } : {}),
|
|
2782
3158
|
fieldCount: p.fieldCount,
|
|
2783
3159
|
invalidCount: p.invalidCount,
|
|
3160
|
+
filledFields: p.filledFields,
|
|
3161
|
+
// Preserve the established response key without retaining values.
|
|
2784
3162
|
filledValues: p.filledValues,
|
|
3163
|
+
valuesRedacted: true,
|
|
2785
3164
|
})),
|
|
2786
3165
|
}));
|
|
2787
3166
|
});
|
|
@@ -2850,12 +3229,19 @@ Returns \`{ pdf, pageUrl }\` where \`pdf\` is the base64-encoded PDF bytes.`, {
|
|
|
2850
3229
|
return err(`PDF generation failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2851
3230
|
}
|
|
2852
3231
|
});
|
|
2853
|
-
server.tool('geometra_disconnect',
|
|
2854
|
-
closeBrowser: z.boolean().optional().default(false).describe(
|
|
3232
|
+
server.tool('geometra_disconnect', 'Disconnect one explicitly resolved Geometra session. Isolated browser sessions are always destroyed. A session created with isolated: false may return its browser to the warm pool; pass closeBrowser: true to destroy that session\'s browser only. This tool never closes another session\'s browser.', {
|
|
3233
|
+
closeBrowser: z.boolean().optional().default(false).describe("Destroy this session's spawned proxy/browser instead of returning an explicitly reusable browser to the warm pool"),
|
|
2855
3234
|
sessionId: sessionIdInput,
|
|
2856
3235
|
}, async ({ closeBrowser, sessionId }) => {
|
|
2857
|
-
|
|
2858
|
-
|
|
3236
|
+
const resolved = resolveToolSession(sessionId);
|
|
3237
|
+
if ('error' in resolved)
|
|
3238
|
+
return resolved.error;
|
|
3239
|
+
const target = resolved.session;
|
|
3240
|
+
disconnect({ closeProxy: closeBrowser, sessionId: target.id });
|
|
3241
|
+
const browserClosed = closeBrowser || target.isolated === true;
|
|
3242
|
+
return ok(browserClosed
|
|
3243
|
+
? `Disconnected session ${target.id} and closed its browser.`
|
|
3244
|
+
: `Disconnected session ${target.id}; its explicitly reusable browser remains warm.`);
|
|
2859
3245
|
});
|
|
2860
3246
|
server.tool('geometra_list_sessions', 'List all active Geometra sessions with their IDs and URLs. Use this to discover available sessions when operating on multiple pages in parallel.', {}, async () => {
|
|
2861
3247
|
const sessions = listSessions();
|
|
@@ -2886,8 +3272,17 @@ function connectPayload(session, opts) {
|
|
|
2886
3272
|
...(opts.detail === 'verbose' && a11y ? { currentUi: sessionOverviewFromA11y(a11y) } : {}),
|
|
2887
3273
|
};
|
|
2888
3274
|
}
|
|
3275
|
+
function sessionHasFreshUi(session) {
|
|
3276
|
+
// `ws` is required on real Sessions. The absence allowance keeps legacy
|
|
3277
|
+
// in-process test doubles compatible while production always verifies the
|
|
3278
|
+
// transport is OPEN, closing the CLOSING -> close-event stale-read window.
|
|
3279
|
+
const transportOpen = !session.ws || session.ws.readyState === session.ws.OPEN;
|
|
3280
|
+
return transportOpen
|
|
3281
|
+
&& session.hasFreshFrame !== false
|
|
3282
|
+
&& Boolean(session.tree && session.layout);
|
|
3283
|
+
}
|
|
2889
3284
|
function sessionA11y(session) {
|
|
2890
|
-
if (!session
|
|
3285
|
+
if (!sessionHasFreshUi(session))
|
|
2891
3286
|
return null;
|
|
2892
3287
|
if (session.cachedA11yRevision === session.updateRevision) {
|
|
2893
3288
|
return session.cachedA11y ?? null;
|
|
@@ -2898,9 +3293,17 @@ function sessionA11y(session) {
|
|
|
2898
3293
|
return a11y;
|
|
2899
3294
|
}
|
|
2900
3295
|
async function ensureSessionUiTree(session, timeoutMs = 4_000) {
|
|
2901
|
-
if (session
|
|
3296
|
+
if (sessionHasFreshUi(session))
|
|
2902
3297
|
return true;
|
|
2903
|
-
|
|
3298
|
+
try {
|
|
3299
|
+
await ensureSessionConnected(session);
|
|
3300
|
+
}
|
|
3301
|
+
catch {
|
|
3302
|
+
return false;
|
|
3303
|
+
}
|
|
3304
|
+
if (sessionHasFreshUi(session))
|
|
3305
|
+
return true;
|
|
3306
|
+
return await waitForUiCondition(session, () => sessionHasFreshUi(session), timeoutMs);
|
|
2904
3307
|
}
|
|
2905
3308
|
async function sessionA11yWhenReady(session, timeoutMs = 4_000) {
|
|
2906
3309
|
const ready = await ensureSessionUiTree(session, timeoutMs);
|
|
@@ -3058,7 +3461,7 @@ function connectResponsePayload(session, opts) {
|
|
|
3058
3461
|
function deferredPageModelConnectPayload(session, options) {
|
|
3059
3462
|
return {
|
|
3060
3463
|
deferred: true,
|
|
3061
|
-
ready:
|
|
3464
|
+
ready: sessionHasFreshUi(session),
|
|
3062
3465
|
tool: 'geometra_page_model',
|
|
3063
3466
|
options: {
|
|
3064
3467
|
maxPrimaryActions: options?.maxPrimaryActions ?? 6,
|
|
@@ -3101,6 +3504,12 @@ async function ensureToolSession(target, missingConnectionMessage = 'Not connect
|
|
|
3101
3504
|
if (!normalized.ok)
|
|
3102
3505
|
return { ok: false, error: normalized.error };
|
|
3103
3506
|
const resolvedTarget = normalized.value;
|
|
3507
|
+
if (resolvedTarget.kind === 'ws' && target.isolated === true) {
|
|
3508
|
+
return {
|
|
3509
|
+
ok: false,
|
|
3510
|
+
error: 'isolated:true is unsupported for direct ws:// endpoints because MCP does not own that browser/runtime. Use pageUrl/HTTP(S) url for an isolated browser, or omit isolated for the externally managed endpoint.',
|
|
3511
|
+
};
|
|
3512
|
+
}
|
|
3104
3513
|
try {
|
|
3105
3514
|
if (resolvedTarget.kind === 'proxy') {
|
|
3106
3515
|
const session = await connectThroughProxy({
|
|
@@ -3112,7 +3521,7 @@ async function ensureToolSession(target, missingConnectionMessage = 'Not connect
|
|
|
3112
3521
|
slowMo: target.slowMo,
|
|
3113
3522
|
...(target.stealth !== undefined && { stealth: target.stealth }),
|
|
3114
3523
|
awaitInitialFrame: target.awaitInitialFrame,
|
|
3115
|
-
isolated: target.isolated,
|
|
3524
|
+
isolated: target.isolated !== false,
|
|
3116
3525
|
});
|
|
3117
3526
|
return {
|
|
3118
3527
|
ok: true,
|
|
@@ -3145,6 +3554,7 @@ function autoConnectionPayload(target) {
|
|
|
3145
3554
|
return {};
|
|
3146
3555
|
return {
|
|
3147
3556
|
autoConnected: true,
|
|
3557
|
+
sessionId: target.session.id,
|
|
3148
3558
|
...(target.transport ? { transport: target.transport } : {}),
|
|
3149
3559
|
...(target.requestedPageUrl ? { pageUrl: target.requestedPageUrl } : {}),
|
|
3150
3560
|
...(target.requestedWsUrl ? { requestedWsUrl: target.requestedWsUrl } : {}),
|
|
@@ -3168,8 +3578,8 @@ function postActionSummary(session, before, wait, detail = 'minimal') {
|
|
|
3168
3578
|
}
|
|
3169
3579
|
if (wait?.status === 'timed_out') {
|
|
3170
3580
|
notes.push(detail === 'verbose'
|
|
3171
|
-
? `No
|
|
3172
|
-
: `No
|
|
3581
|
+
? `No correlated terminal response arrived within ${wait.timeoutMs}ms. The action may still be running or may already have succeeded. ${ambiguousActionGuidance(wait)}`
|
|
3582
|
+
: `No correlated response arrived within ${wait.timeoutMs}ms; the action may still have succeeded. Do not retry blindly.`);
|
|
3173
3583
|
}
|
|
3174
3584
|
if (!after)
|
|
3175
3585
|
return [...notes, 'No UI update received'].filter(Boolean).join('\n');
|
|
@@ -3203,10 +3613,100 @@ function summarizeCompactContext(context) {
|
|
|
3203
3613
|
}
|
|
3204
3614
|
return parts.length > 0 ? `Context: ${parts.join(' | ')}` : '';
|
|
3205
3615
|
}
|
|
3616
|
+
function pathIsStrictDescendant(path, ancestorPath) {
|
|
3617
|
+
return path.length > ancestorPath.length && ancestorPath.every((segment, index) => path[index] === segment);
|
|
3618
|
+
}
|
|
3206
3619
|
function formNodeAtPath(root, path) {
|
|
3207
3620
|
const node = findNodeByPath(root, path);
|
|
3208
3621
|
return node?.role === 'form' ? node : undefined;
|
|
3209
3622
|
}
|
|
3623
|
+
function formScopeNodeAtPath(root, path) {
|
|
3624
|
+
const node = findNodeByPath(root, path);
|
|
3625
|
+
return node && (node.role === 'form' || node.role === 'group' || node.role === 'region')
|
|
3626
|
+
? node
|
|
3627
|
+
: undefined;
|
|
3628
|
+
}
|
|
3629
|
+
function formScopeFieldAnchor(field) {
|
|
3630
|
+
if (field.fieldKey)
|
|
3631
|
+
return `key:${field.fieldKey}`;
|
|
3632
|
+
return [
|
|
3633
|
+
'field',
|
|
3634
|
+
field.kind,
|
|
3635
|
+
normalizeLookupKey(field.label),
|
|
3636
|
+
field.controlType ?? '',
|
|
3637
|
+
field.choiceType ?? '',
|
|
3638
|
+
field.booleanChoice ? 'boolean' : '',
|
|
3639
|
+
].join(':');
|
|
3640
|
+
}
|
|
3641
|
+
function captureFormScopeIdentity(node, schema) {
|
|
3642
|
+
const fieldAnchors = schema.fields.map(formScopeFieldAnchor).sort();
|
|
3643
|
+
const name = normalizeLookupKey(node.name || schema.name || '');
|
|
3644
|
+
if (fieldAnchors.length === 0 && !name) {
|
|
3645
|
+
return {
|
|
3646
|
+
ok: false,
|
|
3647
|
+
error: `Target form ${schema.formId} has no stable semantic or field identity; refusing to submit without a maintainable form scope`,
|
|
3648
|
+
};
|
|
3649
|
+
}
|
|
3650
|
+
return {
|
|
3651
|
+
ok: true,
|
|
3652
|
+
identity: {
|
|
3653
|
+
role: node.role,
|
|
3654
|
+
name,
|
|
3655
|
+
fieldAnchors,
|
|
3656
|
+
},
|
|
3657
|
+
};
|
|
3658
|
+
}
|
|
3659
|
+
function containsFormFieldAnchors(current, expected) {
|
|
3660
|
+
const counts = new Map();
|
|
3661
|
+
for (const anchor of current)
|
|
3662
|
+
counts.set(anchor, (counts.get(anchor) ?? 0) + 1);
|
|
3663
|
+
for (const anchor of expected) {
|
|
3664
|
+
const count = counts.get(anchor) ?? 0;
|
|
3665
|
+
if (count === 0)
|
|
3666
|
+
return false;
|
|
3667
|
+
counts.set(anchor, count - 1);
|
|
3668
|
+
}
|
|
3669
|
+
return true;
|
|
3670
|
+
}
|
|
3671
|
+
function currentFormScope(session, root, scope) {
|
|
3672
|
+
const node = formScopeNodeAtPath(root, scope.path);
|
|
3673
|
+
const schema = getSessionFormSchemas(session, {
|
|
3674
|
+
includeOptions: false,
|
|
3675
|
+
includeContext: 'auto',
|
|
3676
|
+
}).find(candidate => candidate.formId === scope.formId);
|
|
3677
|
+
if (!node || !schema) {
|
|
3678
|
+
return { ok: false, error: `Target form ${scope.formId} is stale or no longer resolves to a form in the current UI` };
|
|
3679
|
+
}
|
|
3680
|
+
const currentName = normalizeLookupKey(node.name || schema.name || '');
|
|
3681
|
+
const currentAnchors = schema.fields.map(formScopeFieldAnchor).sort();
|
|
3682
|
+
if (node.role !== scope.identity.role ||
|
|
3683
|
+
currentName !== scope.identity.name ||
|
|
3684
|
+
!containsFormFieldAnchors(currentAnchors, scope.identity.fieldAnchors)) {
|
|
3685
|
+
return {
|
|
3686
|
+
ok: false,
|
|
3687
|
+
error: `Target form ${scope.formId} was replaced at the same path; its semantic/schema identity no longer matches`,
|
|
3688
|
+
};
|
|
3689
|
+
}
|
|
3690
|
+
return { ok: true, node, schema };
|
|
3691
|
+
}
|
|
3692
|
+
function currentVisibleClickCenter(root, target) {
|
|
3693
|
+
const { x, y, width, height } = target.bounds;
|
|
3694
|
+
const viewport = root.bounds;
|
|
3695
|
+
if (![x, y, width, height, viewport.x, viewport.y, viewport.width, viewport.height].every(Number.isFinite))
|
|
3696
|
+
return undefined;
|
|
3697
|
+
if (width <= 0 || height <= 0 || viewport.width <= 0 || viewport.height <= 0)
|
|
3698
|
+
return undefined;
|
|
3699
|
+
const visibleLeft = Math.max(x, viewport.x);
|
|
3700
|
+
const visibleTop = Math.max(y, viewport.y);
|
|
3701
|
+
const visibleRight = Math.min(x + width, viewport.x + viewport.width);
|
|
3702
|
+
const visibleBottom = Math.min(y + height, viewport.y + viewport.height);
|
|
3703
|
+
if (visibleRight <= visibleLeft || visibleBottom <= visibleTop)
|
|
3704
|
+
return undefined;
|
|
3705
|
+
return {
|
|
3706
|
+
x: visibleLeft + (visibleRight - visibleLeft) / 2,
|
|
3707
|
+
y: visibleTop + (visibleBottom - visibleTop) / 2,
|
|
3708
|
+
};
|
|
3709
|
+
}
|
|
3210
3710
|
function resolveSubmitFormNode(root, formId, submitPath) {
|
|
3211
3711
|
const parsedForm = formId ? parseSectionId(formId) : null;
|
|
3212
3712
|
if (parsedForm?.kind === 'form') {
|
|
@@ -3422,7 +3922,11 @@ function fieldStatePayload(session, fieldLabel, fieldKey) {
|
|
|
3422
3922
|
function waitStatusPayload(wait) {
|
|
3423
3923
|
if (!wait)
|
|
3424
3924
|
return {};
|
|
3425
|
-
const payload = {
|
|
3925
|
+
const payload = {
|
|
3926
|
+
wait: wait.status,
|
|
3927
|
+
requestId: wait.requestId,
|
|
3928
|
+
actionId: wait.actionId,
|
|
3929
|
+
};
|
|
3426
3930
|
// Surface navigation info from proxy click handlers so callers can tell
|
|
3427
3931
|
// when a click triggered a full-page nav (form submit → thank-you page).
|
|
3428
3932
|
// Without this, the proxy session may die on the next request and the
|
|
@@ -3555,9 +4059,19 @@ async function revealSemanticTarget(session, options) {
|
|
|
3555
4059
|
const a11y = sessionA11y(session);
|
|
3556
4060
|
if (!a11y)
|
|
3557
4061
|
return { ok: false, error: 'No UI tree available to reveal from' };
|
|
3558
|
-
|
|
4062
|
+
if (options.formScope) {
|
|
4063
|
+
const scoped = currentFormScope(session, a11y, options.formScope);
|
|
4064
|
+
if (!scoped.ok) {
|
|
4065
|
+
return { ok: false, error: scoped.error, staleFormScope: true };
|
|
4066
|
+
}
|
|
4067
|
+
}
|
|
4068
|
+
// Match against the page root first so prompt/section/item context from
|
|
4069
|
+
// ancestors outside the form remains available, then constrain the
|
|
4070
|
+
// actionable candidates to descendants of the selected current form.
|
|
4071
|
+
const matches = sortA11yNodes(findNodes(a11y, options.filter).filter(node => !options.formScope || pathIsStrictDescendant(node.path, options.formScope.path)));
|
|
3559
4072
|
if (matches.length === 0) {
|
|
3560
|
-
|
|
4073
|
+
const scopeSuffix = options.formScope ? ` within form ${options.formScope.formId}` : '';
|
|
4074
|
+
return { ok: false, error: `No elements found matching ${JSON.stringify(options.filter)}${scopeSuffix}` };
|
|
3561
4075
|
}
|
|
3562
4076
|
if (options.index >= matches.length) {
|
|
3563
4077
|
return {
|
|
@@ -3626,6 +4140,7 @@ async function resolveClickLocation(session, options) {
|
|
|
3626
4140
|
filter: options.filter,
|
|
3627
4141
|
index: options.index ?? 0,
|
|
3628
4142
|
fullyVisible: options.fullyVisible ?? true,
|
|
4143
|
+
formScope: options.formScope,
|
|
3629
4144
|
maxSteps: options.maxRevealSteps,
|
|
3630
4145
|
timeoutMs: options.revealTimeoutMs ?? 2_500,
|
|
3631
4146
|
});
|
|
@@ -3645,6 +4160,8 @@ async function resolveClickLocationWithFallback(session, options) {
|
|
|
3645
4160
|
const first = await resolveClickLocation(session, options);
|
|
3646
4161
|
if (first.ok)
|
|
3647
4162
|
return first;
|
|
4163
|
+
if (first.staleFormScope)
|
|
4164
|
+
return first;
|
|
3648
4165
|
// Fallback only applies to semantic resolves. Explicit coordinates never enter
|
|
3649
4166
|
// the reveal path, so there is nothing to retry.
|
|
3650
4167
|
const hasExplicitCoordinates = options.x !== undefined || options.y !== undefined;
|
|
@@ -3667,6 +4184,8 @@ async function resolveClickLocationWithFallback(session, options) {
|
|
|
3667
4184
|
fallback: { attempted: true, used: true, reason: 'revision-retry', attempts },
|
|
3668
4185
|
};
|
|
3669
4186
|
}
|
|
4187
|
+
if (retry.staleFormScope)
|
|
4188
|
+
return retry;
|
|
3670
4189
|
}
|
|
3671
4190
|
if (options.fullyVisible !== false) {
|
|
3672
4191
|
attempts += 1;
|
|
@@ -3683,6 +4202,8 @@ async function resolveClickLocationWithFallback(session, options) {
|
|
|
3683
4202
|
fallback: { attempted: true, used: true, reason: 'relaxed-visibility', attempts },
|
|
3684
4203
|
};
|
|
3685
4204
|
}
|
|
4205
|
+
if (relaxed.staleFormScope)
|
|
4206
|
+
return relaxed;
|
|
3686
4207
|
}
|
|
3687
4208
|
// All fallback phases tried and none recovered. Carry the trace of what we
|
|
3688
4209
|
// tried so operators see the attempted-but-failed signal alongside the
|
|
@@ -3729,11 +4250,19 @@ function compactFormattedNode(node) {
|
|
|
3729
4250
|
function detailText(summary, compact, detail) {
|
|
3730
4251
|
return detail === 'terse' ? JSON.stringify(compact) : summary;
|
|
3731
4252
|
}
|
|
4253
|
+
function ambiguousActionGuidance(wait) {
|
|
4254
|
+
return `Outcome is ambiguous for actionId ${wait.actionId} (requestId ${wait.requestId}). ` +
|
|
4255
|
+
'Do not retry blindly or skip this step; inspect the current UI first. ' +
|
|
4256
|
+
'An identical retry reuses the same action identity when the proxy supports deduplication.';
|
|
4257
|
+
}
|
|
3732
4258
|
function actionWaitResponse(wait, summary, compact, detail) {
|
|
3733
4259
|
if (wait.status === 'timed_out') {
|
|
3734
|
-
|
|
4260
|
+
const guidance = ambiguousActionGuidance(wait);
|
|
4261
|
+
return err(detailText(`${summary}\n${guidance}`, {
|
|
3735
4262
|
completed: false,
|
|
3736
4263
|
outcome: 'unconfirmed',
|
|
4264
|
+
retrySafety: 'inspect-first',
|
|
4265
|
+
guidance,
|
|
3737
4266
|
...compact,
|
|
3738
4267
|
}, detail));
|
|
3739
4268
|
}
|
|
@@ -3908,6 +4437,25 @@ function pad2(n) {
|
|
|
3908
4437
|
return n < 10 ? `0${n}` : String(n);
|
|
3909
4438
|
}
|
|
3910
4439
|
function plannedFillInputsForField(field, value) {
|
|
4440
|
+
if (field.kind === 'file') {
|
|
4441
|
+
if (!Array.isArray(value)) {
|
|
4442
|
+
return { error: `Field "${field.label}" expects a non-empty string array of proxy-host file paths` };
|
|
4443
|
+
}
|
|
4444
|
+
const paths = value.map(path => path.trim()).filter(Boolean);
|
|
4445
|
+
if (paths.length === 0 || paths.length !== value.length) {
|
|
4446
|
+
return { error: `Field "${field.label}" expects a non-empty string array of proxy-host file paths` };
|
|
4447
|
+
}
|
|
4448
|
+
if (field.format?.multiple !== true && paths.length > 1) {
|
|
4449
|
+
return { error: `Field "${field.label}" accepts only one file` };
|
|
4450
|
+
}
|
|
4451
|
+
return [{
|
|
4452
|
+
kind: 'file',
|
|
4453
|
+
fieldId: field.id,
|
|
4454
|
+
...(field.fieldKey ? { fieldKey: field.fieldKey } : {}),
|
|
4455
|
+
fieldLabel: field.label,
|
|
4456
|
+
paths,
|
|
4457
|
+
}];
|
|
4458
|
+
}
|
|
3911
4459
|
if (field.kind === 'text') {
|
|
3912
4460
|
if (typeof value !== 'string')
|
|
3913
4461
|
return { error: `Field "${field.label}" expects a string value` };
|
|
@@ -4045,13 +4593,12 @@ function isResolvedFillFieldInput(field) {
|
|
|
4045
4593
|
}
|
|
4046
4594
|
function resolveFillFieldInputs(session, fields) {
|
|
4047
4595
|
const unresolved = fields.filter(field => !isResolvedFillFieldInput(field));
|
|
4596
|
+
const requiresSchemaLookup = fields.some(field => typeof field.fieldId === 'string');
|
|
4048
4597
|
const a11y = sessionA11y(session);
|
|
4049
|
-
if (unresolved.length === 0 && !a11y) {
|
|
4598
|
+
if (unresolved.length === 0 && !requiresSchemaLookup && !a11y) {
|
|
4050
4599
|
return { ok: true, fields: fields };
|
|
4051
4600
|
}
|
|
4052
4601
|
if (!a11y) {
|
|
4053
|
-
if (unresolved.length === 0)
|
|
4054
|
-
return { ok: true, fields: fields };
|
|
4055
4602
|
return { ok: false, error: 'No UI tree available to resolve fieldId entries from geometra_form_schema' };
|
|
4056
4603
|
}
|
|
4057
4604
|
const schemas = buildFormSchemas(a11y, { includeOptions: true, includeContext: 'always' });
|
|
@@ -4068,10 +4615,16 @@ function resolveFillFieldInputs(session, fields) {
|
|
|
4068
4615
|
return { ok: false, error: `Unknown form field id ${field.fieldId}. Refresh geometra_form_schema and try again.` };
|
|
4069
4616
|
}
|
|
4070
4617
|
if (schemaField) {
|
|
4071
|
-
|
|
4072
|
-
if (field.kind !== 'file' && schemaField.kind !== expectedKind) {
|
|
4618
|
+
if (schemaField.kind !== field.kind) {
|
|
4073
4619
|
return { ok: false, error: `Field id ${field.fieldId} resolves to kind "${schemaField.kind}", not ${field.kind}.` };
|
|
4074
4620
|
}
|
|
4621
|
+
const suppliedLabel = field.kind === 'toggle' ? field.label : field.fieldLabel;
|
|
4622
|
+
if (normalizeLookupKey(suppliedLabel) !== normalizeLookupKey(schemaField.label)) {
|
|
4623
|
+
return {
|
|
4624
|
+
ok: false,
|
|
4625
|
+
error: `Field id ${field.fieldId} now resolves to "${schemaField.label}", not "${suppliedLabel}". Refresh geometra_form_schema; Geometra refused to downgrade the stale id to a label-only mutation.`,
|
|
4626
|
+
};
|
|
4627
|
+
}
|
|
4075
4628
|
}
|
|
4076
4629
|
const suppliedLabel = field.kind === 'toggle' ? field.label : field.fieldLabel;
|
|
4077
4630
|
const distinctLabelMatches = new Map();
|
|
@@ -4179,14 +4732,20 @@ function resolveFillFieldInputs(session, fields) {
|
|
|
4179
4732
|
});
|
|
4180
4733
|
continue;
|
|
4181
4734
|
}
|
|
4182
|
-
|
|
4183
|
-
ok: false,
|
|
4184
|
-
|
|
4185
|
-
|
|
4735
|
+
if (schemaField.kind !== 'file') {
|
|
4736
|
+
return { ok: false, error: `Field id ${field.fieldId} resolves to kind "${schemaField.kind}", not file.` };
|
|
4737
|
+
}
|
|
4738
|
+
resolved.push({
|
|
4739
|
+
...field,
|
|
4740
|
+
...(schemaField.fieldKey ? { fieldKey: schemaField.fieldKey } : {}),
|
|
4741
|
+
fieldLabel: schemaField.label,
|
|
4742
|
+
});
|
|
4186
4743
|
}
|
|
4187
4744
|
return { ok: true, fields: resolved };
|
|
4188
4745
|
}
|
|
4189
4746
|
function canFallbackToSequentialFill(error) {
|
|
4747
|
+
if (ambiguousOutcomeDetails(error))
|
|
4748
|
+
return false;
|
|
4190
4749
|
const message = error instanceof Error ? error.message : String(error);
|
|
4191
4750
|
try {
|
|
4192
4751
|
const parsed = JSON.parse(message);
|
|
@@ -4236,10 +4795,20 @@ async function tryBatchedResolvedFields(session, fields, detail) {
|
|
|
4236
4795
|
try {
|
|
4237
4796
|
const startRevision = session.updateRevision;
|
|
4238
4797
|
const wait = await sendFillFields(session, toProxyFillFields(fields));
|
|
4239
|
-
|
|
4240
|
-
return { ok: false };
|
|
4798
|
+
assertActionWaitConfirmed('Batched field fill', wait);
|
|
4241
4799
|
const ackResult = parseProxyFillAckResult(wait.result);
|
|
4242
4800
|
batchAckResult = ackResult;
|
|
4801
|
+
if (fields.some(field => field.kind === 'file')) {
|
|
4802
|
+
if (!ackResult) {
|
|
4803
|
+
throw new AmbiguousActionOutcomeError('Batched file fill', wait, 'Batched file fill was acknowledged without structured proxy confirmation; the upload may already have happened. Do not retry blindly.');
|
|
4804
|
+
}
|
|
4805
|
+
return {
|
|
4806
|
+
ok: true,
|
|
4807
|
+
finalSource: 'proxy',
|
|
4808
|
+
final: ackResult,
|
|
4809
|
+
invalidRemaining: ackResult.invalidCount,
|
|
4810
|
+
};
|
|
4811
|
+
}
|
|
4243
4812
|
if (ackResult && ackResult.invalidCount === 0) {
|
|
4244
4813
|
return {
|
|
4245
4814
|
ok: true,
|
|
@@ -4249,27 +4818,31 @@ async function tryBatchedResolvedFields(session, fields, detail) {
|
|
|
4249
4818
|
};
|
|
4250
4819
|
}
|
|
4251
4820
|
await waitForDeferredBatchUpdate(session, startRevision, wait);
|
|
4252
|
-
await waitForBatchFieldReadback(session, fields);
|
|
4821
|
+
const readbackSnapshot = await waitForBatchFieldReadback(session, fields);
|
|
4822
|
+
if (!readbackSnapshot) {
|
|
4823
|
+
if (!sessionA11y(session)) {
|
|
4824
|
+
throw new AmbiguousActionOutcomeError('Batched field fill', wait, 'Batched field fill was sent, but its read-back evidence became unavailable before confirmation. The fields may already have changed. Do not retry blindly.');
|
|
4825
|
+
}
|
|
4826
|
+
return { ok: false };
|
|
4827
|
+
}
|
|
4828
|
+
const signals = collectSessionSignals(readbackSnapshot);
|
|
4829
|
+
const invalidRemaining = signals.invalidFields.length;
|
|
4830
|
+
if ((!batchAckResult || batchAckResult.invalidCount > 0) && invalidRemaining > 0) {
|
|
4831
|
+
return { ok: false };
|
|
4832
|
+
}
|
|
4833
|
+
return {
|
|
4834
|
+
ok: true,
|
|
4835
|
+
finalSource: 'session',
|
|
4836
|
+
final: sessionSignalsPayload(signals, detail),
|
|
4837
|
+
invalidRemaining,
|
|
4838
|
+
readbackSnapshot,
|
|
4839
|
+
};
|
|
4253
4840
|
}
|
|
4254
4841
|
catch (e) {
|
|
4255
4842
|
if (canFallbackToSequentialFill(e))
|
|
4256
4843
|
return { ok: false };
|
|
4257
4844
|
throw e;
|
|
4258
4845
|
}
|
|
4259
|
-
const after = sessionA11y(session);
|
|
4260
|
-
if (!after)
|
|
4261
|
-
return { ok: false };
|
|
4262
|
-
const signals = collectSessionSignals(after);
|
|
4263
|
-
const invalidRemaining = signals.invalidFields.length;
|
|
4264
|
-
if ((!batchAckResult || batchAckResult.invalidCount > 0) && invalidRemaining > 0) {
|
|
4265
|
-
return { ok: false };
|
|
4266
|
-
}
|
|
4267
|
-
return {
|
|
4268
|
-
ok: true,
|
|
4269
|
-
finalSource: 'session',
|
|
4270
|
-
final: sessionSignalsPayload(signals, detail),
|
|
4271
|
-
invalidRemaining,
|
|
4272
|
-
};
|
|
4273
4846
|
}
|
|
4274
4847
|
async function waitForDeferredBatchUpdate(session, startRevision, wait) {
|
|
4275
4848
|
if (wait.status !== 'acknowledged' || session.updateRevision > startRevision)
|
|
@@ -4277,12 +4850,17 @@ async function waitForDeferredBatchUpdate(session, startRevision, wait) {
|
|
|
4277
4850
|
await waitForUiCondition(session, () => session.updateRevision > startRevision, 750);
|
|
4278
4851
|
}
|
|
4279
4852
|
async function waitForBatchFieldReadback(session, fields) {
|
|
4853
|
+
let matchingSnapshot;
|
|
4280
4854
|
await waitForUiCondition(session, () => {
|
|
4281
4855
|
const a11y = sessionA11y(session);
|
|
4282
4856
|
if (!a11y)
|
|
4283
4857
|
return false;
|
|
4284
|
-
|
|
4858
|
+
if (!fields.every(field => batchFieldReadbackMatches(a11y, field)))
|
|
4859
|
+
return false;
|
|
4860
|
+
matchingSnapshot = a11y;
|
|
4861
|
+
return true;
|
|
4285
4862
|
}, 1500);
|
|
4863
|
+
return matchingSnapshot;
|
|
4286
4864
|
}
|
|
4287
4865
|
function batchFieldReadbackMatches(a11y, field) {
|
|
4288
4866
|
switch (field.kind) {
|
|
@@ -4337,9 +4915,50 @@ function canDeferInitialFrameForRunActions(actions) {
|
|
|
4337
4915
|
function assertBatchActionConfirmed(actionType, wait) {
|
|
4338
4916
|
assertActionWaitConfirmed(`${actionType} action`, wait);
|
|
4339
4917
|
}
|
|
4918
|
+
class AmbiguousActionOutcomeError extends Error {
|
|
4919
|
+
wait;
|
|
4920
|
+
constructor(actionName, wait, message) {
|
|
4921
|
+
super(message ?? `${actionName} timed out after ${wait.timeoutMs}ms; its outcome is unconfirmed. ${ambiguousActionGuidance(wait)}`);
|
|
4922
|
+
this.name = 'AmbiguousActionOutcomeError';
|
|
4923
|
+
this.wait = wait;
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
function ambiguousOutcomeDetails(error) {
|
|
4927
|
+
if (error instanceof AmbiguousActionOutcomeError) {
|
|
4928
|
+
return {
|
|
4929
|
+
wait: error.wait,
|
|
4930
|
+
requestId: error.wait.requestId,
|
|
4931
|
+
actionId: error.wait.actionId,
|
|
4932
|
+
};
|
|
4933
|
+
}
|
|
4934
|
+
if (!error || typeof error !== 'object')
|
|
4935
|
+
return undefined;
|
|
4936
|
+
const candidate = error;
|
|
4937
|
+
const code = typeof candidate.code === 'string' ? candidate.code : undefined;
|
|
4938
|
+
const requestId = typeof candidate.requestId === 'string' ? candidate.requestId : undefined;
|
|
4939
|
+
const actionId = typeof candidate.actionId === 'string' ? candidate.actionId : undefined;
|
|
4940
|
+
const message = typeof candidate.message === 'string' ? candidate.message : '';
|
|
4941
|
+
if (code !== 'ACTION_OUTCOME_AMBIGUOUS' &&
|
|
4942
|
+
!(requestId && actionId && message.includes('Outcome is ambiguous')))
|
|
4943
|
+
return undefined;
|
|
4944
|
+
return { ...(requestId ? { requestId } : {}), ...(actionId ? { actionId } : {}), ...(code ? { code } : {}) };
|
|
4945
|
+
}
|
|
4946
|
+
function ambiguousOutcomePayload(details) {
|
|
4947
|
+
return {
|
|
4948
|
+
outcome: 'unconfirmed',
|
|
4949
|
+
retrySafety: 'inspect-first',
|
|
4950
|
+
...(details.wait ? waitStatusPayload(details.wait) : {}),
|
|
4951
|
+
...(!details.wait && details.requestId ? { requestId: details.requestId } : {}),
|
|
4952
|
+
...(!details.wait && details.actionId ? { actionId: details.actionId } : {}),
|
|
4953
|
+
...(details.code ? { code: details.code } : {}),
|
|
4954
|
+
guidance: details.actionId && details.requestId
|
|
4955
|
+
? `Outcome is ambiguous for actionId ${details.actionId} (requestId ${details.requestId}). Do not retry blindly or skip this step; inspect the current UI first.`
|
|
4956
|
+
: 'The action outcome is ambiguous. Do not retry blindly or skip this step; inspect the current UI first.',
|
|
4957
|
+
};
|
|
4958
|
+
}
|
|
4340
4959
|
function assertActionWaitConfirmed(actionName, wait) {
|
|
4341
4960
|
if (wait.status === 'timed_out') {
|
|
4342
|
-
throw new
|
|
4961
|
+
throw new AmbiguousActionOutcomeError(actionName, wait);
|
|
4343
4962
|
}
|
|
4344
4963
|
}
|
|
4345
4964
|
async function executeBatchAction(session, action, detail, includeSteps) {
|
|
@@ -4457,6 +5076,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
4457
5076
|
};
|
|
4458
5077
|
}
|
|
4459
5078
|
case 'upload_files': {
|
|
5079
|
+
const contractError = fileUploadContractError(action);
|
|
5080
|
+
if (contractError)
|
|
5081
|
+
throw new Error(contractError);
|
|
4460
5082
|
const before = sessionA11y(session);
|
|
4461
5083
|
const wait = await sendFileUpload(session, action.paths, {
|
|
4462
5084
|
click: action.x !== undefined && action.y !== undefined ? { x: action.x, y: action.y } : undefined,
|
|
@@ -4464,6 +5086,8 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
4464
5086
|
exact: action.exact,
|
|
4465
5087
|
strategy: action.strategy,
|
|
4466
5088
|
drop: action.dropX !== undefined && action.dropY !== undefined ? { x: action.dropX, y: action.dropY } : undefined,
|
|
5089
|
+
contextText: action.contextText,
|
|
5090
|
+
sectionText: action.sectionText,
|
|
4467
5091
|
}, action.timeoutMs ?? 8_000);
|
|
4468
5092
|
assertBatchActionConfirmed(action.type, wait);
|
|
4469
5093
|
return {
|
|
@@ -4472,16 +5096,20 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
4472
5096
|
fileCount: action.paths.length,
|
|
4473
5097
|
...(action.fieldLabel ? { fieldLabel: action.fieldLabel } : {}),
|
|
4474
5098
|
...(action.strategy ? { strategy: action.strategy } : {}),
|
|
5099
|
+
...(action.contextText ? { contextText: action.contextText } : {}),
|
|
5100
|
+
...(action.sectionText ? { sectionText: action.sectionText } : {}),
|
|
4475
5101
|
...waitStatusPayload(wait),
|
|
4476
5102
|
...(action.fieldLabel ? { readback: fieldStatePayload(session, action.fieldLabel) } : {}),
|
|
4477
5103
|
},
|
|
4478
5104
|
};
|
|
4479
5105
|
}
|
|
4480
5106
|
case 'pick_listbox_option': {
|
|
5107
|
+
const contractError = listboxPickContractError(action);
|
|
5108
|
+
if (contractError)
|
|
5109
|
+
throw new Error(contractError);
|
|
4481
5110
|
const before = sessionA11y(session);
|
|
4482
5111
|
const wait = await sendListboxPick(session, action.label, {
|
|
4483
5112
|
exact: action.exact,
|
|
4484
|
-
open: action.openX !== undefined && action.openY !== undefined ? { x: action.openX, y: action.openY } : undefined,
|
|
4485
5113
|
fieldId: action.fieldId,
|
|
4486
5114
|
fieldKey: action.fieldKey,
|
|
4487
5115
|
fieldLabel: action.fieldLabel,
|
|
@@ -4505,9 +5133,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
4505
5133
|
};
|
|
4506
5134
|
}
|
|
4507
5135
|
case 'select_option': {
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
5136
|
+
const contractError = selectOptionContractError(action);
|
|
5137
|
+
if (contractError)
|
|
5138
|
+
throw new Error(contractError);
|
|
4511
5139
|
const before = sessionA11y(session);
|
|
4512
5140
|
const wait = await sendSelectOption(session, action.x, action.y, {
|
|
4513
5141
|
value: action.value,
|
|
@@ -4647,13 +5275,13 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
4647
5275
|
if (!resolvedFields.ok)
|
|
4648
5276
|
throw new Error(resolvedFields.error);
|
|
4649
5277
|
const verifyFillsFn = action.verifyFills
|
|
4650
|
-
? () => verifyFormFills(session, resolvedFields.fields.map(field => ({ field, confidence: 1.0, matchMethod: 'label-exact' })))
|
|
5278
|
+
? (readbackSnapshot) => verifyFormFills(session, resolvedFields.fields.map(field => ({ field, confidence: 1.0, matchMethod: 'label-exact' })), readbackSnapshot)
|
|
4651
5279
|
: undefined;
|
|
4652
5280
|
let fallbackFromBatch;
|
|
4653
5281
|
if (!includeSteps) {
|
|
4654
5282
|
const batched = await tryBatchedResolvedFields(session, resolvedFields.fields, detail);
|
|
4655
5283
|
if (batched.ok) {
|
|
4656
|
-
const verification = verifyFillsFn?.();
|
|
5284
|
+
const verification = verifyFillsFn?.(batched.readbackSnapshot);
|
|
4657
5285
|
return {
|
|
4658
5286
|
summary: `Filled ${resolvedFields.fields.length} field(s) in one proxy batch.`,
|
|
4659
5287
|
compact: {
|
|
@@ -4736,7 +5364,7 @@ function suggestRecovery(field, error) {
|
|
|
4736
5364
|
}
|
|
4737
5365
|
}
|
|
4738
5366
|
if (lowerError.includes('timeout')) {
|
|
4739
|
-
return
|
|
5367
|
+
return 'Action timed out and may still complete. Do not retry blindly; inspect the current UI or wait for a loading indicator before repeating the identical action.';
|
|
4740
5368
|
}
|
|
4741
5369
|
return undefined;
|
|
4742
5370
|
}
|
|
@@ -4816,8 +5444,8 @@ export function valuesEquivalent(expected, actual) {
|
|
|
4816
5444
|
const aNorm = actual.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
4817
5445
|
return eNorm === aNorm;
|
|
4818
5446
|
}
|
|
4819
|
-
function verifyFormFills(session, planned) {
|
|
4820
|
-
const a11y = sessionA11y(session);
|
|
5447
|
+
function verifyFormFills(session, planned, readbackSnapshot) {
|
|
5448
|
+
const a11y = readbackSnapshot ?? sessionA11y(session);
|
|
4821
5449
|
if (!a11y)
|
|
4822
5450
|
return { verified: 0, mismatches: [] };
|
|
4823
5451
|
const mismatches = [];
|
|
@@ -4980,27 +5608,29 @@ function recordWorkflowFill(session, formId, formName, valuesById, valuesByLabel
|
|
|
4980
5608
|
if (!session.workflowState) {
|
|
4981
5609
|
session.workflowState = { pages: [], startedAt: Date.now() };
|
|
4982
5610
|
}
|
|
4983
|
-
const
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
if (typeof v === 'string' || typeof v === 'boolean')
|
|
4990
|
-
filledValues[k] = v;
|
|
4991
|
-
}
|
|
5611
|
+
const fieldIdentities = new Set([
|
|
5612
|
+
...Object.keys(valuesById ?? {}),
|
|
5613
|
+
...Object.keys(valuesByLabel ?? {}),
|
|
5614
|
+
]);
|
|
5615
|
+
const filledFields = Array.from(fieldIdentities).sort();
|
|
5616
|
+
const filledValues = Object.fromEntries(filledFields.map(identity => [identity, '[REDACTED]']));
|
|
4992
5617
|
const a11y = sessionA11y(session);
|
|
4993
|
-
const pageUrl = a11y?.meta?.pageUrl ?? session.url;
|
|
5618
|
+
const pageUrl = workflowUrlOrigin(a11y?.meta?.pageUrl ?? session.url);
|
|
4994
5619
|
session.workflowState.pages.push({
|
|
4995
5620
|
pageUrl,
|
|
4996
5621
|
formId,
|
|
4997
5622
|
formName,
|
|
4998
5623
|
filledValues,
|
|
5624
|
+
filledFields,
|
|
5625
|
+
valuesRedacted: true,
|
|
4999
5626
|
filledAt: Date.now(),
|
|
5000
5627
|
fieldCount,
|
|
5001
5628
|
invalidCount,
|
|
5002
5629
|
});
|
|
5003
5630
|
}
|
|
5631
|
+
function workflowUrlOrigin(rawUrl) {
|
|
5632
|
+
return sanitizeUrlToOrigin(rawUrl) ?? REDACTED_STATE_URL;
|
|
5633
|
+
}
|
|
5004
5634
|
async function captureScreenshotBase64(session) {
|
|
5005
5635
|
try {
|
|
5006
5636
|
const wait = await sendScreenshot(session);
|