@geometra/mcp 1.19.22 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -15
- package/dist/__tests__/proxy-session-recovery.test.js +91 -2
- package/dist/__tests__/server-batch-results.test.js +262 -2
- package/dist/__tests__/session-model.test.js +48 -0
- package/dist/proxy-spawn.d.ts +3 -0
- package/dist/proxy-spawn.js +3 -0
- package/dist/server.d.ts +3 -0
- package/dist/server.js +596 -80
- package/dist/session.d.ts +45 -0
- package/dist/session.js +372 -32
- package/package.json +2 -2
package/dist/server.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { performance } from 'node:perf_hooks';
|
|
2
3
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
4
|
import { z } from 'zod';
|
|
4
5
|
import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
|
|
5
|
-
import { connect, connectThroughProxy, disconnect, getSession, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
|
|
6
|
+
import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, sendScreenshot, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
|
|
6
7
|
function checkedStateInput() {
|
|
7
8
|
return z
|
|
8
9
|
.union([z.boolean(), z.literal('mixed')])
|
|
@@ -11,10 +12,10 @@ function checkedStateInput() {
|
|
|
11
12
|
}
|
|
12
13
|
function detailInput() {
|
|
13
14
|
return z
|
|
14
|
-
.enum(['minimal', 'verbose'])
|
|
15
|
+
.enum(['terse', 'minimal', 'verbose'])
|
|
15
16
|
.optional()
|
|
16
17
|
.default('minimal')
|
|
17
|
-
.describe('`minimal` (default) returns
|
|
18
|
+
.describe('`terse` returns compact machine-friendly JSON. `minimal` (default) returns short human-readable summaries. `verbose` adds fuller fallback context.');
|
|
18
19
|
}
|
|
19
20
|
function formSchemaFormatInput() {
|
|
20
21
|
return z
|
|
@@ -23,6 +24,13 @@ function formSchemaFormatInput() {
|
|
|
23
24
|
.default('compact')
|
|
24
25
|
.describe('`compact` (default) returns readable JSON fields. Use `packed` for the smallest schema payload with short keys.');
|
|
25
26
|
}
|
|
27
|
+
function pageModelModeInput() {
|
|
28
|
+
return z
|
|
29
|
+
.enum(['inline', 'deferred'])
|
|
30
|
+
.optional()
|
|
31
|
+
.default('inline')
|
|
32
|
+
.describe('When returnPageModel=true, `inline` includes the full page model in the connect response. `deferred` returns connect as soon as the transport is ready and lets the caller fetch geometra_page_model separately.');
|
|
33
|
+
}
|
|
26
34
|
function formSchemaContextInput() {
|
|
27
35
|
return z
|
|
28
36
|
.enum(['auto', 'always', 'none'])
|
|
@@ -37,6 +45,9 @@ function nodeFilterShape() {
|
|
|
37
45
|
name: z.string().optional().describe('Accessible name to match (exact or substring)'),
|
|
38
46
|
text: z.string().optional().describe('Text content to search for (substring match)'),
|
|
39
47
|
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated controls with the same visible name'),
|
|
48
|
+
promptText: z.string().optional().describe('Nearby question/prompt text to disambiguate repeated controls or actions'),
|
|
49
|
+
sectionText: z.string().optional().describe('Containing section/landmark/form/dialog text to disambiguate repeated controls or actions'),
|
|
50
|
+
itemText: z.string().optional().describe('Nearby card/row/item label to disambiguate repeated actions like “Add to cart” or “Open incident”'),
|
|
40
51
|
value: z.string().optional().describe('Displayed / current field value to match (substring match)'),
|
|
41
52
|
checked: checkedStateInput(),
|
|
42
53
|
disabled: z.boolean().optional().describe('Match disabled state'),
|
|
@@ -66,15 +77,22 @@ function waitConditionShape() {
|
|
|
66
77
|
.describe('Maximum time to wait before returning an error (default 10000ms)'),
|
|
67
78
|
};
|
|
68
79
|
}
|
|
69
|
-
const GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE = 'Provide at least one filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
|
|
80
|
+
const GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE = 'Provide at least one filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
|
|
70
81
|
'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text for substring matching. ' +
|
|
71
82
|
'To wait until text disappears from the UI, use geometra_wait_for with text and present: false, or geometra_wait_for_resume_parse for typical resume “Parsing…” banners.';
|
|
72
|
-
const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic filter (id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
|
|
83
|
+
const GEOMETRA_WAIT_FILTER_REQUIRED_MESSAGE = 'Provide at least one semantic filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, or busy). ' +
|
|
73
84
|
'This tool uses a strict schema: unknown keys are rejected. There is no textGone parameter — use text with a distinctive substring and present: false to wait until that text is gone ' +
|
|
74
85
|
'(common for “Parsing…”, “Parsing your resume”, or similar). Passing only present/timeoutMs is not enough without a filter.';
|
|
75
86
|
/** Strict input so unknown keys (e.g. textGone) fail parse; empty-filter checks happen in handlers / waitForSemanticCondition. */
|
|
76
|
-
const geometraQueryInputSchema = z.object(
|
|
77
|
-
|
|
87
|
+
const geometraQueryInputSchema = z.object({
|
|
88
|
+
...nodeFilterShape(),
|
|
89
|
+
maxResults: z.number().int().min(1).max(50).optional().describe('Optional cap on returned matches; terse mode defaults to 8'),
|
|
90
|
+
detail: detailInput(),
|
|
91
|
+
}).strict();
|
|
92
|
+
const geometraWaitForInputSchema = z.object({
|
|
93
|
+
...waitConditionShape(),
|
|
94
|
+
detail: detailInput(),
|
|
95
|
+
}).strict();
|
|
78
96
|
/** Same upper bound as geometra_wait_for; resume uploads often need the full minute. */
|
|
79
97
|
const geometraWaitForResumeParseInputSchema = z
|
|
80
98
|
.object({
|
|
@@ -271,7 +289,7 @@ export function createServer() {
|
|
|
271
289
|
|
|
272
290
|
Use \`url\` (ws://…) only when a Geometra/native server or an already-running proxy is listening. If you accidentally pass \`https://…\` in \`url\`, MCP treats it like \`pageUrl\` and starts the proxy for you.
|
|
273
291
|
|
|
274
|
-
Chromium opens **visible** by default unless \`headless: true\`. File upload / wheel / native \`<select>\` need the proxy path (\`pageUrl\` or ws to proxy). Set \`returnForms: true\` and/or \`returnPageModel: true\` when you want a lower-turn startup response.`, {
|
|
292
|
+
Chromium opens **visible** by default unless \`headless: true\`. File upload / wheel / native \`<select>\` need the proxy path (\`pageUrl\` or ws to proxy). Set \`returnForms: true\` and/or \`returnPageModel: true\` when you want a lower-turn startup response. When connect first-response latency matters more than inlining the page model, pair \`returnPageModel: true\` with \`pageModelMode: "deferred"\` and call \`geometra_page_model\` next.`, {
|
|
275
293
|
url: z
|
|
276
294
|
.string()
|
|
277
295
|
.optional()
|
|
@@ -311,6 +329,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
311
329
|
.optional()
|
|
312
330
|
.default(false)
|
|
313
331
|
.describe('Include geometra_page_model output in the connect response so exploration can start in one turn.'),
|
|
332
|
+
pageModelMode: pageModelModeInput(),
|
|
314
333
|
formId: z.string().optional().describe('Optional form id filter when returnForms=true'),
|
|
315
334
|
maxFields: z.number().int().min(1).max(120).optional().default(80).describe('Cap returned fields per form when returnForms=true'),
|
|
316
335
|
onlyRequiredFields: z.boolean().optional().default(false).describe('Only include required fields when returnForms=true'),
|
|
@@ -341,6 +360,10 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
341
360
|
maxPrimaryActions: input.maxPrimaryActions,
|
|
342
361
|
maxSectionsPerKind: input.maxSectionsPerKind,
|
|
343
362
|
};
|
|
363
|
+
const deferInlinePageModel = input.returnPageModel
|
|
364
|
+
&& input.pageModelMode === 'deferred'
|
|
365
|
+
&& !input.returnForms
|
|
366
|
+
&& input.detail !== 'verbose';
|
|
344
367
|
try {
|
|
345
368
|
if (target.kind === 'proxy') {
|
|
346
369
|
const session = await connectThroughProxy({
|
|
@@ -350,6 +373,8 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
350
373
|
width: input.width,
|
|
351
374
|
height: input.height,
|
|
352
375
|
slowMo: input.slowMo,
|
|
376
|
+
awaitInitialFrame: deferInlinePageModel ? false : undefined,
|
|
377
|
+
eagerInitialExtract: deferInlinePageModel ? true : undefined,
|
|
353
378
|
});
|
|
354
379
|
if (input.returnForms) {
|
|
355
380
|
await stabilizeInlineFormSchemas(session, formSchema);
|
|
@@ -361,6 +386,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
361
386
|
detail: input.detail,
|
|
362
387
|
returnForms: input.returnForms,
|
|
363
388
|
returnPageModel: input.returnPageModel,
|
|
389
|
+
pageModelMode: input.pageModelMode,
|
|
364
390
|
formSchema,
|
|
365
391
|
pageModelOptions,
|
|
366
392
|
}), null, input.detail === 'verbose' ? 2 : undefined));
|
|
@@ -368,6 +394,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
368
394
|
const session = await connect(target.wsUrl, {
|
|
369
395
|
width: input.width,
|
|
370
396
|
height: input.height,
|
|
397
|
+
awaitInitialFrame: deferInlinePageModel ? false : undefined,
|
|
371
398
|
});
|
|
372
399
|
if (input.returnForms) {
|
|
373
400
|
await stabilizeInlineFormSchemas(session, formSchema);
|
|
@@ -379,6 +406,7 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
379
406
|
detail: input.detail,
|
|
380
407
|
returnForms: input.returnForms,
|
|
381
408
|
returnPageModel: input.returnPageModel,
|
|
409
|
+
pageModelMode: input.pageModelMode,
|
|
382
410
|
formSchema,
|
|
383
411
|
pageModelOptions,
|
|
384
412
|
}), null, input.detail === 'verbose' ? 2 : undefined));
|
|
@@ -387,6 +415,43 @@ Chromium opens **visible** by default unless \`headless: true\`. File upload / w
|
|
|
387
415
|
return err(`Failed to connect: ${formatConnectFailureMessage(e, target)}`);
|
|
388
416
|
}
|
|
389
417
|
});
|
|
418
|
+
// ── prepare browser ──────────────────────────────────────────
|
|
419
|
+
server.tool('geometra_prepare_browser', `Pre-launch and pre-navigate a reusable geometra-proxy browser for a normal web page without creating an active MCP session.
|
|
420
|
+
|
|
421
|
+
Use this when you can prepare ahead of the user-facing task so the next \`geometra_connect\` or one-call \`geometra_run_actions\` on the same \`pageUrl\` / viewport / headless settings skips the cold browser launch.`, {
|
|
422
|
+
pageUrl: z
|
|
423
|
+
.string()
|
|
424
|
+
.url()
|
|
425
|
+
.refine(isHttpUrl, 'pageUrl must use http:// or https://')
|
|
426
|
+
.describe('HTTP(S) page to open and keep warm for the next proxy-backed task.'),
|
|
427
|
+
port: z
|
|
428
|
+
.number()
|
|
429
|
+
.int()
|
|
430
|
+
.positive()
|
|
431
|
+
.max(65535)
|
|
432
|
+
.optional()
|
|
433
|
+
.describe('Preferred local port for spawned proxy (default: ephemeral OS-assigned port).'),
|
|
434
|
+
headless: z
|
|
435
|
+
.boolean()
|
|
436
|
+
.optional()
|
|
437
|
+
.describe('Run Chromium headless (default false = visible window).'),
|
|
438
|
+
width: z.number().int().positive().optional().describe('Viewport width for the warmed browser.'),
|
|
439
|
+
height: z.number().int().positive().optional().describe('Viewport height for the warmed browser.'),
|
|
440
|
+
slowMo: z
|
|
441
|
+
.number()
|
|
442
|
+
.int()
|
|
443
|
+
.nonnegative()
|
|
444
|
+
.optional()
|
|
445
|
+
.describe('Playwright slowMo (ms) for the warmed browser.'),
|
|
446
|
+
}, async ({ pageUrl, port, headless, width, height, slowMo }) => {
|
|
447
|
+
try {
|
|
448
|
+
const prepared = await prewarmProxy({ pageUrl, port, headless, width, height, slowMo });
|
|
449
|
+
return ok(JSON.stringify(prepared));
|
|
450
|
+
}
|
|
451
|
+
catch (e) {
|
|
452
|
+
return err(`Failed to prepare browser: ${e instanceof Error ? e.message : String(e)}`);
|
|
453
|
+
}
|
|
454
|
+
});
|
|
390
455
|
// ── query ────────────────────────────────────────────────────
|
|
391
456
|
server.registerTool('geometra_query', {
|
|
392
457
|
description: `Find elements in the current Geometra UI by stable id, role, name, text content, current value, or semantic state. Returns matching elements with their exact pixel bounds {x, y, width, height}, visible in-viewport bounds, an on-screen center point, visibility / scroll-reveal hints, role, name, value, state, and tree path.
|
|
@@ -395,11 +460,11 @@ This is the Geometra equivalent of Playwright's locator — but instant, structu
|
|
|
395
460
|
|
|
396
461
|
Unknown parameter names are rejected (strict schema). To wait until visible text goes away (e.g. a parsing banner), use geometra_wait_for with that substring in text and present: false — there is no textGone field.`,
|
|
397
462
|
inputSchema: geometraQueryInputSchema,
|
|
398
|
-
}, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy }) => {
|
|
463
|
+
}, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, maxResults, detail }) => {
|
|
399
464
|
const session = getSession();
|
|
400
|
-
if (!session
|
|
465
|
+
if (!session)
|
|
401
466
|
return err('Not connected. Call geometra_connect first.');
|
|
402
|
-
const a11y =
|
|
467
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
403
468
|
if (!a11y)
|
|
404
469
|
return err('No UI tree available');
|
|
405
470
|
const filter = {
|
|
@@ -408,6 +473,9 @@ Unknown parameter names are rejected (strict schema). To wait until visible text
|
|
|
408
473
|
name,
|
|
409
474
|
text,
|
|
410
475
|
contextText,
|
|
476
|
+
promptText,
|
|
477
|
+
sectionText,
|
|
478
|
+
itemText,
|
|
411
479
|
value,
|
|
412
480
|
checked,
|
|
413
481
|
disabled,
|
|
@@ -422,19 +490,70 @@ Unknown parameter names are rejected (strict schema). To wait until visible text
|
|
|
422
490
|
return err(GEOMETRA_QUERY_FILTER_REQUIRED_MESSAGE);
|
|
423
491
|
const matches = findNodes(a11y, filter);
|
|
424
492
|
if (matches.length === 0) {
|
|
493
|
+
if (detail === 'terse') {
|
|
494
|
+
return ok(JSON.stringify({ matchCount: 0, filter: compactFilterPayload(filter) }));
|
|
495
|
+
}
|
|
425
496
|
return ok(`No elements found matching ${JSON.stringify(filter)}`);
|
|
426
497
|
}
|
|
427
|
-
const
|
|
498
|
+
const formatted = sortA11yNodes(matches).map(node => formatNode(node, a11y, a11y.bounds));
|
|
499
|
+
if (detail === 'terse') {
|
|
500
|
+
const limited = formatted.slice(0, maxResults ?? 8);
|
|
501
|
+
return ok(JSON.stringify({
|
|
502
|
+
matchCount: formatted.length,
|
|
503
|
+
matches: limited.map(compactFormattedNode),
|
|
504
|
+
}));
|
|
505
|
+
}
|
|
506
|
+
const result = typeof maxResults === 'number' ? formatted.slice(0, maxResults) : formatted;
|
|
428
507
|
return ok(JSON.stringify(result, null, 2));
|
|
429
508
|
});
|
|
509
|
+
server.tool('geometra_find_action', `Resolve a clickable action by action label plus optional section, prompt, or item/card text. This is a narrower, lower-token path for repeated actions like "Open incident" in a queue row or "Add to cart" inside a product card.
|
|
510
|
+
|
|
511
|
+
Use this when geometra_page_model tells you the page shape, but you want one direct semantic action target instead of expanding a whole section.`, {
|
|
512
|
+
name: z.string().describe('Action label / accessible name to match'),
|
|
513
|
+
role: z.enum(['button', 'link']).optional().describe('Optional action role hint (button or link)'),
|
|
514
|
+
sectionText: z.string().optional().describe('Containing section/landmark/form/dialog text to disambiguate repeated actions'),
|
|
515
|
+
promptText: z.string().optional().describe('Nearby question/prompt text to disambiguate repeated actions'),
|
|
516
|
+
itemText: z.string().optional().describe('Nearby card/row/item label to disambiguate repeated actions'),
|
|
517
|
+
maxResults: z.number().int().min(1).max(12).optional().default(6).describe('Maximum number of matches to return'),
|
|
518
|
+
detail: detailInput(),
|
|
519
|
+
}, async ({ name, role, sectionText, promptText, itemText, maxResults, detail }) => {
|
|
520
|
+
const session = getSession();
|
|
521
|
+
if (!session)
|
|
522
|
+
return err('Not connected. Call geometra_connect first.');
|
|
523
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
524
|
+
if (!a11y)
|
|
525
|
+
return err('No UI tree available');
|
|
526
|
+
const filter = {
|
|
527
|
+
...(role ? { role } : {}),
|
|
528
|
+
name,
|
|
529
|
+
...(sectionText ? { sectionText } : {}),
|
|
530
|
+
...(promptText ? { promptText } : {}),
|
|
531
|
+
...(itemText ? { itemText } : {}),
|
|
532
|
+
};
|
|
533
|
+
const matches = sortA11yNodes(findNodes(a11y, filter).filter(node => node.focusable && (node.role === 'button' || node.role === 'link')));
|
|
534
|
+
if (matches.length === 0) {
|
|
535
|
+
if (detail === 'terse') {
|
|
536
|
+
return ok(JSON.stringify({ matchCount: 0, filter: compactFilterPayload(filter) }));
|
|
537
|
+
}
|
|
538
|
+
return ok(`No actions found matching ${JSON.stringify(filter)}`);
|
|
539
|
+
}
|
|
540
|
+
const formatted = matches.slice(0, maxResults).map(node => formatNode(node, a11y, a11y.bounds));
|
|
541
|
+
if (detail === 'terse') {
|
|
542
|
+
return ok(JSON.stringify({
|
|
543
|
+
matchCount: matches.length,
|
|
544
|
+
matches: formatted.map(compactFormattedNode),
|
|
545
|
+
}));
|
|
546
|
+
}
|
|
547
|
+
return ok(JSON.stringify(formatted));
|
|
548
|
+
});
|
|
430
549
|
server.registerTool('geometra_wait_for', {
|
|
431
550
|
description: `Wait for a semantic UI condition without guessing sleep durations. Use this for slow SPA transitions, resume parsing, custom validation alerts, disabled submit buttons, and value/state confirmation before submit.
|
|
432
551
|
|
|
433
552
|
The filter matches the same fields as geometra_query (strict schema — unknown keys error). Set \`present: false\` to wait until **no** node matches — for example Ashby/Lever-style “Parsing your resume” or any “Parsing…” banner: \`{ "text": "Parsing", "present": false }\` (tune the substring to the site). Do not use a textGone parameter; use \`text\` + \`present: false\`, or \`geometra_wait_for_resume_parse\` for the usual post-upload parsing banner.`,
|
|
434
553
|
inputSchema: geometraWaitForInputSchema,
|
|
435
|
-
}, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs }) => {
|
|
554
|
+
}, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, present, timeoutMs, detail }) => {
|
|
436
555
|
const session = getSession();
|
|
437
|
-
if (!session
|
|
556
|
+
if (!session)
|
|
438
557
|
return err('Not connected. Call geometra_connect first.');
|
|
439
558
|
const filterProbe = {
|
|
440
559
|
id,
|
|
@@ -442,6 +561,9 @@ The filter matches the same fields as geometra_query (strict schema — unknown
|
|
|
442
561
|
name,
|
|
443
562
|
text,
|
|
444
563
|
contextText,
|
|
564
|
+
promptText,
|
|
565
|
+
sectionText,
|
|
566
|
+
itemText,
|
|
445
567
|
value,
|
|
446
568
|
checked,
|
|
447
569
|
disabled,
|
|
@@ -461,6 +583,16 @@ The filter matches the same fields as geometra_query (strict schema — unknown
|
|
|
461
583
|
});
|
|
462
584
|
if (!waited.ok)
|
|
463
585
|
return err(waited.error);
|
|
586
|
+
if (detail === 'terse') {
|
|
587
|
+
const compact = waitConditionCompact(waited.value);
|
|
588
|
+
const matches = waited.value.matches
|
|
589
|
+
.slice(0, 3)
|
|
590
|
+
.map(match => compactFormattedNode(match));
|
|
591
|
+
return ok(JSON.stringify({
|
|
592
|
+
...compact,
|
|
593
|
+
...(matches.length > 0 ? { matches } : {}),
|
|
594
|
+
}));
|
|
595
|
+
}
|
|
464
596
|
if (!waited.value.present) {
|
|
465
597
|
return ok(waitConditionSuccessLine(waited.value));
|
|
466
598
|
}
|
|
@@ -473,7 +605,7 @@ Equivalent to \`geometra_wait_for\` with \`present: false\` and \`text\` set to
|
|
|
473
605
|
inputSchema: geometraWaitForResumeParseInputSchema,
|
|
474
606
|
}, async ({ text, timeoutMs }) => {
|
|
475
607
|
const session = getSession();
|
|
476
|
-
if (!session
|
|
608
|
+
if (!session)
|
|
477
609
|
return err('Not connected. Call geometra_connect first.');
|
|
478
610
|
const filter = { text };
|
|
479
611
|
const waited = await waitForSemanticCondition(session, {
|
|
@@ -508,6 +640,30 @@ Use \`kind: "text"\` for textboxes / textareas, \`"choice"\` for selects / combo
|
|
|
508
640
|
const resolvedFields = resolveFillFieldInputs(session, fields);
|
|
509
641
|
if (!resolvedFields.ok)
|
|
510
642
|
return err(resolvedFields.error);
|
|
643
|
+
if (!includeSteps) {
|
|
644
|
+
try {
|
|
645
|
+
const batched = await tryBatchedResolvedFields(session, resolvedFields.fields, detail);
|
|
646
|
+
if (batched.ok) {
|
|
647
|
+
const payload = {
|
|
648
|
+
completed: true,
|
|
649
|
+
execution: 'batched',
|
|
650
|
+
finalSource: batched.finalSource,
|
|
651
|
+
fieldCount: resolvedFields.fields.length,
|
|
652
|
+
successCount: resolvedFields.fields.length,
|
|
653
|
+
errorCount: 0,
|
|
654
|
+
final: batched.final,
|
|
655
|
+
};
|
|
656
|
+
if (failOnInvalid && batched.invalidRemaining > 0) {
|
|
657
|
+
return err(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
658
|
+
}
|
|
659
|
+
return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
catch (e) {
|
|
663
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
664
|
+
return err(message);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
511
667
|
const steps = [];
|
|
512
668
|
let stoppedAt;
|
|
513
669
|
for (let index = 0; index < resolvedFields.fields.length; index++) {
|
|
@@ -570,8 +726,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
570
726
|
.optional()
|
|
571
727
|
.default(false)
|
|
572
728
|
.describe('Include per-field step results in the JSON payload (default false for the smallest response)'),
|
|
729
|
+
resumeFromIndex: z
|
|
730
|
+
.number()
|
|
731
|
+
.int()
|
|
732
|
+
.min(0)
|
|
733
|
+
.optional()
|
|
734
|
+
.describe('Resume a partial fill from this field index (from a previous stoppedAt + 1). Skips already-filled fields.'),
|
|
573
735
|
detail: detailInput(),
|
|
574
|
-
}, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, detail }) => {
|
|
736
|
+
}, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, resumeFromIndex, detail }) => {
|
|
575
737
|
const directFields = !includeSteps && !formId && Object.keys(valuesById ?? {}).length === 0
|
|
576
738
|
? directLabelBatchFields(valuesByLabel)
|
|
577
739
|
: null;
|
|
@@ -720,7 +882,8 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
720
882
|
}
|
|
721
883
|
const steps = [];
|
|
722
884
|
let stoppedAt;
|
|
723
|
-
|
|
885
|
+
const startIndex = resumeFromIndex ?? 0;
|
|
886
|
+
for (let index = startIndex; index < planned.fields.length; index++) {
|
|
724
887
|
const field = planned.fields[index];
|
|
725
888
|
try {
|
|
726
889
|
const result = await executeFillField(session, field, detail);
|
|
@@ -744,15 +907,16 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
744
907
|
const errorCount = steps.length - successCount;
|
|
745
908
|
const payload = {
|
|
746
909
|
...connection,
|
|
747
|
-
completed: stoppedAt === undefined && steps.length === planned.fields.length,
|
|
910
|
+
completed: stoppedAt === undefined && (startIndex + steps.length) === planned.fields.length,
|
|
748
911
|
execution: 'sequential',
|
|
749
912
|
formId: schema.formId,
|
|
750
913
|
requestedValueCount: entryCount,
|
|
751
914
|
fieldCount: planned.fields.length,
|
|
752
915
|
successCount,
|
|
753
916
|
errorCount,
|
|
917
|
+
...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
|
|
754
918
|
...(includeSteps ? { steps } : {}),
|
|
755
|
-
...(stoppedAt !== undefined ? { stoppedAt } : {}),
|
|
919
|
+
...(stoppedAt !== undefined ? { stoppedAt, resumeFromIndex: stoppedAt + 1 } : {}),
|
|
756
920
|
...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
|
|
757
921
|
};
|
|
758
922
|
if (failOnInvalid && invalidRemaining > 0) {
|
|
@@ -762,7 +926,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
762
926
|
});
|
|
763
927
|
server.tool('geometra_run_actions', `Execute several Geometra actions in one MCP round trip and return one consolidated result. This is the preferred path for long, multi-step form fills where one-tool-per-field would otherwise create too much chatter.
|
|
764
928
|
|
|
765
|
-
Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_listbox_option\`, \`select_option\`, \`set_checked\`, \`wheel\`, \`wait_for\`, and \`fill_fields\`. \`click\` steps can also carry a nested \`waitFor\` condition.`, {
|
|
929
|
+
Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_listbox_option\`, \`select_option\`, \`set_checked\`, \`wheel\`, \`wait_for\`, and \`fill_fields\`. \`click\` steps can also carry a nested \`waitFor\` condition. Pass \`pageUrl\` / \`url\` to auto-connect so an entire flow can run in one MCP call.`, {
|
|
930
|
+
url: z.string().optional().describe('Optional target URL. Use a ws:// Geometra server URL or an http(s) page URL to auto-connect before running actions.'),
|
|
931
|
+
pageUrl: z.string().optional().describe('Optional http(s) page URL to auto-connect before running actions. Prefer this over url for browser pages.'),
|
|
932
|
+
port: z.number().int().min(0).max(65535).optional().describe('Preferred local port for an auto-spawned proxy (default: ephemeral OS-assigned port).'),
|
|
933
|
+
headless: z.boolean().optional().describe('Run Chromium headless when auto-spawning a proxy (default false = visible window).'),
|
|
934
|
+
width: z.number().int().positive().optional().describe('Viewport width for auto-connected sessions.'),
|
|
935
|
+
height: z.number().int().positive().optional().describe('Viewport height for auto-connected sessions.'),
|
|
936
|
+
slowMo: z.number().int().nonnegative().optional().describe('Playwright slowMo (ms) when auto-spawning a proxy.'),
|
|
766
937
|
actions: z.array(batchActionSchema).min(1).max(80).describe('Ordered high-level action steps to run sequentially'),
|
|
767
938
|
stopOnError: z.boolean().optional().default(true).describe('Stop at the first failing step (default true)'),
|
|
768
939
|
includeSteps: z
|
|
@@ -770,24 +941,81 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
770
941
|
.optional()
|
|
771
942
|
.default(true)
|
|
772
943
|
.describe('Include per-action step results in the JSON payload (default true). Set false for the smallest batch response.'),
|
|
944
|
+
output: z.enum(['full', 'final']).optional().default('full').describe('`full` (default) returns counts and optional step listings. `final` keeps only completion state plus final semantic signals.'),
|
|
773
945
|
detail: detailInput(),
|
|
774
|
-
}, async ({ actions, stopOnError, includeSteps, detail }) => {
|
|
775
|
-
const
|
|
776
|
-
|
|
777
|
-
|
|
946
|
+
}, async ({ url, pageUrl, port, headless, width, height, slowMo, actions, stopOnError, includeSteps, output, detail }) => {
|
|
947
|
+
const resolved = await ensureToolSession({
|
|
948
|
+
url,
|
|
949
|
+
pageUrl,
|
|
950
|
+
port,
|
|
951
|
+
headless,
|
|
952
|
+
width,
|
|
953
|
+
height,
|
|
954
|
+
slowMo,
|
|
955
|
+
awaitInitialFrame: canDeferInitialFrameForRunActions(actions) ? false : undefined,
|
|
956
|
+
}, 'Not connected. Call geometra_connect first, or pass pageUrl/url to geometra_run_actions.');
|
|
957
|
+
if (!resolved.ok)
|
|
958
|
+
return err(resolved.error);
|
|
959
|
+
const session = resolved.session;
|
|
960
|
+
const connection = autoConnectionPayload(resolved);
|
|
778
961
|
const steps = [];
|
|
779
962
|
let stoppedAt;
|
|
963
|
+
const batchStartedAt = performance.now();
|
|
780
964
|
for (let index = 0; index < actions.length; index++) {
|
|
781
965
|
const action = actions[index];
|
|
966
|
+
const startedAt = performance.now();
|
|
967
|
+
let uiTreeWaitMs = 0;
|
|
782
968
|
try {
|
|
969
|
+
if (actionNeedsUiTree(action) && (!session.tree || !session.layout)) {
|
|
970
|
+
const uiTreeWaitStartedAt = performance.now();
|
|
971
|
+
await waitForUiCondition(session, () => Boolean(session.tree && session.layout), 2_000);
|
|
972
|
+
uiTreeWaitMs = performance.now() - uiTreeWaitStartedAt;
|
|
973
|
+
}
|
|
783
974
|
const result = await executeBatchAction(session, action, detail, includeSteps);
|
|
975
|
+
const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
|
|
976
|
+
const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
|
|
977
|
+
const stepSignals = includeSteps ? (() => {
|
|
978
|
+
const a = sessionA11y(session);
|
|
979
|
+
if (!a)
|
|
980
|
+
return undefined;
|
|
981
|
+
const s = collectSessionSignals(a);
|
|
982
|
+
return { invalidCount: s.invalidFields.length, alertCount: s.alerts.length, dialogCount: s.dialogCount, busyCount: s.busyCount };
|
|
983
|
+
})() : undefined;
|
|
784
984
|
steps.push(detail === 'verbose'
|
|
785
|
-
? {
|
|
786
|
-
|
|
985
|
+
? {
|
|
986
|
+
index,
|
|
987
|
+
type: action.type,
|
|
988
|
+
ok: true,
|
|
989
|
+
elapsedMs,
|
|
990
|
+
cumulativeMs,
|
|
991
|
+
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
992
|
+
summary: result.summary,
|
|
993
|
+
...(stepSignals ? { signals: stepSignals } : {}),
|
|
994
|
+
}
|
|
995
|
+
: {
|
|
996
|
+
index,
|
|
997
|
+
type: action.type,
|
|
998
|
+
ok: true,
|
|
999
|
+
elapsedMs,
|
|
1000
|
+
cumulativeMs,
|
|
1001
|
+
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
1002
|
+
...result.compact,
|
|
1003
|
+
...(stepSignals ? { signals: stepSignals } : {}),
|
|
1004
|
+
});
|
|
787
1005
|
}
|
|
788
1006
|
catch (e) {
|
|
789
1007
|
const message = e instanceof Error ? e.message : String(e);
|
|
790
|
-
|
|
1008
|
+
const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
|
|
1009
|
+
const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
|
|
1010
|
+
steps.push({
|
|
1011
|
+
index,
|
|
1012
|
+
type: action.type,
|
|
1013
|
+
ok: false,
|
|
1014
|
+
elapsedMs,
|
|
1015
|
+
cumulativeMs,
|
|
1016
|
+
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
1017
|
+
error: message,
|
|
1018
|
+
});
|
|
791
1019
|
if (stopOnError) {
|
|
792
1020
|
stoppedAt = index;
|
|
793
1021
|
break;
|
|
@@ -797,15 +1025,23 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
797
1025
|
const after = sessionA11y(session);
|
|
798
1026
|
const successCount = steps.filter(step => step.ok === true).length;
|
|
799
1027
|
const errorCount = steps.length - successCount;
|
|
800
|
-
const payload =
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
1028
|
+
const payload = output === 'final'
|
|
1029
|
+
? {
|
|
1030
|
+
...connection,
|
|
1031
|
+
completed: stoppedAt === undefined && steps.length === actions.length,
|
|
1032
|
+
...(stoppedAt !== undefined ? { stoppedAt } : {}),
|
|
1033
|
+
...(after ? { final: sessionSignalsPayload(collectSessionSignals(after), detail) } : {}),
|
|
1034
|
+
}
|
|
1035
|
+
: {
|
|
1036
|
+
...connection,
|
|
1037
|
+
completed: stoppedAt === undefined && steps.length === actions.length,
|
|
1038
|
+
stepCount: actions.length,
|
|
1039
|
+
successCount,
|
|
1040
|
+
errorCount,
|
|
1041
|
+
...(includeSteps ? { steps } : {}),
|
|
1042
|
+
...(stoppedAt !== undefined ? { stoppedAt } : {}),
|
|
1043
|
+
...(after ? { final: sessionSignalsPayload(collectSessionSignals(after), detail) } : {}),
|
|
1044
|
+
};
|
|
809
1045
|
return ok(JSON.stringify(payload, null, detail === 'verbose' ? 2 : undefined));
|
|
810
1046
|
});
|
|
811
1047
|
// ── page model ────────────────────────────────────────────────
|
|
@@ -828,15 +1064,21 @@ Use this first on normal HTML pages when you want to understand the page shape w
|
|
|
828
1064
|
.optional()
|
|
829
1065
|
.default(8)
|
|
830
1066
|
.describe('Cap returned landmarks/forms/dialogs/lists per kind (default 8).'),
|
|
831
|
-
|
|
1067
|
+
includeScreenshot: z
|
|
1068
|
+
.boolean()
|
|
1069
|
+
.optional()
|
|
1070
|
+
.default(false)
|
|
1071
|
+
.describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy. Use when geometry alone is ambiguous (icon-only buttons, visual styling cues).'),
|
|
1072
|
+
}, async ({ maxPrimaryActions, maxSectionsPerKind, includeScreenshot }) => {
|
|
832
1073
|
const session = getSession();
|
|
833
|
-
if (!session
|
|
1074
|
+
if (!session)
|
|
834
1075
|
return err('Not connected. Call geometra_connect first.');
|
|
835
|
-
const a11y =
|
|
1076
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
836
1077
|
if (!a11y)
|
|
837
1078
|
return err('No UI tree available');
|
|
838
1079
|
const model = buildPageModel(a11y, { maxPrimaryActions, maxSectionsPerKind });
|
|
839
|
-
|
|
1080
|
+
const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
|
|
1081
|
+
return ok(JSON.stringify(model), screenshot);
|
|
840
1082
|
});
|
|
841
1083
|
server.tool('geometra_form_schema', `Get a compact, fill-oriented schema for forms on the page. This is the preferred discovery step before geometra_fill_form.
|
|
842
1084
|
|
|
@@ -861,6 +1103,9 @@ Unlike geometra_expand_section, this collapses repeated radio/button groups into
|
|
|
861
1103
|
if (!resolved.ok)
|
|
862
1104
|
return err(resolved.error);
|
|
863
1105
|
const session = resolved.session;
|
|
1106
|
+
if (!(await ensureSessionUiTree(session, 4_000))) {
|
|
1107
|
+
return err('Timed out waiting for the initial UI tree after connect.');
|
|
1108
|
+
}
|
|
864
1109
|
const payload = formSchemaResponsePayload(session, {
|
|
865
1110
|
formId,
|
|
866
1111
|
maxFields,
|
|
@@ -898,9 +1143,9 @@ Use this after geometra_page_model when you know which form/dialog/list/landmark
|
|
|
898
1143
|
includeBounds: z.boolean().optional().default(false).describe('Include bounds for fields/actions/headings/items'),
|
|
899
1144
|
}, async ({ id, maxHeadings, maxFields, fieldOffset, onlyRequiredFields, onlyInvalidFields, maxActions, actionOffset, maxLists, listOffset, maxItems, itemOffset, maxTextPreview, includeBounds, }) => {
|
|
900
1145
|
const session = getSession();
|
|
901
|
-
if (!session
|
|
1146
|
+
if (!session)
|
|
902
1147
|
return err('Not connected. Call geometra_connect first.');
|
|
903
|
-
const a11y =
|
|
1148
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
904
1149
|
if (!a11y)
|
|
905
1150
|
return err('No UI tree available');
|
|
906
1151
|
const detail = expandPageSection(a11y, id, {
|
|
@@ -937,7 +1182,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
|
|
|
937
1182
|
.optional()
|
|
938
1183
|
.default(2_500)
|
|
939
1184
|
.describe('Per-scroll wait timeout (default 2500ms)'),
|
|
940
|
-
}, async ({ id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxSteps, timeoutMs }) => {
|
|
1185
|
+
}, async ({ id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxSteps, timeoutMs }) => {
|
|
941
1186
|
const session = getSession();
|
|
942
1187
|
if (!session)
|
|
943
1188
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -947,6 +1192,9 @@ Use the same filters as geometra_query, plus an optional match index when repeat
|
|
|
947
1192
|
name,
|
|
948
1193
|
text,
|
|
949
1194
|
contextText,
|
|
1195
|
+
promptText,
|
|
1196
|
+
sectionText,
|
|
1197
|
+
itemText,
|
|
950
1198
|
value,
|
|
951
1199
|
checked,
|
|
952
1200
|
disabled,
|
|
@@ -958,7 +1206,7 @@ Use the same filters as geometra_query, plus an optional match index when repeat
|
|
|
958
1206
|
busy,
|
|
959
1207
|
};
|
|
960
1208
|
if (!hasNodeFilter(filter))
|
|
961
|
-
return err('Provide at least one reveal filter (id, role, name, text, contextText, value, or state)');
|
|
1209
|
+
return err('Provide at least one reveal filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, or state)');
|
|
962
1210
|
const revealed = await revealSemanticTarget(session, {
|
|
963
1211
|
filter,
|
|
964
1212
|
index: index ?? 0,
|
|
@@ -1001,7 +1249,7 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
1001
1249
|
.optional()
|
|
1002
1250
|
.describe('Optional action wait timeout (use a longer value for slow submits or route transitions)'),
|
|
1003
1251
|
detail: detailInput(),
|
|
1004
|
-
}, async ({ x, y, id, role, name, text, contextText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxRevealSteps, revealTimeoutMs, waitFor, timeoutMs, detail }) => {
|
|
1252
|
+
}, async ({ x, y, id, role, name, text, contextText, promptText, sectionText, itemText, value, checked, disabled, focused, selected, expanded, invalid, required, busy, index, fullyVisible, maxRevealSteps, revealTimeoutMs, waitFor, timeoutMs, detail }) => {
|
|
1005
1253
|
const session = getSession();
|
|
1006
1254
|
if (!session)
|
|
1007
1255
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1015,6 +1263,9 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
1015
1263
|
name,
|
|
1016
1264
|
text,
|
|
1017
1265
|
contextText,
|
|
1266
|
+
promptText,
|
|
1267
|
+
sectionText,
|
|
1268
|
+
itemText,
|
|
1018
1269
|
value,
|
|
1019
1270
|
checked,
|
|
1020
1271
|
disabled,
|
|
@@ -1046,6 +1297,9 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
1046
1297
|
name: waitFor.name,
|
|
1047
1298
|
text: waitFor.text,
|
|
1048
1299
|
contextText: waitFor.contextText,
|
|
1300
|
+
promptText: waitFor.promptText,
|
|
1301
|
+
sectionText: waitFor.sectionText,
|
|
1302
|
+
itemText: waitFor.itemText,
|
|
1049
1303
|
value: waitFor.value,
|
|
1050
1304
|
checked: waitFor.checked,
|
|
1051
1305
|
disabled: waitFor.disabled,
|
|
@@ -1062,8 +1316,20 @@ After clicking, returns a compact semantic delta when possible (dialogs/forms/li
|
|
|
1062
1316
|
if (!postWait.ok)
|
|
1063
1317
|
return err([...lines, postWait.error].join('\n'));
|
|
1064
1318
|
lines.push(`Post-click ${waitConditionSuccessLine(postWait.value)}`);
|
|
1319
|
+
const compact = {
|
|
1320
|
+
at: { x: resolved.value.x, y: resolved.value.y },
|
|
1321
|
+
...(resolved.value.target ? { target: compactNodeReference(resolved.value.target), revealSteps: resolved.value.revealAttempts ?? 0 } : {}),
|
|
1322
|
+
...waitStatusPayload(wait),
|
|
1323
|
+
postWait: waitConditionCompact(postWait.value),
|
|
1324
|
+
};
|
|
1325
|
+
return ok(detailText(lines.filter(Boolean).join('\n'), compact, detail));
|
|
1065
1326
|
}
|
|
1066
|
-
|
|
1327
|
+
const compact = {
|
|
1328
|
+
at: { x: resolved.value.x, y: resolved.value.y },
|
|
1329
|
+
...(resolved.value.target ? { target: compactNodeReference(resolved.value.target), revealSteps: resolved.value.revealAttempts ?? 0 } : {}),
|
|
1330
|
+
...waitStatusPayload(wait),
|
|
1331
|
+
};
|
|
1332
|
+
return ok(detailText(lines.filter(Boolean).join('\n'), compact, detail));
|
|
1067
1333
|
});
|
|
1068
1334
|
// ── type ─────────────────────────────────────────────────────
|
|
1069
1335
|
server.tool('geometra_type', `Type text into the currently focused element. First click a textbox/input with geometra_click to focus it, then use this to type.
|
|
@@ -1085,7 +1351,10 @@ Each character is sent as a key event through the geometry protocol. Returns a c
|
|
|
1085
1351
|
const before = sessionA11y(session);
|
|
1086
1352
|
const wait = await sendType(session, text, timeoutMs);
|
|
1087
1353
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1088
|
-
return ok(`Typed "${text}".\n${summary}
|
|
1354
|
+
return ok(detailText(`Typed "${text}".\n${summary}`, {
|
|
1355
|
+
...compactTextValue(text),
|
|
1356
|
+
...waitStatusPayload(wait),
|
|
1357
|
+
}, detail));
|
|
1089
1358
|
});
|
|
1090
1359
|
// ── key ──────────────────────────────────────────────────────
|
|
1091
1360
|
server.tool('geometra_key', `Send a special key press (Enter, Tab, Escape, ArrowDown, etc.) to the Geometra UI. Useful for form submission, focus navigation, and keyboard shortcuts.`, {
|
|
@@ -1109,7 +1378,10 @@ Each character is sent as a key event through the geometry protocol. Returns a c
|
|
|
1109
1378
|
const before = sessionA11y(session);
|
|
1110
1379
|
const wait = await sendKey(session, key, { shift, ctrl, meta, alt }, timeoutMs);
|
|
1111
1380
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1112
|
-
return ok(`Pressed ${formatKeyCombo(key, { shift, ctrl, meta, alt })}.\n${summary}
|
|
1381
|
+
return ok(detailText(`Pressed ${formatKeyCombo(key, { shift, ctrl, meta, alt })}.\n${summary}`, {
|
|
1382
|
+
key: formatKeyCombo(key, { shift, ctrl, meta, alt }),
|
|
1383
|
+
...waitStatusPayload(wait),
|
|
1384
|
+
}, detail));
|
|
1113
1385
|
});
|
|
1114
1386
|
// ── upload files (proxy) ───────────────────────────────────────
|
|
1115
1387
|
server.tool('geometra_upload_files', `Attach local files to a file input. Requires \`@geometra/proxy\` (paths exist on the proxy host).
|
|
@@ -1126,6 +1398,8 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
1126
1398
|
.describe('Upload strategy (default auto)'),
|
|
1127
1399
|
dropX: z.number().optional().describe('Drop target X (viewport) for strategy drop'),
|
|
1128
1400
|
dropY: z.number().optional().describe('Drop target Y (viewport) for strategy drop'),
|
|
1401
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated file inputs'),
|
|
1402
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated file inputs'),
|
|
1129
1403
|
timeoutMs: z
|
|
1130
1404
|
.number()
|
|
1131
1405
|
.int()
|
|
@@ -1134,7 +1408,7 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
1134
1408
|
.optional()
|
|
1135
1409
|
.describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
|
|
1136
1410
|
detail: detailInput(),
|
|
1137
|
-
}, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, timeoutMs, detail }) => {
|
|
1411
|
+
}, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1138
1412
|
const session = getSession();
|
|
1139
1413
|
if (!session)
|
|
1140
1414
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1148,7 +1422,13 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
1148
1422
|
drop: dropX !== undefined && dropY !== undefined ? { x: dropX, y: dropY } : undefined,
|
|
1149
1423
|
}, timeoutMs ?? 8_000);
|
|
1150
1424
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1151
|
-
return ok(`Uploaded ${paths.length} file(s).\n${summary}
|
|
1425
|
+
return ok(detailText(`Uploaded ${paths.length} file(s).\n${summary}`, {
|
|
1426
|
+
fileCount: paths.length,
|
|
1427
|
+
...(fieldLabel ? { fieldLabel } : {}),
|
|
1428
|
+
...(strategy ? { strategy } : {}),
|
|
1429
|
+
...waitStatusPayload(wait),
|
|
1430
|
+
...(fieldLabel ? { readback: fieldStatePayload(session, fieldLabel) } : {}),
|
|
1431
|
+
}, detail));
|
|
1152
1432
|
}
|
|
1153
1433
|
catch (e) {
|
|
1154
1434
|
return err(e.message);
|
|
@@ -1162,6 +1442,8 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
1162
1442
|
openX: z.number().optional().describe('Click to open dropdown'),
|
|
1163
1443
|
openY: z.number().optional().describe('Click to open dropdown'),
|
|
1164
1444
|
fieldLabel: z.string().optional().describe('Field label of the dropdown/combobox to open semantically (e.g. "Location")'),
|
|
1445
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated dropdowns with the same label'),
|
|
1446
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated dropdowns'),
|
|
1165
1447
|
query: z.string().optional().describe('Optional text to type into a searchable combobox before selecting'),
|
|
1166
1448
|
timeoutMs: z
|
|
1167
1449
|
.number()
|
|
@@ -1171,7 +1453,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
1171
1453
|
.optional()
|
|
1172
1454
|
.describe('Optional action wait timeout for slow dropdowns / remote search results'),
|
|
1173
1455
|
detail: detailInput(),
|
|
1174
|
-
}, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs, detail }) => {
|
|
1456
|
+
}, async ({ label, exact, openX, openY, fieldLabel, contextText, sectionText, query, timeoutMs, detail }) => {
|
|
1175
1457
|
const session = getSession();
|
|
1176
1458
|
if (!session)
|
|
1177
1459
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1185,11 +1467,17 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
1185
1467
|
}, timeoutMs);
|
|
1186
1468
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1187
1469
|
const fieldSummary = fieldLabel ? summarizeFieldLabelState(session, fieldLabel) : undefined;
|
|
1188
|
-
|
|
1470
|
+
const summaryText = [
|
|
1189
1471
|
`Picked listbox option "${label}".`,
|
|
1190
1472
|
fieldSummary,
|
|
1191
1473
|
summary,
|
|
1192
|
-
].filter(Boolean).join('\n')
|
|
1474
|
+
].filter(Boolean).join('\n');
|
|
1475
|
+
return ok(detailText(summaryText, {
|
|
1476
|
+
label,
|
|
1477
|
+
...(fieldLabel ? { fieldLabel } : {}),
|
|
1478
|
+
...waitStatusPayload(wait),
|
|
1479
|
+
...(fieldLabel ? { readback: fieldStatePayload(session, fieldLabel) } : {}),
|
|
1480
|
+
}, detail));
|
|
1193
1481
|
}
|
|
1194
1482
|
catch (e) {
|
|
1195
1483
|
return err(e.message);
|
|
@@ -1204,6 +1492,8 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
|
|
|
1204
1492
|
value: z.string().optional().describe('Option value= attribute'),
|
|
1205
1493
|
label: z.string().optional().describe('Visible option label (substring match)'),
|
|
1206
1494
|
index: z.number().int().min(0).optional().describe('Zero-based option index'),
|
|
1495
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated selects'),
|
|
1496
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated selects'),
|
|
1207
1497
|
timeoutMs: z
|
|
1208
1498
|
.number()
|
|
1209
1499
|
.int()
|
|
@@ -1212,7 +1502,7 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
|
|
|
1212
1502
|
.optional()
|
|
1213
1503
|
.describe('Optional action wait timeout'),
|
|
1214
1504
|
detail: detailInput(),
|
|
1215
|
-
}, async ({ x, y, value, label, index, timeoutMs, detail }) => {
|
|
1505
|
+
}, async ({ x, y, value, label, index, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1216
1506
|
const session = getSession();
|
|
1217
1507
|
if (!session)
|
|
1218
1508
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1223,7 +1513,13 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
|
|
|
1223
1513
|
try {
|
|
1224
1514
|
const wait = await sendSelectOption(session, x, y, { value, label, index }, timeoutMs);
|
|
1225
1515
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1226
|
-
return ok(`Selected option.\n${summary}
|
|
1516
|
+
return ok(detailText(`Selected option.\n${summary}`, {
|
|
1517
|
+
at: { x, y },
|
|
1518
|
+
...(value !== undefined ? { value } : {}),
|
|
1519
|
+
...(label !== undefined ? { label } : {}),
|
|
1520
|
+
...(index !== undefined ? { index } : {}),
|
|
1521
|
+
...waitStatusPayload(wait),
|
|
1522
|
+
}, detail));
|
|
1227
1523
|
}
|
|
1228
1524
|
catch (e) {
|
|
1229
1525
|
return err(e.message);
|
|
@@ -1236,6 +1532,8 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1236
1532
|
checked: z.boolean().optional().default(true).describe('Desired checked state (radios only support true)'),
|
|
1237
1533
|
exact: z.boolean().optional().describe('Exact label match'),
|
|
1238
1534
|
controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
|
|
1535
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated checkboxes/radios'),
|
|
1536
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated checkboxes/radios'),
|
|
1239
1537
|
timeoutMs: z
|
|
1240
1538
|
.number()
|
|
1241
1539
|
.int()
|
|
@@ -1244,7 +1542,7 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1244
1542
|
.optional()
|
|
1245
1543
|
.describe('Optional action wait timeout'),
|
|
1246
1544
|
detail: detailInput(),
|
|
1247
|
-
}, async ({ label, checked, exact, controlType, timeoutMs, detail }) => {
|
|
1545
|
+
}, async ({ label, checked, exact, controlType, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1248
1546
|
const session = getSession();
|
|
1249
1547
|
if (!session)
|
|
1250
1548
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1252,7 +1550,12 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1252
1550
|
try {
|
|
1253
1551
|
const wait = await sendSetChecked(session, label, { checked, exact, controlType }, timeoutMs);
|
|
1254
1552
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1255
|
-
return ok(`Set ${controlType ?? 'checkbox/radio'} "${label}" to ${String(checked ?? true)}.\n${summary}
|
|
1553
|
+
return ok(detailText(`Set ${controlType ?? 'checkbox/radio'} "${label}" to ${String(checked ?? true)}.\n${summary}`, {
|
|
1554
|
+
label,
|
|
1555
|
+
checked: checked ?? true,
|
|
1556
|
+
...(controlType ? { controlType } : {}),
|
|
1557
|
+
...waitStatusPayload(wait),
|
|
1558
|
+
}, detail));
|
|
1256
1559
|
}
|
|
1257
1560
|
catch (e) {
|
|
1258
1561
|
return err(e.message);
|
|
@@ -1280,12 +1583,68 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1280
1583
|
try {
|
|
1281
1584
|
const wait = await sendWheel(session, deltaY, { deltaX, x, y }, timeoutMs);
|
|
1282
1585
|
const summary = postActionSummary(session, before, wait, detail);
|
|
1283
|
-
return ok(`Wheel delta (${deltaX ?? 0}, ${deltaY}).\n${summary}
|
|
1586
|
+
return ok(detailText(`Wheel delta (${deltaX ?? 0}, ${deltaY}).\n${summary}`, {
|
|
1587
|
+
deltaY,
|
|
1588
|
+
...(deltaX !== undefined ? { deltaX } : {}),
|
|
1589
|
+
...(x !== undefined && y !== undefined ? { at: { x, y } } : {}),
|
|
1590
|
+
...waitStatusPayload(wait),
|
|
1591
|
+
}, detail));
|
|
1284
1592
|
}
|
|
1285
1593
|
catch (e) {
|
|
1286
1594
|
return err(e.message);
|
|
1287
1595
|
}
|
|
1288
1596
|
});
|
|
1597
|
+
// ── list items (virtualized list pagination) ─────────────────
|
|
1598
|
+
server.tool('geometra_list_items', `Auto-scroll a virtualized or long list and collect all visible items across scroll positions. Requires \`@geometra/proxy\`.
|
|
1599
|
+
|
|
1600
|
+
Use this for dropdowns, location pickers, or any scrollable list where items are rendered on demand. Scrolls down in steps, collecting new items each time, until no new items appear or the cap is reached.`, {
|
|
1601
|
+
listId: z.string().optional().describe('Stable section id from geometra_page_model (e.g. ls:2.1) to scope item collection'),
|
|
1602
|
+
role: z.string().optional().describe('Role filter for list items (default: listitem)'),
|
|
1603
|
+
scrollX: z.number().optional().describe('X coordinate to position mouse for scrolling (default: viewport center)'),
|
|
1604
|
+
scrollY: z.number().optional().describe('Y coordinate to position mouse for scrolling (default: viewport center)'),
|
|
1605
|
+
maxItems: z.number().int().min(1).max(500).optional().default(100).describe('Cap collected items (default 100)'),
|
|
1606
|
+
maxScrollSteps: z.number().int().min(1).max(50).optional().default(20).describe('Max scroll steps before stopping (default 20)'),
|
|
1607
|
+
scrollDelta: z.number().optional().default(300).describe('Vertical scroll delta per step (default 300)'),
|
|
1608
|
+
}, async ({ listId: _listId, role, scrollX, scrollY, maxItems, maxScrollSteps, scrollDelta }) => {
|
|
1609
|
+
const session = getSession();
|
|
1610
|
+
if (!session)
|
|
1611
|
+
return err('Not connected. Call geometra_connect first.');
|
|
1612
|
+
const itemRole = role ?? 'listitem';
|
|
1613
|
+
const collected = new Map();
|
|
1614
|
+
const cx = scrollX ?? 400;
|
|
1615
|
+
const cy = scrollY ?? 400;
|
|
1616
|
+
for (let step = 0; step < maxScrollSteps; step++) {
|
|
1617
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
1618
|
+
if (!a11y)
|
|
1619
|
+
break;
|
|
1620
|
+
const items = findNodes(a11y, { role: itemRole });
|
|
1621
|
+
let newCount = 0;
|
|
1622
|
+
for (const item of items) {
|
|
1623
|
+
const id = nodeIdForPath(item.path);
|
|
1624
|
+
if (!collected.has(id)) {
|
|
1625
|
+
collected.set(id, {
|
|
1626
|
+
...(item.name ? { name: item.name } : {}),
|
|
1627
|
+
...(item.value ? { value: item.value } : {}),
|
|
1628
|
+
});
|
|
1629
|
+
newCount++;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
if (collected.size >= maxItems || newCount === 0)
|
|
1633
|
+
break;
|
|
1634
|
+
try {
|
|
1635
|
+
await sendWheel(session, scrollDelta, { x: cx, y: cy }, 1_000);
|
|
1636
|
+
}
|
|
1637
|
+
catch {
|
|
1638
|
+
break;
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
const items = [...collected.entries()].slice(0, maxItems).map(([id, data]) => ({ id, ...data }));
|
|
1642
|
+
return ok(JSON.stringify({
|
|
1643
|
+
itemCount: items.length,
|
|
1644
|
+
items,
|
|
1645
|
+
truncated: collected.size > maxItems,
|
|
1646
|
+
}));
|
|
1647
|
+
});
|
|
1289
1648
|
// ── snapshot ─────────────────────────────────────────────────
|
|
1290
1649
|
server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout. Use **form-required** to list required fields across forms, including offscreen ones, with bounds + visibility + scroll hints for long application flows.
|
|
1291
1650
|
|
|
@@ -1306,15 +1665,21 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1306
1665
|
formId: z.string().optional().describe('Optional form id from geometra_form_schema / geometra_page_model when view=form-required'),
|
|
1307
1666
|
maxFields: z.number().int().min(1).max(200).optional().default(80).describe('Per-form field cap when view=form-required'),
|
|
1308
1667
|
includeOptions: z.boolean().optional().default(false).describe('Include explicit choice option labels when view=form-required'),
|
|
1309
|
-
|
|
1668
|
+
includeScreenshot: z
|
|
1669
|
+
.boolean()
|
|
1670
|
+
.optional()
|
|
1671
|
+
.default(false)
|
|
1672
|
+
.describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy.'),
|
|
1673
|
+
}, async ({ view, maxNodes, formId, maxFields, includeOptions, includeScreenshot }) => {
|
|
1310
1674
|
const session = getSession();
|
|
1311
|
-
if (!session
|
|
1675
|
+
if (!session)
|
|
1312
1676
|
return err('Not connected. Call geometra_connect first.');
|
|
1313
|
-
const a11y =
|
|
1677
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
1314
1678
|
if (!a11y)
|
|
1315
1679
|
return err('No UI tree available');
|
|
1680
|
+
const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
|
|
1316
1681
|
if (view === 'full') {
|
|
1317
|
-
return ok(JSON.stringify(a11y, null, 2));
|
|
1682
|
+
return ok(JSON.stringify(a11y, null, 2), screenshot);
|
|
1318
1683
|
}
|
|
1319
1684
|
if (view === 'form-required') {
|
|
1320
1685
|
const payload = {
|
|
@@ -1327,7 +1692,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1327
1692
|
includeContext: 'auto',
|
|
1328
1693
|
}),
|
|
1329
1694
|
};
|
|
1330
|
-
return ok(JSON.stringify(payload));
|
|
1695
|
+
return ok(JSON.stringify(payload), screenshot);
|
|
1331
1696
|
}
|
|
1332
1697
|
const { nodes, truncated, context } = buildCompactUiIndex(a11y, { maxNodes });
|
|
1333
1698
|
const payload = {
|
|
@@ -1337,7 +1702,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1337
1702
|
nodes,
|
|
1338
1703
|
truncated,
|
|
1339
1704
|
};
|
|
1340
|
-
return ok(JSON.stringify(payload));
|
|
1705
|
+
return ok(JSON.stringify(payload), screenshot);
|
|
1341
1706
|
});
|
|
1342
1707
|
// ── layout ───────────────────────────────────────────────────
|
|
1343
1708
|
server.tool('geometra_layout', `Get the raw computed layout geometry — the exact {x, y, width, height} for every node in the UI tree. This is the lowest-level view, useful for pixel-precise assertions in tests.
|
|
@@ -1365,7 +1730,7 @@ function compactSessionSummary(session) {
|
|
|
1365
1730
|
return sessionOverviewFromA11y(a11y);
|
|
1366
1731
|
}
|
|
1367
1732
|
function connectPayload(session, opts) {
|
|
1368
|
-
const a11y = sessionA11y(session);
|
|
1733
|
+
const a11y = opts.detail === 'verbose' ? sessionA11y(session) : null;
|
|
1369
1734
|
return {
|
|
1370
1735
|
connected: true,
|
|
1371
1736
|
transport: opts.transport,
|
|
@@ -1387,6 +1752,17 @@ function sessionA11y(session) {
|
|
|
1387
1752
|
session.cachedA11yRevision = session.updateRevision;
|
|
1388
1753
|
return a11y;
|
|
1389
1754
|
}
|
|
1755
|
+
async function ensureSessionUiTree(session, timeoutMs = 4_000) {
|
|
1756
|
+
if (session.tree && session.layout)
|
|
1757
|
+
return true;
|
|
1758
|
+
return await waitForUiCondition(session, () => Boolean(session.tree && session.layout), timeoutMs);
|
|
1759
|
+
}
|
|
1760
|
+
async function sessionA11yWhenReady(session, timeoutMs = 4_000) {
|
|
1761
|
+
const ready = await ensureSessionUiTree(session, timeoutMs);
|
|
1762
|
+
if (!ready)
|
|
1763
|
+
return null;
|
|
1764
|
+
return sessionA11y(session);
|
|
1765
|
+
}
|
|
1390
1766
|
function shortHash(value) {
|
|
1391
1767
|
return createHash('sha1').update(value).digest('hex').slice(0, 12);
|
|
1392
1768
|
}
|
|
@@ -1438,8 +1814,10 @@ function packedFormSchemas(forms) {
|
|
|
1438
1814
|
...(field.valueLength !== undefined ? { vl: field.valueLength } : {}),
|
|
1439
1815
|
...(field.checked !== undefined ? { c: field.checked ? 1 : 0 } : {}),
|
|
1440
1816
|
...(field.values && field.values.length > 0 ? { vs: field.values } : {}),
|
|
1817
|
+
...(field.aliases ? { al: field.aliases } : {}),
|
|
1441
1818
|
...(field.context ? { x: field.context } : {}),
|
|
1442
1819
|
})),
|
|
1820
|
+
...(form.sections ? { s: form.sections.map(s => ({ n: s.name, fi: s.fieldIds })) } : {}),
|
|
1443
1821
|
}));
|
|
1444
1822
|
}
|
|
1445
1823
|
function formSchemaResponsePayload(session, opts) {
|
|
@@ -1506,10 +1884,23 @@ function connectResponsePayload(session, opts) {
|
|
|
1506
1884
|
nextPayload.formSchema = formSchemaResponsePayload(session, opts.formSchema ?? {});
|
|
1507
1885
|
}
|
|
1508
1886
|
if (opts.returnPageModel) {
|
|
1509
|
-
nextPayload.pageModel =
|
|
1887
|
+
nextPayload.pageModel = opts.pageModelMode === 'deferred'
|
|
1888
|
+
? deferredPageModelConnectPayload(session, opts.pageModelOptions)
|
|
1889
|
+
: pageModelResponsePayload(session, opts.pageModelOptions);
|
|
1510
1890
|
}
|
|
1511
1891
|
return nextPayload;
|
|
1512
1892
|
}
|
|
1893
|
+
function deferredPageModelConnectPayload(session, options) {
|
|
1894
|
+
return {
|
|
1895
|
+
deferred: true,
|
|
1896
|
+
ready: Boolean(session.tree && session.layout),
|
|
1897
|
+
tool: 'geometra_page_model',
|
|
1898
|
+
options: {
|
|
1899
|
+
maxPrimaryActions: options?.maxPrimaryActions ?? 6,
|
|
1900
|
+
maxSectionsPerKind: options?.maxSectionsPerKind ?? 8,
|
|
1901
|
+
},
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1513
1904
|
function pageModelResponsePayload(session, options) {
|
|
1514
1905
|
const a11y = sessionA11y(session);
|
|
1515
1906
|
if (!a11y) {
|
|
@@ -1737,8 +2128,15 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
|
|
|
1737
2128
|
busyCount: signals.busyCount,
|
|
1738
2129
|
alertCount: signals.alerts.length,
|
|
1739
2130
|
invalidCount: signals.invalidFields.length,
|
|
1740
|
-
|
|
1741
|
-
|
|
2131
|
+
...(detail === 'verbose'
|
|
2132
|
+
? {
|
|
2133
|
+
alerts: signals.alerts,
|
|
2134
|
+
invalidFields: signals.invalidFields,
|
|
2135
|
+
}
|
|
2136
|
+
: {
|
|
2137
|
+
alerts: signals.alerts.slice(0, 2),
|
|
2138
|
+
invalidFields: signals.invalidFields.slice(0, 6),
|
|
2139
|
+
}),
|
|
1742
2140
|
};
|
|
1743
2141
|
}
|
|
1744
2142
|
function compactTextValue(value, inlineLimit = 48) {
|
|
@@ -1834,6 +2232,10 @@ function inferRevealStepBudget(target, viewport) {
|
|
|
1834
2232
|
return clamp(Math.max(6, Math.max(verticalSteps, horizontalSteps) + 1), 6, 48);
|
|
1835
2233
|
}
|
|
1836
2234
|
async function revealSemanticTarget(session, options) {
|
|
2235
|
+
const initialTreeReady = await ensureSessionUiTree(session, Math.max(4_000, options.timeoutMs));
|
|
2236
|
+
if (!initialTreeReady) {
|
|
2237
|
+
return { ok: false, error: 'Timed out waiting for the initial UI tree after connect.' };
|
|
2238
|
+
}
|
|
1837
2239
|
let attempts = 0;
|
|
1838
2240
|
let stepBudget = options.maxSteps;
|
|
1839
2241
|
while (attempts <= (stepBudget ?? 48)) {
|
|
@@ -1904,7 +2306,7 @@ async function resolveClickLocation(session, options) {
|
|
|
1904
2306
|
if (!hasNodeFilter(options.filter)) {
|
|
1905
2307
|
return {
|
|
1906
2308
|
ok: false,
|
|
1907
|
-
error: 'Provide x and y, or at least one semantic target filter (id, role, name, text, contextText, value, or state)',
|
|
2309
|
+
error: 'Provide x and y, or at least one semantic target filter (id, role, name, text, contextText, promptText, sectionText, itemText, value, or state)',
|
|
1908
2310
|
};
|
|
1909
2311
|
}
|
|
1910
2312
|
const revealed = await revealSemanticTarget(session, {
|
|
@@ -1936,6 +2338,18 @@ function compactNodeReference(node) {
|
|
|
1936
2338
|
...(node.name ? { name: node.name } : {}),
|
|
1937
2339
|
};
|
|
1938
2340
|
}
|
|
2341
|
+
function compactFormattedNode(node) {
|
|
2342
|
+
return {
|
|
2343
|
+
...compactNodeReference(node),
|
|
2344
|
+
...(node.context ? { context: node.context } : {}),
|
|
2345
|
+
...(node.value ? { value: node.value } : {}),
|
|
2346
|
+
center: node.center,
|
|
2347
|
+
bounds: node.bounds,
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
function detailText(summary, compact, detail) {
|
|
2351
|
+
return detail === 'terse' ? JSON.stringify(compact) : summary;
|
|
2352
|
+
}
|
|
1939
2353
|
function normalizeLookupKey(value) {
|
|
1940
2354
|
return value.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
1941
2355
|
}
|
|
@@ -2182,12 +2596,14 @@ function parseProxyFillAckResult(value) {
|
|
|
2182
2596
|
typeof candidate.busyCount !== 'number') {
|
|
2183
2597
|
return undefined;
|
|
2184
2598
|
}
|
|
2599
|
+
const invalidFields = Array.isArray(candidate.invalidFields) ? candidate.invalidFields : undefined;
|
|
2185
2600
|
return {
|
|
2186
2601
|
...(typeof candidate.pageUrl === 'string' ? { pageUrl: candidate.pageUrl } : {}),
|
|
2187
2602
|
invalidCount: candidate.invalidCount,
|
|
2188
2603
|
alertCount: candidate.alertCount,
|
|
2189
2604
|
dialogCount: candidate.dialogCount,
|
|
2190
2605
|
busyCount: candidate.busyCount,
|
|
2606
|
+
...(invalidFields && invalidFields.length > 0 ? { invalidFields } : {}),
|
|
2191
2607
|
};
|
|
2192
2608
|
}
|
|
2193
2609
|
function directLabelBatchFields(valuesByLabel) {
|
|
@@ -2202,6 +2618,44 @@ function directLabelBatchFields(valuesByLabel) {
|
|
|
2202
2618
|
}
|
|
2203
2619
|
return fields;
|
|
2204
2620
|
}
|
|
2621
|
+
async function tryBatchedResolvedFields(session, fields, detail) {
|
|
2622
|
+
let batchAckResult;
|
|
2623
|
+
try {
|
|
2624
|
+
const startRevision = session.updateRevision;
|
|
2625
|
+
const wait = await sendFillFields(session, fields);
|
|
2626
|
+
const ackResult = parseProxyFillAckResult(wait.result);
|
|
2627
|
+
batchAckResult = ackResult;
|
|
2628
|
+
if (ackResult && ackResult.invalidCount === 0) {
|
|
2629
|
+
return {
|
|
2630
|
+
ok: true,
|
|
2631
|
+
finalSource: 'proxy',
|
|
2632
|
+
final: ackResult,
|
|
2633
|
+
invalidRemaining: 0,
|
|
2634
|
+
};
|
|
2635
|
+
}
|
|
2636
|
+
await waitForDeferredBatchUpdate(session, startRevision, wait);
|
|
2637
|
+
await waitForBatchFieldReadback(session, fields);
|
|
2638
|
+
}
|
|
2639
|
+
catch (e) {
|
|
2640
|
+
if (canFallbackToSequentialFill(e))
|
|
2641
|
+
return { ok: false };
|
|
2642
|
+
throw e;
|
|
2643
|
+
}
|
|
2644
|
+
const after = sessionA11y(session);
|
|
2645
|
+
if (!after)
|
|
2646
|
+
return { ok: false };
|
|
2647
|
+
const signals = collectSessionSignals(after);
|
|
2648
|
+
const invalidRemaining = signals.invalidFields.length;
|
|
2649
|
+
if ((!batchAckResult || batchAckResult.invalidCount > 0) && invalidRemaining > 0) {
|
|
2650
|
+
return { ok: false };
|
|
2651
|
+
}
|
|
2652
|
+
return {
|
|
2653
|
+
ok: true,
|
|
2654
|
+
finalSource: 'session',
|
|
2655
|
+
final: sessionSignalsPayload(signals, detail),
|
|
2656
|
+
invalidRemaining,
|
|
2657
|
+
};
|
|
2658
|
+
}
|
|
2205
2659
|
async function waitForDeferredBatchUpdate(session, startRevision, wait) {
|
|
2206
2660
|
if (wait.status !== 'acknowledged' || session.updateRevision > startRevision)
|
|
2207
2661
|
return;
|
|
@@ -2242,6 +2696,22 @@ function batchFieldReadbackMatches(a11y, field) {
|
|
|
2242
2696
|
}
|
|
2243
2697
|
}
|
|
2244
2698
|
}
|
|
2699
|
+
function actionNeedsUiTree(action) {
|
|
2700
|
+
switch (action.type) {
|
|
2701
|
+
case 'wait_for':
|
|
2702
|
+
return true;
|
|
2703
|
+
case 'click':
|
|
2704
|
+
return action.x === undefined || action.y === undefined || Boolean(action.waitFor);
|
|
2705
|
+
default:
|
|
2706
|
+
return false;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
function canDeferInitialFrameForRunActions(actions) {
|
|
2710
|
+
const first = actions[0];
|
|
2711
|
+
if (!first)
|
|
2712
|
+
return false;
|
|
2713
|
+
return first.type === 'fill_fields';
|
|
2714
|
+
}
|
|
2245
2715
|
async function executeBatchAction(session, action, detail, includeSteps) {
|
|
2246
2716
|
switch (action.type) {
|
|
2247
2717
|
case 'click': {
|
|
@@ -2255,6 +2725,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
2255
2725
|
name: action.name,
|
|
2256
2726
|
text: action.text,
|
|
2257
2727
|
contextText: action.contextText,
|
|
2728
|
+
promptText: action.promptText,
|
|
2729
|
+
sectionText: action.sectionText,
|
|
2730
|
+
itemText: action.itemText,
|
|
2258
2731
|
value: action.value,
|
|
2259
2732
|
checked: action.checked,
|
|
2260
2733
|
disabled: action.disabled,
|
|
@@ -2286,6 +2759,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
2286
2759
|
name: action.waitFor.name,
|
|
2287
2760
|
text: action.waitFor.text,
|
|
2288
2761
|
contextText: action.waitFor.contextText,
|
|
2762
|
+
promptText: action.waitFor.promptText,
|
|
2763
|
+
sectionText: action.waitFor.sectionText,
|
|
2764
|
+
itemText: action.waitFor.itemText,
|
|
2289
2765
|
value: action.waitFor.value,
|
|
2290
2766
|
checked: action.waitFor.checked,
|
|
2291
2767
|
disabled: action.waitFor.disabled,
|
|
@@ -2442,6 +2918,9 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
2442
2918
|
name: action.name,
|
|
2443
2919
|
text: action.text,
|
|
2444
2920
|
contextText: action.contextText,
|
|
2921
|
+
promptText: action.promptText,
|
|
2922
|
+
sectionText: action.sectionText,
|
|
2923
|
+
itemText: action.itemText,
|
|
2445
2924
|
value: action.value,
|
|
2446
2925
|
checked: action.checked,
|
|
2447
2926
|
disabled: action.disabled,
|
|
@@ -2479,6 +2958,20 @@ async function executeBatchAction(session, action, detail, includeSteps) {
|
|
|
2479
2958
|
const resolvedFields = resolveFillFieldInputs(session, action.fields);
|
|
2480
2959
|
if (!resolvedFields.ok)
|
|
2481
2960
|
throw new Error(resolvedFields.error);
|
|
2961
|
+
if (!includeSteps) {
|
|
2962
|
+
const batched = await tryBatchedResolvedFields(session, resolvedFields.fields, detail);
|
|
2963
|
+
if (batched.ok) {
|
|
2964
|
+
return {
|
|
2965
|
+
summary: `Filled ${resolvedFields.fields.length} field(s) in one proxy batch.`,
|
|
2966
|
+
compact: {
|
|
2967
|
+
fieldCount: resolvedFields.fields.length,
|
|
2968
|
+
execution: 'batched',
|
|
2969
|
+
finalSource: batched.finalSource,
|
|
2970
|
+
final: batched.final,
|
|
2971
|
+
},
|
|
2972
|
+
};
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2482
2975
|
const steps = [];
|
|
2483
2976
|
for (let index = 0; index < resolvedFields.fields.length; index++) {
|
|
2484
2977
|
const field = resolvedFields.fields[index];
|
|
@@ -2571,8 +3064,24 @@ async function executeFillField(session, field, detail) {
|
|
|
2571
3064
|
}
|
|
2572
3065
|
}
|
|
2573
3066
|
}
|
|
2574
|
-
function ok(text) {
|
|
2575
|
-
|
|
3067
|
+
function ok(text, screenshot) {
|
|
3068
|
+
const content = [
|
|
3069
|
+
{ type: 'text', text },
|
|
3070
|
+
];
|
|
3071
|
+
if (screenshot) {
|
|
3072
|
+
content.push({ type: 'image', data: screenshot, mimeType: 'image/png' });
|
|
3073
|
+
}
|
|
3074
|
+
return { content };
|
|
3075
|
+
}
|
|
3076
|
+
async function captureScreenshotBase64(session) {
|
|
3077
|
+
try {
|
|
3078
|
+
const wait = await sendScreenshot(session);
|
|
3079
|
+
const result = wait.result;
|
|
3080
|
+
return typeof result?.screenshot === 'string' ? result.screenshot : undefined;
|
|
3081
|
+
}
|
|
3082
|
+
catch {
|
|
3083
|
+
return undefined;
|
|
3084
|
+
}
|
|
2576
3085
|
}
|
|
2577
3086
|
function err(text) {
|
|
2578
3087
|
return { content: [{ type: 'text', text }], isError: true };
|
|
@@ -2673,10 +3182,10 @@ function sectionContext(root, node) {
|
|
|
2673
3182
|
}
|
|
2674
3183
|
return undefined;
|
|
2675
3184
|
}
|
|
2676
|
-
function nodeContextText(
|
|
2677
|
-
return [
|
|
3185
|
+
function nodeContextText(context) {
|
|
3186
|
+
return [context?.prompt, context?.section, context?.item].filter(Boolean).join(' | ') || undefined;
|
|
2678
3187
|
}
|
|
2679
|
-
function nodeMatchesFilter(node, filter,
|
|
3188
|
+
function nodeMatchesFilter(node, filter, context) {
|
|
2680
3189
|
if (filter.id && nodeIdForPath(node.path) !== filter.id)
|
|
2681
3190
|
return false;
|
|
2682
3191
|
if (filter.role && node.role !== filter.role)
|
|
@@ -2688,7 +3197,13 @@ function nodeMatchesFilter(node, filter, contextText) {
|
|
|
2688
3197
|
if (filter.text &&
|
|
2689
3198
|
!textMatches(`${node.name ?? ''} ${node.value ?? ''} ${node.validation?.error ?? ''} ${node.validation?.description ?? ''}`.trim(), filter.text))
|
|
2690
3199
|
return false;
|
|
2691
|
-
if (!textMatches(
|
|
3200
|
+
if (!textMatches(nodeContextText(context), filter.contextText))
|
|
3201
|
+
return false;
|
|
3202
|
+
if (!textMatches(context?.prompt, filter.promptText))
|
|
3203
|
+
return false;
|
|
3204
|
+
if (!textMatches(context?.section, filter.sectionText))
|
|
3205
|
+
return false;
|
|
3206
|
+
if (!textMatches(context?.item, filter.itemText))
|
|
2692
3207
|
return false;
|
|
2693
3208
|
if (filter.checked !== undefined && node.state?.checked !== filter.checked)
|
|
2694
3209
|
return false;
|
|
@@ -2711,8 +3226,10 @@ function nodeMatchesFilter(node, filter, contextText) {
|
|
|
2711
3226
|
export function findNodes(node, filter) {
|
|
2712
3227
|
const matches = [];
|
|
2713
3228
|
function walk(n) {
|
|
2714
|
-
const
|
|
2715
|
-
|
|
3229
|
+
const context = filter.contextText || filter.promptText || filter.sectionText || filter.itemText
|
|
3230
|
+
? nodeContextForNode(node, n)
|
|
3231
|
+
: undefined;
|
|
3232
|
+
if (nodeMatchesFilter(n, filter, context) && hasNodeFilter(filter))
|
|
2716
3233
|
matches.push(n);
|
|
2717
3234
|
for (const child of n.children)
|
|
2718
3235
|
walk(child);
|
|
@@ -2755,14 +3272,13 @@ function formatNode(node, root, viewport) {
|
|
|
2755
3272
|
: Math.round(Math.min(Math.max(node.bounds.y + node.bounds.height / 2, 0), viewport.height));
|
|
2756
3273
|
const revealDeltaX = Math.round(node.bounds.x + node.bounds.width / 2 - viewport.width / 2);
|
|
2757
3274
|
const revealDeltaY = Math.round(node.bounds.y + node.bounds.height / 2 - viewport.height / 2);
|
|
2758
|
-
const
|
|
2759
|
-
const section = sectionContext(root, node);
|
|
3275
|
+
const context = nodeContextForNode(root, node);
|
|
2760
3276
|
return {
|
|
2761
3277
|
id: nodeIdForPath(node.path),
|
|
2762
3278
|
role: node.role,
|
|
2763
3279
|
name: node.name,
|
|
2764
3280
|
...(node.value ? { value: node.value } : {}),
|
|
2765
|
-
...(
|
|
3281
|
+
...(context ? { context } : {}),
|
|
2766
3282
|
bounds: node.bounds,
|
|
2767
3283
|
visibleBounds: {
|
|
2768
3284
|
x: visibleLeft,
|