@geometra/mcp 1.19.23 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/server-batch-results.test.js +1 -1
- package/dist/server.js +135 -23
- package/dist/session.d.ts +8 -0
- package/dist/session.js +74 -1
- package/package.json +1 -1
|
@@ -281,7 +281,7 @@ describe('batch MCP result shaping', () => {
|
|
|
281
281
|
alertCount: 1,
|
|
282
282
|
invalidCount: 5,
|
|
283
283
|
});
|
|
284
|
-
expect(final.invalidFields.length).toBe(
|
|
284
|
+
expect(final.invalidFields.length).toBe(5);
|
|
285
285
|
expect(final.alerts.length).toBe(1);
|
|
286
286
|
});
|
|
287
287
|
it('uses the proxy batch path for fill_fields when step output is omitted', async () => {
|
package/dist/server.js
CHANGED
|
@@ -3,7 +3,7 @@ import { performance } from 'node:perf_hooks';
|
|
|
3
3
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { formatConnectFailureMessage, isHttpUrl, normalizeConnectTarget } from './connect-utils.js';
|
|
6
|
-
import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
|
|
6
|
+
import { connect, connectThroughProxy, disconnect, getSession, prewarmProxy, sendClick, sendFillFields, sendType, sendKey, sendFileUpload, sendFieldText, sendFieldChoice, sendListboxPick, sendSelectOption, sendSetChecked, sendWheel, sendScreenshot, buildA11yTree, buildCompactUiIndex, buildFormRequiredSnapshot, buildPageModel, buildFormSchemas, expandPageSection, buildUiDelta, hasUiDelta, nodeIdForPath, nodeContextForNode, summarizeCompactIndex, summarizePageModel, summarizeUiDelta, waitForUiCondition, } from './session.js';
|
|
7
7
|
function checkedStateInput() {
|
|
8
8
|
return z
|
|
9
9
|
.union([z.boolean(), z.literal('mixed')])
|
|
@@ -726,8 +726,14 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
726
726
|
.optional()
|
|
727
727
|
.default(false)
|
|
728
728
|
.describe('Include per-field step results in the JSON payload (default false for the smallest response)'),
|
|
729
|
+
resumeFromIndex: z
|
|
730
|
+
.number()
|
|
731
|
+
.int()
|
|
732
|
+
.min(0)
|
|
733
|
+
.optional()
|
|
734
|
+
.describe('Resume a partial fill from this field index (from a previous stoppedAt + 1). Skips already-filled fields.'),
|
|
729
735
|
detail: detailInput(),
|
|
730
|
-
}, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, detail }) => {
|
|
736
|
+
}, async ({ url, pageUrl, port, headless, width, height, slowMo, formId, valuesById, valuesByLabel, stopOnError, failOnInvalid, includeSteps, resumeFromIndex, detail }) => {
|
|
731
737
|
const directFields = !includeSteps && !formId && Object.keys(valuesById ?? {}).length === 0
|
|
732
738
|
? directLabelBatchFields(valuesByLabel)
|
|
733
739
|
: null;
|
|
@@ -876,7 +882,8 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
876
882
|
}
|
|
877
883
|
const steps = [];
|
|
878
884
|
let stoppedAt;
|
|
879
|
-
|
|
885
|
+
const startIndex = resumeFromIndex ?? 0;
|
|
886
|
+
for (let index = startIndex; index < planned.fields.length; index++) {
|
|
880
887
|
const field = planned.fields[index];
|
|
881
888
|
try {
|
|
882
889
|
const result = await executeFillField(session, field, detail);
|
|
@@ -900,15 +907,16 @@ Pass \`valuesById\` with field ids from \`geometra_form_schema\` for the most st
|
|
|
900
907
|
const errorCount = steps.length - successCount;
|
|
901
908
|
const payload = {
|
|
902
909
|
...connection,
|
|
903
|
-
completed: stoppedAt === undefined && steps.length === planned.fields.length,
|
|
910
|
+
completed: stoppedAt === undefined && (startIndex + steps.length) === planned.fields.length,
|
|
904
911
|
execution: 'sequential',
|
|
905
912
|
formId: schema.formId,
|
|
906
913
|
requestedValueCount: entryCount,
|
|
907
914
|
fieldCount: planned.fields.length,
|
|
908
915
|
successCount,
|
|
909
916
|
errorCount,
|
|
917
|
+
...(startIndex > 0 ? { resumedFromIndex: startIndex } : {}),
|
|
910
918
|
...(includeSteps ? { steps } : {}),
|
|
911
|
-
...(stoppedAt !== undefined ? { stoppedAt } : {}),
|
|
919
|
+
...(stoppedAt !== undefined ? { stoppedAt, resumeFromIndex: stoppedAt + 1 } : {}),
|
|
912
920
|
...(signals ? { final: sessionSignalsPayload(signals, detail) } : {}),
|
|
913
921
|
};
|
|
914
922
|
if (failOnInvalid && invalidRemaining > 0) {
|
|
@@ -952,6 +960,7 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
952
960
|
const connection = autoConnectionPayload(resolved);
|
|
953
961
|
const steps = [];
|
|
954
962
|
let stoppedAt;
|
|
963
|
+
const batchStartedAt = performance.now();
|
|
955
964
|
for (let index = 0; index < actions.length; index++) {
|
|
956
965
|
const action = actions[index];
|
|
957
966
|
const startedAt = performance.now();
|
|
@@ -964,32 +973,46 @@ Supported step types: \`click\`, \`type\`, \`key\`, \`upload_files\`, \`pick_lis
|
|
|
964
973
|
}
|
|
965
974
|
const result = await executeBatchAction(session, action, detail, includeSteps);
|
|
966
975
|
const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
|
|
976
|
+
const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
|
|
977
|
+
const stepSignals = includeSteps ? (() => {
|
|
978
|
+
const a = sessionA11y(session);
|
|
979
|
+
if (!a)
|
|
980
|
+
return undefined;
|
|
981
|
+
const s = collectSessionSignals(a);
|
|
982
|
+
return { invalidCount: s.invalidFields.length, alertCount: s.alerts.length, dialogCount: s.dialogCount, busyCount: s.busyCount };
|
|
983
|
+
})() : undefined;
|
|
967
984
|
steps.push(detail === 'verbose'
|
|
968
985
|
? {
|
|
969
986
|
index,
|
|
970
987
|
type: action.type,
|
|
971
988
|
ok: true,
|
|
972
989
|
elapsedMs,
|
|
990
|
+
cumulativeMs,
|
|
973
991
|
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
974
992
|
summary: result.summary,
|
|
993
|
+
...(stepSignals ? { signals: stepSignals } : {}),
|
|
975
994
|
}
|
|
976
995
|
: {
|
|
977
996
|
index,
|
|
978
997
|
type: action.type,
|
|
979
998
|
ok: true,
|
|
980
999
|
elapsedMs,
|
|
1000
|
+
cumulativeMs,
|
|
981
1001
|
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
982
1002
|
...result.compact,
|
|
1003
|
+
...(stepSignals ? { signals: stepSignals } : {}),
|
|
983
1004
|
});
|
|
984
1005
|
}
|
|
985
1006
|
catch (e) {
|
|
986
1007
|
const message = e instanceof Error ? e.message : String(e);
|
|
987
1008
|
const elapsedMs = Number((performance.now() - startedAt).toFixed(1));
|
|
1009
|
+
const cumulativeMs = Number((performance.now() - batchStartedAt).toFixed(1));
|
|
988
1010
|
steps.push({
|
|
989
1011
|
index,
|
|
990
1012
|
type: action.type,
|
|
991
1013
|
ok: false,
|
|
992
1014
|
elapsedMs,
|
|
1015
|
+
cumulativeMs,
|
|
993
1016
|
...(uiTreeWaitMs > 0 ? { uiTreeWaitMs: Number(uiTreeWaitMs.toFixed(1)) } : {}),
|
|
994
1017
|
error: message,
|
|
995
1018
|
});
|
|
@@ -1041,7 +1064,12 @@ Use this first on normal HTML pages when you want to understand the page shape w
|
|
|
1041
1064
|
.optional()
|
|
1042
1065
|
.default(8)
|
|
1043
1066
|
.describe('Cap returned landmarks/forms/dialogs/lists per kind (default 8).'),
|
|
1044
|
-
|
|
1067
|
+
includeScreenshot: z
|
|
1068
|
+
.boolean()
|
|
1069
|
+
.optional()
|
|
1070
|
+
.default(false)
|
|
1071
|
+
.describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy. Use when geometry alone is ambiguous (icon-only buttons, visual styling cues).'),
|
|
1072
|
+
}, async ({ maxPrimaryActions, maxSectionsPerKind, includeScreenshot }) => {
|
|
1045
1073
|
const session = getSession();
|
|
1046
1074
|
if (!session)
|
|
1047
1075
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1049,7 +1077,8 @@ Use this first on normal HTML pages when you want to understand the page shape w
|
|
|
1049
1077
|
if (!a11y)
|
|
1050
1078
|
return err('No UI tree available');
|
|
1051
1079
|
const model = buildPageModel(a11y, { maxPrimaryActions, maxSectionsPerKind });
|
|
1052
|
-
|
|
1080
|
+
const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
|
|
1081
|
+
return ok(JSON.stringify(model), screenshot);
|
|
1053
1082
|
});
|
|
1054
1083
|
server.tool('geometra_form_schema', `Get a compact, fill-oriented schema for forms on the page. This is the preferred discovery step before geometra_fill_form.
|
|
1055
1084
|
|
|
@@ -1369,6 +1398,8 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
1369
1398
|
.describe('Upload strategy (default auto)'),
|
|
1370
1399
|
dropX: z.number().optional().describe('Drop target X (viewport) for strategy drop'),
|
|
1371
1400
|
dropY: z.number().optional().describe('Drop target Y (viewport) for strategy drop'),
|
|
1401
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated file inputs'),
|
|
1402
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated file inputs'),
|
|
1372
1403
|
timeoutMs: z
|
|
1373
1404
|
.number()
|
|
1374
1405
|
.int()
|
|
@@ -1377,7 +1408,7 @@ Strategies: **auto** (default) tries chooser click if x,y given, else a labeled
|
|
|
1377
1408
|
.optional()
|
|
1378
1409
|
.describe('Optional action wait timeout (resume parsing / SPA upload flows often need longer than a normal click)'),
|
|
1379
1410
|
detail: detailInput(),
|
|
1380
|
-
}, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, timeoutMs, detail }) => {
|
|
1411
|
+
}, async ({ paths, x, y, fieldLabel, exact, strategy, dropX, dropY, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1381
1412
|
const session = getSession();
|
|
1382
1413
|
if (!session)
|
|
1383
1414
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1411,6 +1442,8 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
1411
1442
|
openX: z.number().optional().describe('Click to open dropdown'),
|
|
1412
1443
|
openY: z.number().optional().describe('Click to open dropdown'),
|
|
1413
1444
|
fieldLabel: z.string().optional().describe('Field label of the dropdown/combobox to open semantically (e.g. "Location")'),
|
|
1445
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated dropdowns with the same label'),
|
|
1446
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated dropdowns'),
|
|
1414
1447
|
query: z.string().optional().describe('Optional text to type into a searchable combobox before selecting'),
|
|
1415
1448
|
timeoutMs: z
|
|
1416
1449
|
.number()
|
|
@@ -1420,7 +1453,7 @@ Pass \`fieldLabel\` to open a labeled dropdown semantically instead of relying o
|
|
|
1420
1453
|
.optional()
|
|
1421
1454
|
.describe('Optional action wait timeout for slow dropdowns / remote search results'),
|
|
1422
1455
|
detail: detailInput(),
|
|
1423
|
-
}, async ({ label, exact, openX, openY, fieldLabel, query, timeoutMs, detail }) => {
|
|
1456
|
+
}, async ({ label, exact, openX, openY, fieldLabel, contextText, sectionText, query, timeoutMs, detail }) => {
|
|
1424
1457
|
const session = getSession();
|
|
1425
1458
|
if (!session)
|
|
1426
1459
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1459,6 +1492,8 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
|
|
|
1459
1492
|
value: z.string().optional().describe('Option value= attribute'),
|
|
1460
1493
|
label: z.string().optional().describe('Visible option label (substring match)'),
|
|
1461
1494
|
index: z.number().int().min(0).optional().describe('Zero-based option index'),
|
|
1495
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated selects'),
|
|
1496
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated selects'),
|
|
1462
1497
|
timeoutMs: z
|
|
1463
1498
|
.number()
|
|
1464
1499
|
.int()
|
|
@@ -1467,7 +1502,7 @@ Custom React/Vue dropdowns are not supported here — use \`geometra_pick_listbo
|
|
|
1467
1502
|
.optional()
|
|
1468
1503
|
.describe('Optional action wait timeout'),
|
|
1469
1504
|
detail: detailInput(),
|
|
1470
|
-
}, async ({ x, y, value, label, index, timeoutMs, detail }) => {
|
|
1505
|
+
}, async ({ x, y, value, label, index, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1471
1506
|
const session = getSession();
|
|
1472
1507
|
if (!session)
|
|
1473
1508
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1497,6 +1532,8 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1497
1532
|
checked: z.boolean().optional().default(true).describe('Desired checked state (radios only support true)'),
|
|
1498
1533
|
exact: z.boolean().optional().describe('Exact label match'),
|
|
1499
1534
|
controlType: z.enum(['checkbox', 'radio']).optional().describe('Limit matching to checkbox or radio'),
|
|
1535
|
+
contextText: z.string().optional().describe('Ancestor / prompt text to disambiguate repeated checkboxes/radios'),
|
|
1536
|
+
sectionText: z.string().optional().describe('Containing section text to disambiguate repeated checkboxes/radios'),
|
|
1500
1537
|
timeoutMs: z
|
|
1501
1538
|
.number()
|
|
1502
1539
|
.int()
|
|
@@ -1505,7 +1542,7 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1505
1542
|
.optional()
|
|
1506
1543
|
.describe('Optional action wait timeout'),
|
|
1507
1544
|
detail: detailInput(),
|
|
1508
|
-
}, async ({ label, checked, exact, controlType, timeoutMs, detail }) => {
|
|
1545
|
+
}, async ({ label, checked, exact, controlType, contextText: _contextText, sectionText: _sectionText, timeoutMs, detail }) => {
|
|
1509
1546
|
const session = getSession();
|
|
1510
1547
|
if (!session)
|
|
1511
1548
|
return err('Not connected. Call geometra_connect first.');
|
|
@@ -1557,6 +1594,57 @@ Prefer this over raw coordinate clicks for custom forms that keep the real input
|
|
|
1557
1594
|
return err(e.message);
|
|
1558
1595
|
}
|
|
1559
1596
|
});
|
|
1597
|
+
// ── list items (virtualized list pagination) ─────────────────
|
|
1598
|
+
server.tool('geometra_list_items', `Auto-scroll a virtualized or long list and collect all visible items across scroll positions. Requires \`@geometra/proxy\`.
|
|
1599
|
+
|
|
1600
|
+
Use this for dropdowns, location pickers, or any scrollable list where items are rendered on demand. Scrolls down in steps, collecting new items each time, until no new items appear or the cap is reached.`, {
|
|
1601
|
+
listId: z.string().optional().describe('Stable section id from geometra_page_model (e.g. ls:2.1) to scope item collection'),
|
|
1602
|
+
role: z.string().optional().describe('Role filter for list items (default: listitem)'),
|
|
1603
|
+
scrollX: z.number().optional().describe('X coordinate to position mouse for scrolling (default: viewport center)'),
|
|
1604
|
+
scrollY: z.number().optional().describe('Y coordinate to position mouse for scrolling (default: viewport center)'),
|
|
1605
|
+
maxItems: z.number().int().min(1).max(500).optional().default(100).describe('Cap collected items (default 100)'),
|
|
1606
|
+
maxScrollSteps: z.number().int().min(1).max(50).optional().default(20).describe('Max scroll steps before stopping (default 20)'),
|
|
1607
|
+
scrollDelta: z.number().optional().default(300).describe('Vertical scroll delta per step (default 300)'),
|
|
1608
|
+
}, async ({ listId: _listId, role, scrollX, scrollY, maxItems, maxScrollSteps, scrollDelta }) => {
|
|
1609
|
+
const session = getSession();
|
|
1610
|
+
if (!session)
|
|
1611
|
+
return err('Not connected. Call geometra_connect first.');
|
|
1612
|
+
const itemRole = role ?? 'listitem';
|
|
1613
|
+
const collected = new Map();
|
|
1614
|
+
const cx = scrollX ?? 400;
|
|
1615
|
+
const cy = scrollY ?? 400;
|
|
1616
|
+
for (let step = 0; step < maxScrollSteps; step++) {
|
|
1617
|
+
const a11y = await sessionA11yWhenReady(session);
|
|
1618
|
+
if (!a11y)
|
|
1619
|
+
break;
|
|
1620
|
+
const items = findNodes(a11y, { role: itemRole });
|
|
1621
|
+
let newCount = 0;
|
|
1622
|
+
for (const item of items) {
|
|
1623
|
+
const id = nodeIdForPath(item.path);
|
|
1624
|
+
if (!collected.has(id)) {
|
|
1625
|
+
collected.set(id, {
|
|
1626
|
+
...(item.name ? { name: item.name } : {}),
|
|
1627
|
+
...(item.value ? { value: item.value } : {}),
|
|
1628
|
+
});
|
|
1629
|
+
newCount++;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
if (collected.size >= maxItems || newCount === 0)
|
|
1633
|
+
break;
|
|
1634
|
+
try {
|
|
1635
|
+
await sendWheel(session, scrollDelta, { x: cx, y: cy }, 1_000);
|
|
1636
|
+
}
|
|
1637
|
+
catch {
|
|
1638
|
+
break;
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
const items = [...collected.entries()].slice(0, maxItems).map(([id, data]) => ({ id, ...data }));
|
|
1642
|
+
return ok(JSON.stringify({
|
|
1643
|
+
itemCount: items.length,
|
|
1644
|
+
items,
|
|
1645
|
+
truncated: collected.size > maxItems,
|
|
1646
|
+
}));
|
|
1647
|
+
});
|
|
1560
1648
|
// ── snapshot ─────────────────────────────────────────────────
|
|
1561
1649
|
server.tool('geometra_snapshot', `Get the current UI as JSON. Default **compact** view: flat list of viewport-visible actionable nodes plus a few pinned context anchors (for example tab strips / form roots) and root context like URL, scroll, and focus — far fewer tokens than a full nested tree. Use **full** for complete nested a11y + every wrapper when debugging layout. Use **form-required** to list required fields across forms, including offscreen ones, with bounds + visibility + scroll hints for long application flows.
|
|
1562
1650
|
|
|
@@ -1577,15 +1665,21 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1577
1665
|
formId: z.string().optional().describe('Optional form id from geometra_form_schema / geometra_page_model when view=form-required'),
|
|
1578
1666
|
maxFields: z.number().int().min(1).max(200).optional().default(80).describe('Per-form field cap when view=form-required'),
|
|
1579
1667
|
includeOptions: z.boolean().optional().default(false).describe('Include explicit choice option labels when view=form-required'),
|
|
1580
|
-
|
|
1668
|
+
includeScreenshot: z
|
|
1669
|
+
.boolean()
|
|
1670
|
+
.optional()
|
|
1671
|
+
.default(false)
|
|
1672
|
+
.describe('Attach a base64 PNG viewport screenshot. Requires @geometra/proxy.'),
|
|
1673
|
+
}, async ({ view, maxNodes, formId, maxFields, includeOptions, includeScreenshot }) => {
|
|
1581
1674
|
const session = getSession();
|
|
1582
1675
|
if (!session)
|
|
1583
1676
|
return err('Not connected. Call geometra_connect first.');
|
|
1584
1677
|
const a11y = await sessionA11yWhenReady(session);
|
|
1585
1678
|
if (!a11y)
|
|
1586
1679
|
return err('No UI tree available');
|
|
1680
|
+
const screenshot = includeScreenshot ? await captureScreenshotBase64(session) : undefined;
|
|
1587
1681
|
if (view === 'full') {
|
|
1588
|
-
return ok(JSON.stringify(a11y, null, 2));
|
|
1682
|
+
return ok(JSON.stringify(a11y, null, 2), screenshot);
|
|
1589
1683
|
}
|
|
1590
1684
|
if (view === 'form-required') {
|
|
1591
1685
|
const payload = {
|
|
@@ -1598,7 +1692,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1598
1692
|
includeContext: 'auto',
|
|
1599
1693
|
}),
|
|
1600
1694
|
};
|
|
1601
|
-
return ok(JSON.stringify(payload));
|
|
1695
|
+
return ok(JSON.stringify(payload), screenshot);
|
|
1602
1696
|
}
|
|
1603
1697
|
const { nodes, truncated, context } = buildCompactUiIndex(a11y, { maxNodes });
|
|
1604
1698
|
const payload = {
|
|
@@ -1608,7 +1702,7 @@ JSON is minified in compact view to save tokens. For a summary-first overview, u
|
|
|
1608
1702
|
nodes,
|
|
1609
1703
|
truncated,
|
|
1610
1704
|
};
|
|
1611
|
-
return ok(JSON.stringify(payload));
|
|
1705
|
+
return ok(JSON.stringify(payload), screenshot);
|
|
1612
1706
|
});
|
|
1613
1707
|
// ── layout ───────────────────────────────────────────────────
|
|
1614
1708
|
server.tool('geometra_layout', `Get the raw computed layout geometry — the exact {x, y, width, height} for every node in the UI tree. This is the lowest-level view, useful for pixel-precise assertions in tests.
|
|
@@ -1720,8 +1814,10 @@ function packedFormSchemas(forms) {
|
|
|
1720
1814
|
...(field.valueLength !== undefined ? { vl: field.valueLength } : {}),
|
|
1721
1815
|
...(field.checked !== undefined ? { c: field.checked ? 1 : 0 } : {}),
|
|
1722
1816
|
...(field.values && field.values.length > 0 ? { vs: field.values } : {}),
|
|
1817
|
+
...(field.aliases ? { al: field.aliases } : {}),
|
|
1723
1818
|
...(field.context ? { x: field.context } : {}),
|
|
1724
1819
|
})),
|
|
1820
|
+
...(form.sections ? { s: form.sections.map(s => ({ n: s.name, fi: s.fieldIds })) } : {}),
|
|
1725
1821
|
}));
|
|
1726
1822
|
}
|
|
1727
1823
|
function formSchemaResponsePayload(session, opts) {
|
|
@@ -2037,12 +2133,10 @@ function sessionSignalsPayload(signals, detail = 'minimal') {
|
|
|
2037
2133
|
alerts: signals.alerts,
|
|
2038
2134
|
invalidFields: signals.invalidFields,
|
|
2039
2135
|
}
|
|
2040
|
-
:
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
}
|
|
2045
|
-
: {}),
|
|
2136
|
+
: {
|
|
2137
|
+
alerts: signals.alerts.slice(0, 2),
|
|
2138
|
+
invalidFields: signals.invalidFields.slice(0, 6),
|
|
2139
|
+
}),
|
|
2046
2140
|
};
|
|
2047
2141
|
}
|
|
2048
2142
|
function compactTextValue(value, inlineLimit = 48) {
|
|
@@ -2502,12 +2596,14 @@ function parseProxyFillAckResult(value) {
|
|
|
2502
2596
|
typeof candidate.busyCount !== 'number') {
|
|
2503
2597
|
return undefined;
|
|
2504
2598
|
}
|
|
2599
|
+
const invalidFields = Array.isArray(candidate.invalidFields) ? candidate.invalidFields : undefined;
|
|
2505
2600
|
return {
|
|
2506
2601
|
...(typeof candidate.pageUrl === 'string' ? { pageUrl: candidate.pageUrl } : {}),
|
|
2507
2602
|
invalidCount: candidate.invalidCount,
|
|
2508
2603
|
alertCount: candidate.alertCount,
|
|
2509
2604
|
dialogCount: candidate.dialogCount,
|
|
2510
2605
|
busyCount: candidate.busyCount,
|
|
2606
|
+
...(invalidFields && invalidFields.length > 0 ? { invalidFields } : {}),
|
|
2511
2607
|
};
|
|
2512
2608
|
}
|
|
2513
2609
|
function directLabelBatchFields(valuesByLabel) {
|
|
@@ -2968,8 +3064,24 @@ async function executeFillField(session, field, detail) {
|
|
|
2968
3064
|
}
|
|
2969
3065
|
}
|
|
2970
3066
|
}
|
|
2971
|
-
function ok(text) {
|
|
2972
|
-
|
|
3067
|
+
function ok(text, screenshot) {
|
|
3068
|
+
const content = [
|
|
3069
|
+
{ type: 'text', text },
|
|
3070
|
+
];
|
|
3071
|
+
if (screenshot) {
|
|
3072
|
+
content.push({ type: 'image', data: screenshot, mimeType: 'image/png' });
|
|
3073
|
+
}
|
|
3074
|
+
return { content };
|
|
3075
|
+
}
|
|
3076
|
+
async function captureScreenshotBase64(session) {
|
|
3077
|
+
try {
|
|
3078
|
+
const wait = await sendScreenshot(session);
|
|
3079
|
+
const result = wait.result;
|
|
3080
|
+
return typeof result?.screenshot === 'string' ? result.screenshot : undefined;
|
|
3081
|
+
}
|
|
3082
|
+
catch {
|
|
3083
|
+
return undefined;
|
|
3084
|
+
}
|
|
2973
3085
|
}
|
|
2974
3086
|
function err(text) {
|
|
2975
3087
|
return { content: [{ type: 'text', text }], isError: true };
|
package/dist/session.d.ts
CHANGED
|
@@ -263,8 +263,13 @@ export interface FormSchemaField {
|
|
|
263
263
|
values?: string[];
|
|
264
264
|
optionCount?: number;
|
|
265
265
|
options?: string[];
|
|
266
|
+
aliases?: Record<string, string[]>;
|
|
266
267
|
context?: NodeContextModel;
|
|
267
268
|
}
|
|
269
|
+
export interface FormSchemaSection {
|
|
270
|
+
name: string;
|
|
271
|
+
fieldIds: string[];
|
|
272
|
+
}
|
|
268
273
|
export interface FormSchemaModel {
|
|
269
274
|
formId: string;
|
|
270
275
|
name?: string;
|
|
@@ -272,6 +277,7 @@ export interface FormSchemaModel {
|
|
|
272
277
|
requiredCount: number;
|
|
273
278
|
invalidCount: number;
|
|
274
279
|
fields: FormSchemaField[];
|
|
280
|
+
sections?: FormSchemaSection[];
|
|
275
281
|
}
|
|
276
282
|
export interface FormRequiredFieldSnapshot extends FormSchemaField {
|
|
277
283
|
bounds: {
|
|
@@ -531,6 +537,8 @@ export declare function sendWheel(session: Session, deltaY: number, opts?: {
|
|
|
531
537
|
x?: number;
|
|
532
538
|
y?: number;
|
|
533
539
|
}, timeoutMs?: number): Promise<UpdateWaitResult>;
|
|
540
|
+
/** Capture a viewport screenshot from the proxy (base64 PNG). */
|
|
541
|
+
export declare function sendScreenshot(session: Session, timeoutMs?: number): Promise<UpdateWaitResult>;
|
|
534
542
|
/** Navigate the proxy page to a new URL while keeping the browser process alive. */
|
|
535
543
|
export declare function sendNavigate(session: Session, url: string, timeoutMs?: number): Promise<UpdateWaitResult>;
|
|
536
544
|
/**
|
package/dist/session.js
CHANGED
|
@@ -875,6 +875,10 @@ export function sendWheel(session, deltaY, opts, timeoutMs) {
|
|
|
875
875
|
...(opts?.y !== undefined ? { y: opts.y } : {}),
|
|
876
876
|
}, timeoutMs);
|
|
877
877
|
}
|
|
878
|
+
/** Capture a viewport screenshot from the proxy (base64 PNG). */
|
|
879
|
+
export function sendScreenshot(session, timeoutMs = 10_000) {
|
|
880
|
+
return sendAndWaitForUpdate(session, { type: 'screenshot' }, timeoutMs);
|
|
881
|
+
}
|
|
878
882
|
/** Navigate the proxy page to a new URL while keeping the browser process alive. */
|
|
879
883
|
export function sendNavigate(session, url, timeoutMs = 15_000) {
|
|
880
884
|
return sendAndWaitForUpdate(session, {
|
|
@@ -1566,6 +1570,33 @@ function groupedChoiceForNode(root, formNode, seed) {
|
|
|
1566
1570
|
const controls = sortByBounds(collectDescendants(formNode, matchesPrompt));
|
|
1567
1571
|
return controls.length >= 2 ? { container: formNode, prompt, controls } : null;
|
|
1568
1572
|
}
|
|
1573
|
+
const SEMANTIC_ALIAS_GROUPS = [
|
|
1574
|
+
{ triggers: ['yes', 'true'], aliases: ['yes', 'true', 'agree', 'agreed', 'accept', 'accepted', 'consent', 'acknowledge', 'opt in'] },
|
|
1575
|
+
{ triggers: ['no', 'false'], aliases: ['no', 'false', 'decline', 'declined', 'disagree', 'deny', 'opt out', 'prefer not'] },
|
|
1576
|
+
{ triggers: ['decline'], aliases: ['decline', 'prefer not', 'opt out', 'do not'] },
|
|
1577
|
+
{ triggers: ['atx', 'austin'], aliases: ['atx', 'austin', 'austin tx', 'austin texas'] },
|
|
1578
|
+
{ triggers: ['nyc', 'new york'], aliases: ['nyc', 'new york', 'new york ny'] },
|
|
1579
|
+
{ triggers: ['sf', 'san francisco'], aliases: ['sf', 'san francisco', 'san francisco ca'] },
|
|
1580
|
+
{ triggers: ['la', 'los angeles'], aliases: ['la', 'los angeles', 'los angeles ca'] },
|
|
1581
|
+
{ triggers: ['dc', 'washington dc'], aliases: ['dc', 'washington dc', 'washington d c'] },
|
|
1582
|
+
{ triggers: ['us', 'usa', 'united states'], aliases: ['us', 'usa', 'united states'] },
|
|
1583
|
+
];
|
|
1584
|
+
function computeOptionAliases(options) {
|
|
1585
|
+
const result = {};
|
|
1586
|
+
for (const option of options) {
|
|
1587
|
+
const normalized = option.toLowerCase().trim();
|
|
1588
|
+
for (const group of SEMANTIC_ALIAS_GROUPS) {
|
|
1589
|
+
if (group.triggers.some(t => normalized === t || normalized.includes(t))) {
|
|
1590
|
+
const relevant = group.aliases.filter(a => a !== normalized);
|
|
1591
|
+
if (relevant.length > 0) {
|
|
1592
|
+
result[option] = relevant;
|
|
1593
|
+
break;
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
return Object.keys(result).length > 0 ? result : undefined;
|
|
1599
|
+
}
|
|
1569
1600
|
function simpleSchemaField(root, node) {
|
|
1570
1601
|
const context = nodeContextForNode(root, node);
|
|
1571
1602
|
const label = fieldLabel(node) ?? sanitizeInlineName(node.name, 80) ?? context?.prompt;
|
|
@@ -1617,6 +1648,7 @@ function groupedSchemaField(root, grouped) {
|
|
|
1617
1648
|
}),
|
|
1618
1649
|
optionCount: options.length,
|
|
1619
1650
|
options,
|
|
1651
|
+
...(computeOptionAliases(options) ? { aliases: computeOptionAliases(options) } : {}),
|
|
1620
1652
|
...(compactSchemaContext(context, grouped.prompt) ? { context: compactSchemaContext(context, grouped.prompt) } : {}),
|
|
1621
1653
|
};
|
|
1622
1654
|
}
|
|
@@ -1637,6 +1669,41 @@ function toggleSchemaField(root, node) {
|
|
|
1637
1669
|
...(compactSchemaContext(context, label) ? { context: compactSchemaContext(context, label) } : {}),
|
|
1638
1670
|
};
|
|
1639
1671
|
}
|
|
1672
|
+
function detectFormSections(formNode, fields) {
|
|
1673
|
+
const sectionRoles = new Set(['group', 'region']);
|
|
1674
|
+
const sectionNodes = [];
|
|
1675
|
+
function walk(node) {
|
|
1676
|
+
if (sectionRoles.has(node.role) && node.name && node.path.length > formNode.path.length) {
|
|
1677
|
+
sectionNodes.push({ name: node.name, path: node.path });
|
|
1678
|
+
}
|
|
1679
|
+
for (const child of node.children)
|
|
1680
|
+
walk(child);
|
|
1681
|
+
}
|
|
1682
|
+
walk(formNode);
|
|
1683
|
+
if (sectionNodes.length === 0)
|
|
1684
|
+
return [];
|
|
1685
|
+
const fieldIdToPath = new Map();
|
|
1686
|
+
for (const field of fields) {
|
|
1687
|
+
const parsed = parseFormFieldId(field.id);
|
|
1688
|
+
if (parsed)
|
|
1689
|
+
fieldIdToPath.set(field.id, parsed);
|
|
1690
|
+
}
|
|
1691
|
+
const sections = [];
|
|
1692
|
+
for (const sec of sectionNodes) {
|
|
1693
|
+
const fieldIds = fields
|
|
1694
|
+
.filter(field => {
|
|
1695
|
+
const fieldPath = fieldIdToPath.get(field.id);
|
|
1696
|
+
if (!fieldPath || fieldPath.length <= sec.path.length)
|
|
1697
|
+
return false;
|
|
1698
|
+
return sec.path.every((v, i) => fieldPath[i] === v);
|
|
1699
|
+
})
|
|
1700
|
+
.map(field => field.id);
|
|
1701
|
+
if (fieldIds.length > 0) {
|
|
1702
|
+
sections.push({ name: sec.name, fieldIds });
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
return sections;
|
|
1706
|
+
}
|
|
1640
1707
|
function buildFormSchemaForNode(root, formNode, options) {
|
|
1641
1708
|
const candidates = sortByBounds(collectDescendants(formNode, candidate => candidate.role === 'textbox' ||
|
|
1642
1709
|
candidate.role === 'combobox' ||
|
|
@@ -1690,6 +1757,10 @@ function buildFormSchemaForNode(root, formNode, options) {
|
|
|
1690
1757
|
requiredCount: compactFields.filter(field => field.required).length,
|
|
1691
1758
|
invalidCount: compactFields.filter(field => field.invalid).length,
|
|
1692
1759
|
fields: pageFields,
|
|
1760
|
+
...(() => {
|
|
1761
|
+
const sections = detectFormSections(formNode, pageFields);
|
|
1762
|
+
return sections.length > 0 ? { sections } : {};
|
|
1763
|
+
})(),
|
|
1693
1764
|
};
|
|
1694
1765
|
}
|
|
1695
1766
|
function trimSchemaFieldContexts(fields) {
|
|
@@ -1712,8 +1783,10 @@ function presentFormSchemaFields(fields, options) {
|
|
|
1712
1783
|
const next = { ...field };
|
|
1713
1784
|
if (booleanChoice)
|
|
1714
1785
|
next.booleanChoice = true;
|
|
1715
|
-
if (!includeOptions)
|
|
1786
|
+
if (!includeOptions) {
|
|
1716
1787
|
delete next.options;
|
|
1788
|
+
delete next.aliases;
|
|
1789
|
+
}
|
|
1717
1790
|
if (includeContext === 'none') {
|
|
1718
1791
|
delete next.context;
|
|
1719
1792
|
return next;
|
package/package.json
CHANGED