chrometools-mcp 3.5.6 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +92 -5
- package/browser/browser-manager.js +26 -2
- package/browser/page-manager.js +89 -0
- package/element-finder-utils.js +38 -0
- package/index.js +208 -56
- package/package.json +1 -1
- package/server/tool-definitions.js +21 -0
- package/server/tool-groups.js +1 -1
- package/server/tool-schemas.js +9 -0
- package/utils/actions/click-action.js +38 -1
- package/utils/element-actions.js +8 -5
- package/utils/platform-utils.js +17 -2
- package/utils/post-click-diagnostics.js +21 -4
- package/specs/SEGM-537-UNBLOCKERS_PROGRESS.md +0 -94
- package/specs/SEGM-537-UNBLOCKERS_SPEC.md +0 -187
package/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
import {Server} from "@modelcontextprotocol/sdk/server/index.js";
|
|
4
4
|
import {StdioServerTransport} from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
@@ -37,7 +37,12 @@ import {
|
|
|
37
37
|
getAllPages,
|
|
38
38
|
switchToPage,
|
|
39
39
|
connectToTabByUrl,
|
|
40
|
-
getNetworkCDPClient
|
|
40
|
+
getNetworkCDPClient,
|
|
41
|
+
getTargetFrame,
|
|
42
|
+
setActiveFrame,
|
|
43
|
+
clearActiveFrame,
|
|
44
|
+
getActiveFrameMatcher,
|
|
45
|
+
listFramesForPage
|
|
41
46
|
} from './browser/page-manager.js';
|
|
42
47
|
|
|
43
48
|
// Import image processing utilities
|
|
@@ -535,13 +540,19 @@ async function executeToolInternal(name, args) {
|
|
|
535
540
|
// Get identifier (id or selector)
|
|
536
541
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
537
542
|
|
|
543
|
+
// Resolve/query inside the active frame (P0-1). The ElementHandle from
|
|
544
|
+
// ctx.$() runs in the frame's context, so element.click()/.evaluate()
|
|
545
|
+
// in executeClickAction operate inside the iframe. Page-level diagnostics
|
|
546
|
+
// (network/screenshot) stay on `page`.
|
|
547
|
+
const ctx = await getTargetFrame(page);
|
|
548
|
+
|
|
538
549
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
539
|
-
const resolved = await resolveSelector(
|
|
550
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
540
551
|
if (!resolved.found) {
|
|
541
552
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
542
553
|
}
|
|
543
554
|
|
|
544
|
-
const element = await
|
|
555
|
+
const element = await ctx.$(resolved.selector);
|
|
545
556
|
if (!element) {
|
|
546
557
|
throw new Error(`Element not found: ${identifier}`);
|
|
547
558
|
}
|
|
@@ -550,7 +561,7 @@ async function executeToolInternal(name, args) {
|
|
|
550
561
|
// currently in the tree so we can diff the post-click state.
|
|
551
562
|
let preSnap = null;
|
|
552
563
|
if (validatedArgs.autoAnalyzeAfter) {
|
|
553
|
-
preSnap = await getApomSnapshot(
|
|
564
|
+
preSnap = await getApomSnapshot(ctx);
|
|
554
565
|
}
|
|
555
566
|
|
|
556
567
|
// Use shared click action handler
|
|
@@ -560,15 +571,16 @@ async function executeToolInternal(name, args) {
|
|
|
560
571
|
skipNetworkWait: validatedArgs.skipNetworkWait,
|
|
561
572
|
networkWaitTimeout: validatedArgs.networkWaitTimeout,
|
|
562
573
|
waitForSelector: validatedArgs.waitForSelector,
|
|
563
|
-
waitTimeoutMs: validatedArgs.waitTimeoutMs
|
|
574
|
+
waitTimeoutMs: validatedArgs.waitTimeoutMs,
|
|
575
|
+
waitForRouteChange: validatedArgs.waitForRouteChange
|
|
564
576
|
});
|
|
565
577
|
|
|
566
578
|
// Post-click APOM delta: re-register new ids so callers can immediately
|
|
567
579
|
// use them in follow-up click/type calls without an extra analyzePage call.
|
|
568
580
|
if (validatedArgs.autoAnalyzeAfter && preSnap) {
|
|
569
581
|
try {
|
|
570
|
-
await quickRegisterElements(
|
|
571
|
-
const postSnap = await getApomSnapshot(
|
|
582
|
+
await quickRegisterElements(ctx);
|
|
583
|
+
const postSnap = await getApomSnapshot(ctx);
|
|
572
584
|
const added = Object.keys(postSnap).filter(id => !preSnap[id]);
|
|
573
585
|
const removed = Object.keys(preSnap).filter(id => !postSnap[id]);
|
|
574
586
|
|
|
@@ -621,13 +633,16 @@ async function executeToolInternal(name, args) {
|
|
|
621
633
|
// Get identifier (id or selector)
|
|
622
634
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
623
635
|
|
|
636
|
+
// Resolve/query inside the active frame (P0-1)
|
|
637
|
+
const ctx = await getTargetFrame(page);
|
|
638
|
+
|
|
624
639
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
625
|
-
const resolved = await resolveSelector(
|
|
640
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
626
641
|
if (!resolved.found) {
|
|
627
642
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
628
643
|
}
|
|
629
644
|
|
|
630
|
-
const element = await
|
|
645
|
+
const element = await ctx.$(resolved.selector);
|
|
631
646
|
if (!element) {
|
|
632
647
|
throw new Error(`Element not found: ${identifier}`);
|
|
633
648
|
}
|
|
@@ -984,13 +999,16 @@ async function executeToolInternal(name, args) {
|
|
|
984
999
|
// Get identifier (id or selector)
|
|
985
1000
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
986
1001
|
|
|
1002
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1003
|
+
const ctx = await getTargetFrame(page);
|
|
1004
|
+
|
|
987
1005
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
988
|
-
const resolved = await resolveSelector(
|
|
1006
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
989
1007
|
if (!resolved.found) {
|
|
990
1008
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
991
1009
|
}
|
|
992
1010
|
|
|
993
|
-
const element = await
|
|
1011
|
+
const element = await ctx.$(resolved.selector);
|
|
994
1012
|
if (!element) {
|
|
995
1013
|
throw new Error(`Element not found: ${identifier}`);
|
|
996
1014
|
}
|
|
@@ -1008,17 +1026,20 @@ async function executeToolInternal(name, args) {
|
|
|
1008
1026
|
const timeout = validatedArgs.timeout || 5000;
|
|
1009
1027
|
const waitForVisible = validatedArgs.visible !== false;
|
|
1010
1028
|
|
|
1029
|
+
// Wait/query inside the active frame (P0-1)
|
|
1030
|
+
const ctx = await getTargetFrame(page);
|
|
1031
|
+
|
|
1011
1032
|
try {
|
|
1012
1033
|
if (waitForVisible) {
|
|
1013
1034
|
// Wait for element to be visible
|
|
1014
|
-
await
|
|
1035
|
+
await ctx.waitForSelector(validatedArgs.selector, { timeout, visible: true });
|
|
1015
1036
|
} else {
|
|
1016
1037
|
// Just wait for element to exist in DOM
|
|
1017
|
-
await
|
|
1038
|
+
await ctx.waitForSelector(validatedArgs.selector, { timeout });
|
|
1018
1039
|
}
|
|
1019
1040
|
|
|
1020
1041
|
// Get info about the element
|
|
1021
|
-
const elementInfo = await
|
|
1042
|
+
const elementInfo = await ctx.evaluate((selector) => {
|
|
1022
1043
|
const el = document.querySelector(selector);
|
|
1023
1044
|
if (!el) return null;
|
|
1024
1045
|
|
|
@@ -1052,31 +1073,38 @@ async function executeToolInternal(name, args) {
|
|
|
1052
1073
|
const page = await getLastOpenPage();
|
|
1053
1074
|
const timeout = validatedArgs.timeout || 30000;
|
|
1054
1075
|
|
|
1055
|
-
//
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
// case from QA reports). Skip when the snippet declares a function — that signals
|
|
1059
|
-
// an explicit user-defined scope and an implicit-return result is likely intended.
|
|
1060
|
-
let scriptToRun = validatedArgs.script;
|
|
1061
|
-
const trimmedHead = scriptToRun.replace(/^\s+/, '');
|
|
1062
|
-
if (/^return[\s;]/.test(trimmedHead) && !/\bfunction\s*[\w*(]/.test(scriptToRun)) {
|
|
1063
|
-
scriptToRun = `(async () => { ${scriptToRun} })()`;
|
|
1064
|
-
}
|
|
1076
|
+
// Run in the active frame (P0-1) — defaults to main frame
|
|
1077
|
+
const ctx = await getTargetFrame(page);
|
|
1078
|
+
const scriptToRun = validatedArgs.script;
|
|
1065
1079
|
|
|
1066
1080
|
// Wrap operation in timeout
|
|
1067
1081
|
const executeOperation = async () => {
|
|
1068
|
-
//
|
|
1069
|
-
//
|
|
1070
|
-
|
|
1071
|
-
|
|
1082
|
+
// Two-pass eval (P1-4): first run the snippet as-is so bare expressions
|
|
1083
|
+
// (`document.title`) keep returning their value. If that throws an
|
|
1084
|
+
// "Illegal return statement" SyntaxError (top-level `return ...`), retry
|
|
1085
|
+
// wrapped in an async IIFE. Robust, no fragile `^return`/`!function` heuristic.
|
|
1086
|
+
const result = await ctx.evaluate(async (code) => {
|
|
1087
|
+
const runEval = async (src) => {
|
|
1072
1088
|
// eslint-disable-next-line no-eval
|
|
1073
|
-
let evalResult = eval(
|
|
1089
|
+
let evalResult = eval(src);
|
|
1074
1090
|
// Await promises (async IIFE, fetch, etc.)
|
|
1075
1091
|
if (evalResult && typeof evalResult === 'object' && typeof evalResult.then === 'function') {
|
|
1076
1092
|
evalResult = await evalResult;
|
|
1077
1093
|
}
|
|
1078
|
-
return
|
|
1094
|
+
return evalResult;
|
|
1095
|
+
};
|
|
1096
|
+
try {
|
|
1097
|
+
return { success: true, result: await runEval(code) };
|
|
1079
1098
|
} catch (error) {
|
|
1099
|
+
const isIllegalReturn = error instanceof SyntaxError &&
|
|
1100
|
+
/return/i.test(error.message) && /illegal|outside|unexpected/i.test(error.message);
|
|
1101
|
+
if (isIllegalReturn) {
|
|
1102
|
+
try {
|
|
1103
|
+
return { success: true, result: await runEval(`(async () => { ${code} })()`) };
|
|
1104
|
+
} catch (wrapErr) {
|
|
1105
|
+
return { success: false, error: wrapErr.message };
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1080
1108
|
return { success: false, error: error.message };
|
|
1081
1109
|
}
|
|
1082
1110
|
}, scriptToRun);
|
|
@@ -1340,13 +1368,16 @@ async function executeToolInternal(name, args) {
|
|
|
1340
1368
|
// Get identifier (id or selector)
|
|
1341
1369
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
1342
1370
|
|
|
1371
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1372
|
+
const ctx = await getTargetFrame(page);
|
|
1373
|
+
|
|
1343
1374
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1344
|
-
const resolved = await resolveSelector(
|
|
1375
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1345
1376
|
if (!resolved.found) {
|
|
1346
1377
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1347
1378
|
}
|
|
1348
1379
|
|
|
1349
|
-
const element = await
|
|
1380
|
+
const element = await ctx.$(resolved.selector);
|
|
1350
1381
|
if (!element) {
|
|
1351
1382
|
throw new Error(`Element not found: ${identifier}`);
|
|
1352
1383
|
}
|
|
@@ -1363,13 +1394,16 @@ async function executeToolInternal(name, args) {
|
|
|
1363
1394
|
let element = null;
|
|
1364
1395
|
|
|
1365
1396
|
if (identifier) {
|
|
1397
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1398
|
+
const ctx = await getTargetFrame(page);
|
|
1399
|
+
|
|
1366
1400
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1367
|
-
const resolved = await resolveSelector(
|
|
1401
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1368
1402
|
if (!resolved.found) {
|
|
1369
1403
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1370
1404
|
}
|
|
1371
1405
|
|
|
1372
|
-
element = await
|
|
1406
|
+
element = await ctx.$(resolved.selector);
|
|
1373
1407
|
if (!element) {
|
|
1374
1408
|
throw new Error(`Element not found: ${identifier}`);
|
|
1375
1409
|
}
|
|
@@ -1390,13 +1424,16 @@ async function executeToolInternal(name, args) {
|
|
|
1390
1424
|
// Get identifier (id or selector)
|
|
1391
1425
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
1392
1426
|
|
|
1427
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1428
|
+
const ctx = await getTargetFrame(page);
|
|
1429
|
+
|
|
1393
1430
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1394
|
-
const resolved = await resolveSelector(
|
|
1431
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1395
1432
|
if (!resolved.found) {
|
|
1396
1433
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1397
1434
|
}
|
|
1398
1435
|
|
|
1399
|
-
const element = await
|
|
1436
|
+
const element = await ctx.$(resolved.selector);
|
|
1400
1437
|
if (!element) {
|
|
1401
1438
|
throw new Error(`Element not found: ${identifier}`);
|
|
1402
1439
|
}
|
|
@@ -1772,6 +1809,10 @@ async function executeToolInternal(name, args) {
|
|
|
1772
1809
|
await page.goto(validatedArgs.url, { waitUntil: validatedArgs.waitUntil || 'networkidle2' });
|
|
1773
1810
|
}
|
|
1774
1811
|
|
|
1812
|
+
// Top-level navigation invalidates any previously-selected iframe (P0-1).
|
|
1813
|
+
// Reset to main frame so subsequent tools don't target a dead frame.
|
|
1814
|
+
clearActiveFrame(page);
|
|
1815
|
+
|
|
1775
1816
|
const title = await page.title();
|
|
1776
1817
|
|
|
1777
1818
|
// Run post-navigation diagnostics (same as post-click)
|
|
@@ -1811,6 +1852,70 @@ async function executeToolInternal(name, args) {
|
|
|
1811
1852
|
};
|
|
1812
1853
|
}
|
|
1813
1854
|
|
|
1855
|
+
if (name === "listFrames") {
|
|
1856
|
+
schemas.ListFramesSchema.parse(args);
|
|
1857
|
+
const page = await getLastOpenPage();
|
|
1858
|
+
const frames = listFramesForPage(page);
|
|
1859
|
+
const matcher = getActiveFrameMatcher(page);
|
|
1860
|
+
|
|
1861
|
+
return {
|
|
1862
|
+
content: [{
|
|
1863
|
+
type: "text",
|
|
1864
|
+
text: JSON.stringify({
|
|
1865
|
+
count: frames.length,
|
|
1866
|
+
activeFrame: matcher || null, // null = main frame
|
|
1867
|
+
frames,
|
|
1868
|
+
hint: frames.length > 1
|
|
1869
|
+
? 'Use switchFrame({ frameUrl }) to target a child frame (e.g. a cross-origin iframe). switchFrame() with no args resets to main frame.'
|
|
1870
|
+
: 'Only the main frame is present.'
|
|
1871
|
+
}, null, 2)
|
|
1872
|
+
}],
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
if (name === "switchFrame") {
|
|
1877
|
+
const validatedArgs = schemas.SwitchFrameSchema.parse(args);
|
|
1878
|
+
const page = await getLastOpenPage();
|
|
1879
|
+
|
|
1880
|
+
// No args → reset to main frame
|
|
1881
|
+
if (!validatedArgs.frameUrl && !validatedArgs.frameSelector) {
|
|
1882
|
+
clearActiveFrame(page);
|
|
1883
|
+
return {
|
|
1884
|
+
content: [{
|
|
1885
|
+
type: "text",
|
|
1886
|
+
text: JSON.stringify({ active: null, message: 'Reset to main frame.' }, null, 2)
|
|
1887
|
+
}],
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// Validate the matcher resolves before storing it, so switchFrame fails loudly
|
|
1892
|
+
// instead of every later tool failing.
|
|
1893
|
+
const matcher = validatedArgs.frameSelector
|
|
1894
|
+
? { frameSelector: validatedArgs.frameSelector }
|
|
1895
|
+
: { frameUrl: validatedArgs.frameUrl };
|
|
1896
|
+
setActiveFrame(page, matcher);
|
|
1897
|
+
|
|
1898
|
+
let frame;
|
|
1899
|
+
try {
|
|
1900
|
+
frame = await getTargetFrame(page);
|
|
1901
|
+
} catch (e) {
|
|
1902
|
+
clearActiveFrame(page); // don't leave a broken active frame set
|
|
1903
|
+
throw e;
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
return {
|
|
1907
|
+
content: [{
|
|
1908
|
+
type: "text",
|
|
1909
|
+
text: JSON.stringify({
|
|
1910
|
+
active: frame.url(),
|
|
1911
|
+
matcher,
|
|
1912
|
+
frames: listFramesForPage(page),
|
|
1913
|
+
message: `Switched to frame: ${frame.url()}. Subsequent click/type/analyzePage/find/executeScript run inside it until switchFrame() resets.`
|
|
1914
|
+
}, null, 2)
|
|
1915
|
+
}],
|
|
1916
|
+
};
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1814
1919
|
// Figma tools
|
|
1815
1920
|
if (name === "getFigmaFrame") {
|
|
1816
1921
|
const validatedArgs = schemas.GetFigmaFrameSchema.parse(args);
|
|
@@ -2383,10 +2488,12 @@ Start coding now.`;
|
|
|
2383
2488
|
if (name === "smartFindElement") {
|
|
2384
2489
|
const validatedArgs = schemas.SmartFindElementSchema.parse(args);
|
|
2385
2490
|
const page = await getLastOpenPage();
|
|
2491
|
+
const ctx = await getTargetFrame(page); // search inside active frame (P0-1)
|
|
2386
2492
|
const maxResults = validatedArgs.maxResults || 5;
|
|
2493
|
+
const minConfidence = validatedArgs.minConfidence != null ? validatedArgs.minConfidence : 0.6;
|
|
2387
2494
|
|
|
2388
2495
|
// Execute smart search in page context
|
|
2389
|
-
const results = await
|
|
2496
|
+
const results = await ctx.evaluate((description, maxResults, utilsCode, selectorResolverCode) => {
|
|
2390
2497
|
// Inject utilities into page context
|
|
2391
2498
|
eval(utilsCode);
|
|
2392
2499
|
if (typeof registerElement === 'undefined') {
|
|
@@ -2396,25 +2503,38 @@ Start coding now.`;
|
|
|
2396
2503
|
// Determine element type from description
|
|
2397
2504
|
const elementType = determineElementType(description);
|
|
2398
2505
|
|
|
2399
|
-
// Build candidate selectors based on element type
|
|
2400
|
-
|
|
2506
|
+
// Build candidate selectors based on element type.
|
|
2507
|
+
// Dedup with a Set — broadened selectors overlap (a button can match
|
|
2508
|
+
// several queries) and we don't want the same element scored twice.
|
|
2509
|
+
const candidateSet = new Set();
|
|
2510
|
+
const add = (sel) => { try { document.querySelectorAll(sel).forEach(el => candidateSet.add(el)); } catch (e) {} };
|
|
2401
2511
|
|
|
2402
2512
|
if (elementType.type === 'input' || elementType.type === 'any') {
|
|
2403
|
-
|
|
2404
|
-
|
|
2513
|
+
add('input');
|
|
2514
|
+
add('textarea');
|
|
2405
2515
|
}
|
|
2406
2516
|
|
|
2407
2517
|
if (elementType.type === 'button' || elementType.type === 'any') {
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2518
|
+
add('button');
|
|
2519
|
+
add('input[type="submit"]');
|
|
2520
|
+
add('input[type="button"]');
|
|
2521
|
+
add('[role="button"]');
|
|
2522
|
+
// P1-3: broaden to non-semantic clickables that QA hit on real apps —
|
|
2523
|
+
// menu items rendered as div/span[onclick], nav links, tabs.
|
|
2524
|
+
add('[onclick]');
|
|
2525
|
+
add('[role="menuitem"]');
|
|
2526
|
+
add('[role="tab"]');
|
|
2527
|
+
add('nav a');
|
|
2528
|
+
add('[role="navigation"] a');
|
|
2529
|
+
add('[role="menu"] [role="menuitem"]');
|
|
2412
2530
|
}
|
|
2413
2531
|
|
|
2414
2532
|
if (elementType.type === 'link' || elementType.type === 'any') {
|
|
2415
|
-
|
|
2533
|
+
add('a');
|
|
2416
2534
|
}
|
|
2417
2535
|
|
|
2536
|
+
const candidates = Array.from(candidateSet);
|
|
2537
|
+
|
|
2418
2538
|
// Analyze each candidate
|
|
2419
2539
|
const analyzed = candidates.map(el => {
|
|
2420
2540
|
const context = analyzeButtonContextInPage(el);
|
|
@@ -2478,11 +2598,26 @@ Start coding now.`;
|
|
|
2478
2598
|
hints
|
|
2479
2599
|
};
|
|
2480
2600
|
|
|
2481
|
-
// Execute action if provided
|
|
2601
|
+
// Execute action if provided — but only when we're confident (P1-3).
|
|
2602
|
+
// Previously results[0] was clicked unconditionally, so a high-scoring form
|
|
2603
|
+
// submit could be auto-clicked when the user asked for something else.
|
|
2604
|
+
// Guard on minConfidence AND a clear gap over the runner-up.
|
|
2482
2605
|
if (validatedArgs.action && results.length > 0) {
|
|
2483
2606
|
const bestMatch = results[0];
|
|
2607
|
+
const runnerUp = results[1];
|
|
2608
|
+
const gapOk = !runnerUp || (bestMatch.score - runnerUp.score) >= 10;
|
|
2609
|
+
const confident = bestMatch.confidence >= minConfidence && gapOk;
|
|
2610
|
+
|
|
2611
|
+
if (!confident) {
|
|
2612
|
+
response.actionSkipped = {
|
|
2613
|
+
reason: bestMatch.confidence < minConfidence
|
|
2614
|
+
? `bestMatch confidence ${bestMatch.confidence.toFixed(2)} < minConfidence ${minConfidence}`
|
|
2615
|
+
: `ambiguous: top-2 score gap (${bestMatch.score - runnerUp.score}) too small`,
|
|
2616
|
+
hint: 'Refine the description, pass an explicit id/selector to click, or lower minConfidence.',
|
|
2617
|
+
};
|
|
2618
|
+
} else {
|
|
2484
2619
|
try {
|
|
2485
|
-
const actionResult = await executeElementAction(page, bestMatch.selector, validatedArgs.action);
|
|
2620
|
+
const actionResult = await executeElementAction(page, bestMatch.selector, validatedArgs.action, ctx);
|
|
2486
2621
|
response.actionExecuted = actionResult;
|
|
2487
2622
|
|
|
2488
2623
|
// If screenshot was captured, add it to content
|
|
@@ -2497,6 +2632,7 @@ Start coding now.`;
|
|
|
2497
2632
|
} catch (error) {
|
|
2498
2633
|
response.actionError = error.message;
|
|
2499
2634
|
}
|
|
2635
|
+
}
|
|
2500
2636
|
}
|
|
2501
2637
|
|
|
2502
2638
|
return {
|
|
@@ -2515,10 +2651,14 @@ Start coding now.`;
|
|
|
2515
2651
|
if (name === "analyzePage") {
|
|
2516
2652
|
const validatedArgs = schemas.AnalyzePageSchema.parse(args);
|
|
2517
2653
|
const page = await getLastOpenPage();
|
|
2518
|
-
|
|
2654
|
+
// Analyze the active frame (P0-1) — defaults to main frame. `frameList` is
|
|
2655
|
+
// surfaced in the output so the agent can discover/switch frames.
|
|
2656
|
+
const ctx = await getTargetFrame(page);
|
|
2657
|
+
const frameList = listFramesForPage(page);
|
|
2658
|
+
const pageUrl = ctx.url() || page.url();
|
|
2519
2659
|
|
|
2520
2660
|
// Check for non-HTML pages (JSON, plain text, XML) — return raw content instead of empty tree
|
|
2521
|
-
const contentType = await
|
|
2661
|
+
const contentType = await ctx.evaluate(() => document.contentType || '');
|
|
2522
2662
|
if (contentType && !contentType.includes('html')) {
|
|
2523
2663
|
const rawContent = await page.evaluate(() => {
|
|
2524
2664
|
// For JSON/text pages, browser wraps content in <pre> inside <body>
|
|
@@ -2544,7 +2684,7 @@ Start coding now.`;
|
|
|
2544
2684
|
}
|
|
2545
2685
|
|
|
2546
2686
|
// APOM Tree format (default) - v2 with tree structure and positioning
|
|
2547
|
-
const apomResult = await
|
|
2687
|
+
const apomResult = await ctx.evaluate(async (apomTreeConverterCode, selectorResolverCode, modelsCode, shouldRegister, includeAll, viewportOnly, portalOpts) => {
|
|
2548
2688
|
// Inject utilities if not already loaded
|
|
2549
2689
|
if (typeof buildAPOMTree === 'undefined') {
|
|
2550
2690
|
eval(apomTreeConverterCode);
|
|
@@ -2619,6 +2759,13 @@ Start coding now.`;
|
|
|
2619
2759
|
selectors: validatedArgs.portalSelectors || undefined
|
|
2620
2760
|
});
|
|
2621
2761
|
|
|
2762
|
+
// Surface frames so the agent can discover/switch into iframes (P0-1).
|
|
2763
|
+
// Only when there's more than the main frame — keeps single-frame output lean.
|
|
2764
|
+
if (frameList.length > 1) {
|
|
2765
|
+
apomResult.frames = frameList;
|
|
2766
|
+
apomResult.activeFrame = ctx === page.mainFrame() ? null : (ctx.url() || true);
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2622
2769
|
// Handle diff mode
|
|
2623
2770
|
if (validatedArgs.diff) {
|
|
2624
2771
|
const previousAnalysis = global.previousApomAnalysis.get(pageUrl);
|
|
@@ -2641,7 +2788,8 @@ Start coding now.`;
|
|
|
2641
2788
|
previousTimestamp: previousAnalysis.timestamp,
|
|
2642
2789
|
diff,
|
|
2643
2790
|
metadata: apomResult.metadata,
|
|
2644
|
-
alerts: apomResult.alerts
|
|
2791
|
+
alerts: apomResult.alerts,
|
|
2792
|
+
frames: frameList.length > 1 ? frameList : undefined
|
|
2645
2793
|
})
|
|
2646
2794
|
}]
|
|
2647
2795
|
};
|
|
@@ -2889,8 +3037,9 @@ Start coding now.`;
|
|
|
2889
3037
|
if (name === "findElementsByText") {
|
|
2890
3038
|
const validatedArgs = schemas.FindElementsByTextSchema.parse(args);
|
|
2891
3039
|
const page = await getLastOpenPage();
|
|
3040
|
+
const ctx = await getTargetFrame(page); // search inside active frame (P0-1)
|
|
2892
3041
|
|
|
2893
|
-
const elements = await
|
|
3042
|
+
const elements = await ctx.evaluate((text, exact, caseSensitive, utilsCode, selectorResolverCode) => {
|
|
2894
3043
|
eval(utilsCode);
|
|
2895
3044
|
if (typeof registerElement === 'undefined') {
|
|
2896
3045
|
eval(selectorResolverCode);
|
|
@@ -2956,11 +3105,14 @@ Start coding now.`;
|
|
|
2956
3105
|
truncated: elements.length > 20
|
|
2957
3106
|
};
|
|
2958
3107
|
|
|
2959
|
-
// Execute action if provided and elements found
|
|
3108
|
+
// Execute action if provided and elements found.
|
|
3109
|
+
// Pass `ctx` so the raw selector resolves inside the active frame (P0-1) —
|
|
3110
|
+
// the FIND ran in the frame, so the ACTION must too, otherwise page.$
|
|
3111
|
+
// looks in the main frame and "Element not found for action".
|
|
2960
3112
|
if (validatedArgs.action && elements.length > 0) {
|
|
2961
3113
|
const firstMatch = elements[0];
|
|
2962
3114
|
try {
|
|
2963
|
-
const actionResult = await executeElementAction(page, firstMatch.selector, validatedArgs.action);
|
|
3115
|
+
const actionResult = await executeElementAction(page, firstMatch.selector, validatedArgs.action, ctx);
|
|
2964
3116
|
response.actionExecuted = actionResult;
|
|
2965
3117
|
|
|
2966
3118
|
// If screenshot was captured, add it to content
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "chrometools-mcp",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.6.0",
|
|
4
4
|
"description": "MCP (Model Context Protocol) server for Chrome automation using Puppeteer. Persistent browser sessions, UI framework detection (MUI, Ant Design, etc.), Page Object support, visual testing, Figma comparison. Works seamlessly in WSL, Linux, macOS, and Windows.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
@@ -39,6 +39,7 @@ export const toolDefinitions = [
|
|
|
39
39
|
timeout: { type: "number", description: "Max wait ms (default: 30000)" },
|
|
40
40
|
waitForSelector: { type: "string", description: "CSS selector to wait for after click (atomic click+wait). Use for dropdowns/popups that render into portals." },
|
|
41
41
|
waitTimeoutMs: { type: "number", description: "Timeout for waitForSelector in ms (default: 2000)." },
|
|
42
|
+
waitForRouteChange: { type: "boolean", description: "SPA route wait: after click, wait for location.pathname+search to change vs before; reports 'routeChanged:true/false'. Does not fail click on timeout. For URL-less view changes use waitForSelector." },
|
|
42
43
|
autoAnalyzeAfter: { type: "boolean", description: "After click, diff APOM and append '+N appeared: id:\"text\"' delta to result. New ids are pre-registered for follow-up clicks. Use for dropdowns/menus opening with new options." },
|
|
43
44
|
},
|
|
44
45
|
},
|
|
@@ -367,6 +368,25 @@ Examples:
|
|
|
367
368
|
required: ["url"],
|
|
368
369
|
},
|
|
369
370
|
},
|
|
371
|
+
{
|
|
372
|
+
name: "listFrames",
|
|
373
|
+
description: "List all frames (main + iframes) on the current page with url/name/isMain, plus the currently active frame. Use to discover cross-origin iframes (e.g. app.example.com), then switchFrame into one.",
|
|
374
|
+
inputSchema: {
|
|
375
|
+
type: "object",
|
|
376
|
+
properties: {},
|
|
377
|
+
},
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
name: "switchFrame",
|
|
381
|
+
description: "Set the active frame so click/type/hover/analyzePage/find/executeScript/waitForElement run INSIDE it — required to automate cross-origin iframes (resolved via CDP, bypassing Same-Origin Policy). Call with no args to reset to the main frame. Auto-resets on navigateTo.",
|
|
382
|
+
inputSchema: {
|
|
383
|
+
type: "object",
|
|
384
|
+
properties: {
|
|
385
|
+
frameUrl: { type: "string", description: "Substring matched against each frame's URL (e.g. 'app.example.com'). Mutually exclusive with frameSelector." },
|
|
386
|
+
frameSelector: { type: "string", description: "CSS selector of the <iframe> element; its content frame becomes active. Mutually exclusive with frameUrl." },
|
|
387
|
+
},
|
|
388
|
+
},
|
|
389
|
+
},
|
|
370
390
|
{
|
|
371
391
|
name: "getFigmaFrame",
|
|
372
392
|
description: "Export Figma frame as PNG. Requires API token and file/node IDs.",
|
|
@@ -506,6 +526,7 @@ Examples:
|
|
|
506
526
|
properties: {
|
|
507
527
|
description: { type: "string", description: "Natural language description" },
|
|
508
528
|
maxResults: { type: "number", minimum: 1, maximum: 20, description: "Max candidates (default: 5)" },
|
|
529
|
+
minConfidence: { type: "number", minimum: 0, maximum: 1, description: "Confidence threshold (default: 0.6) for auto-executing `action`. Below it (or too close to runner-up), action is skipped and candidates returned with 'actionSkipped'. Prevents auto-clicking the wrong control." },
|
|
509
530
|
action: {
|
|
510
531
|
type: "object",
|
|
511
532
|
properties: {
|
package/server/tool-groups.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
export const toolGroups = {
|
|
8
8
|
core: ['ping', 'openBrowser', 'executeScript', 'navigateTo'],
|
|
9
9
|
|
|
10
|
-
interaction: ['click', 'type', 'scrollTo', 'waitForElement', 'hover', 'selectOption', 'selectFromGroup', 'drag', 'scrollHorizontal'],
|
|
10
|
+
interaction: ['click', 'type', 'scrollTo', 'waitForElement', 'hover', 'selectOption', 'selectFromGroup', 'drag', 'scrollHorizontal', 'switchFrame', 'listFrames'],
|
|
11
11
|
|
|
12
12
|
inspection: ['getElement', 'getComputedCss', 'getBoxModel', 'screenshot', 'saveScreenshot'],
|
|
13
13
|
|
package/server/tool-schemas.js
CHANGED
|
@@ -25,6 +25,7 @@ export const ClickSchema = z.object({
|
|
|
25
25
|
networkWaitTimeout: z.number().optional().describe("Maximum time to wait for network requests in ms (default: 3000). Only used if skipNetworkWait is false."),
|
|
26
26
|
waitForSelector: z.string().optional().describe("CSS selector to wait for after click — atomic click+wait. Useful for dropdowns/popups in portals (e.g. '#menu-popup-root > div') that otherwise race against the next MCP call."),
|
|
27
27
|
waitTimeoutMs: z.number().optional().describe("Timeout for waitForSelector in ms (default: 2000)."),
|
|
28
|
+
waitForRouteChange: z.boolean().optional().describe("For SPAs (React Router etc.): after click, wait for location.pathname+search to change relative to before. Surfaces 'routeChanged:true/false' so success means the view actually navigated, not just that the click was delivered. Does not fail the click on timeout. For content that renders without a URL change, prefer waitForSelector."),
|
|
28
29
|
autoAnalyzeAfter: z.boolean().optional().describe("After click, automatically diff APOM state and append a delta to the result: '+N appeared: id1:\"text\", id2:\"text\"'. New ids are re-registered so callers can use them directly in the next click/type call without an extra analyzePage. Use for dropdowns and menus that reveal new options on click."),
|
|
29
30
|
}).refine(data => (data.id && !data.selector) || (!data.id && data.selector), {
|
|
30
31
|
message: "Either 'id' or 'selector' must be provided, but not both"
|
|
@@ -156,6 +157,13 @@ export const NavigateToSchema = z.object({
|
|
|
156
157
|
.describe("Wait until event (default: networkidle2)"),
|
|
157
158
|
});
|
|
158
159
|
|
|
160
|
+
export const ListFramesSchema = z.object({});
|
|
161
|
+
|
|
162
|
+
export const SwitchFrameSchema = z.object({
|
|
163
|
+
frameUrl: z.string().optional().describe("Substring matched against each frame's URL (e.g. 'app.example.com'). Selects the first matching frame, including cross-origin iframes (resolved via CDP, so Same-Origin Policy does not block access). Mutually exclusive with frameSelector."),
|
|
164
|
+
frameSelector: z.string().optional().describe("CSS selector of the <iframe> element in the current document; its content frame becomes active. Mutually exclusive with frameUrl."),
|
|
165
|
+
}).describe("Set the active frame for subsequent click/type/hover/analyzePage/find/executeScript/waitForElement. Call with NO arguments to reset back to the main frame. Active frame is also reset automatically on navigateTo. Use listFrames() to discover available frames.");
|
|
166
|
+
|
|
159
167
|
export const SetViewportSchema = z.object({
|
|
160
168
|
width: z.number().min(320).max(4000).describe("Viewport width in pixels (320-4000)"),
|
|
161
169
|
height: z.number().min(200).max(3000).describe("Viewport height in pixels (200-3000)"),
|
|
@@ -272,6 +280,7 @@ export const ConvertFigmaToCodeSchema = z.object({
|
|
|
272
280
|
export const SmartFindElementSchema = z.object({
|
|
273
281
|
description: z.string().describe("Natural language description of element to find (e.g., 'login button', 'email field')"),
|
|
274
282
|
maxResults: z.number().min(1).max(20).optional().describe("Maximum number of candidates to return (default: 5)"),
|
|
283
|
+
minConfidence: z.number().min(0).max(1).optional().describe("Confidence threshold (0-1, default: 0.6) for auto-executing `action`. If the best match scores below this, OR is too close to the runner-up, the action is SKIPPED and candidates are returned with an 'actionSkipped' reason — prevents auto-clicking the wrong control (e.g. a primary form submit when you asked for a menu item). Lower it to act on weaker matches."),
|
|
275
284
|
action: z.object({
|
|
276
285
|
type: z.enum(['click', 'type', 'scrollTo', 'screenshot', 'hover', 'setStyles']).describe("Action to perform on the best match"),
|
|
277
286
|
text: z.string().optional().describe("Text to type (required for 'type' action)"),
|
|
@@ -28,7 +28,8 @@ export async function executeClickAction(page, element, options = {}) {
|
|
|
28
28
|
skipNetworkWait = false,
|
|
29
29
|
networkWaitTimeout = 3000,
|
|
30
30
|
waitForSelector = null,
|
|
31
|
-
waitTimeoutMs = 2000
|
|
31
|
+
waitTimeoutMs = 2000,
|
|
32
|
+
waitForRouteChange = false
|
|
32
33
|
} = options;
|
|
33
34
|
|
|
34
35
|
// Capture timestamp and URL BEFORE click for diagnostics
|
|
@@ -150,6 +151,33 @@ export async function executeClickAction(page, element, options = {}) {
|
|
|
150
151
|
}
|
|
151
152
|
}
|
|
152
153
|
|
|
154
|
+
// SPA route-change wait (P1-5): SPAs navigate via history.pushState, so page.url()
|
|
155
|
+
// diagnostics below may not register a "navigation". Wait for location.pathname+search
|
|
156
|
+
// to actually change relative to before. Best-effort — a timeout never fails the click.
|
|
157
|
+
let routeChangeResult = null;
|
|
158
|
+
if (waitForRouteChange) {
|
|
159
|
+
let pathBefore;
|
|
160
|
+
try {
|
|
161
|
+
pathBefore = await page.evaluate(() => location.pathname + location.search);
|
|
162
|
+
} catch (e) {
|
|
163
|
+
pathBefore = null;
|
|
164
|
+
}
|
|
165
|
+
if (pathBefore !== null) {
|
|
166
|
+
const routeStart = Date.now();
|
|
167
|
+
try {
|
|
168
|
+
await page.waitForFunction(
|
|
169
|
+
(before) => (location.pathname + location.search) !== before,
|
|
170
|
+
{ timeout: waitTimeoutMs },
|
|
171
|
+
pathBefore
|
|
172
|
+
);
|
|
173
|
+
const pathAfter = await page.evaluate(() => location.pathname + location.search);
|
|
174
|
+
routeChangeResult = { routeChanged: true, from: pathBefore, to: pathAfter, changedInMs: Date.now() - routeStart };
|
|
175
|
+
} catch (e) {
|
|
176
|
+
routeChangeResult = { routeChanged: false, from: pathBefore, timeoutMs: waitTimeoutMs };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
153
181
|
// Check if element was detached from DOM during click (Angular *ngFor + Zone.js pattern)
|
|
154
182
|
let elementDetached = false;
|
|
155
183
|
try {
|
|
@@ -230,6 +258,15 @@ export async function executeClickAction(page, element, options = {}) {
|
|
|
230
258
|
hintsText += `\nSuggested next: ${hints.suggestedNext.join('; ')}`;
|
|
231
259
|
}
|
|
232
260
|
|
|
261
|
+
// Surface SPA route-change outcome to the caller
|
|
262
|
+
if (routeChangeResult) {
|
|
263
|
+
if (routeChangeResult.routeChanged) {
|
|
264
|
+
hintsText += `\nRoute changed: "${routeChangeResult.from}" → "${routeChangeResult.to}" (${routeChangeResult.changedInMs}ms)`;
|
|
265
|
+
} else {
|
|
266
|
+
hintsText += `\n⚠️ ROUTE_UNCHANGED: location did not change within ${routeChangeResult.timeoutMs}ms after click (still "${routeChangeResult.from}"). View may render without a URL change — use waitForSelector instead.`;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
233
270
|
// Surface waitForSelector outcome to the caller (success or timeout)
|
|
234
271
|
if (waitResult) {
|
|
235
272
|
if (waitResult.appeared) {
|