chrometools-mcp 3.5.5 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +104 -8
- package/browser/browser-manager.js +26 -2
- package/browser/page-manager.js +89 -0
- package/element-finder-utils.js +38 -0
- package/index.js +330 -52
- package/package.json +1 -1
- package/pom/apom-tree-converter.js +56 -1
- package/server/tool-definitions.js +30 -4
- package/server/tool-groups.js +1 -1
- package/server/tool-schemas.js +19 -5
- package/utils/actions/click-action.js +70 -1
- package/utils/element-actions.js +8 -5
- package/utils/platform-utils.js +17 -2
- package/utils/post-click-diagnostics.js +21 -4
- package/NUL +0 -1
package/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
import {Server} from "@modelcontextprotocol/sdk/server/index.js";
|
|
4
4
|
import {StdioServerTransport} from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
@@ -37,7 +37,12 @@ import {
|
|
|
37
37
|
getAllPages,
|
|
38
38
|
switchToPage,
|
|
39
39
|
connectToTabByUrl,
|
|
40
|
-
getNetworkCDPClient
|
|
40
|
+
getNetworkCDPClient,
|
|
41
|
+
getTargetFrame,
|
|
42
|
+
setActiveFrame,
|
|
43
|
+
clearActiveFrame,
|
|
44
|
+
getActiveFrameMatcher,
|
|
45
|
+
listFramesForPage
|
|
41
46
|
} from './browser/page-manager.js';
|
|
42
47
|
|
|
43
48
|
// Import image processing utilities
|
|
@@ -403,8 +408,13 @@ async function executeToolInternal(name, args) {
|
|
|
403
408
|
* Quick element registration - runs APOM analysis and registers elements
|
|
404
409
|
*/
|
|
405
410
|
async function quickRegisterElements(page) {
|
|
406
|
-
await page.evaluate((apomTreeConverterCode, selectorResolverCode) => {
|
|
407
|
-
// Inject utilities
|
|
411
|
+
await page.evaluate((apomTreeConverterCode, selectorResolverCode, modelsCode) => {
|
|
412
|
+
// Inject utilities. Models must be loaded BEFORE buildAPOMTree because
|
|
413
|
+
// initializeModelRegistry() inside it references window.ModelRegistry. After a
|
|
414
|
+
// navigation the browser window is wiped, so we always re-eval if missing.
|
|
415
|
+
if (typeof window.ModelRegistry === 'undefined') {
|
|
416
|
+
eval(modelsCode);
|
|
417
|
+
}
|
|
408
418
|
if (typeof buildAPOMTree === 'undefined') {
|
|
409
419
|
eval(apomTreeConverterCode);
|
|
410
420
|
}
|
|
@@ -441,7 +451,7 @@ async function executeToolInternal(name, args) {
|
|
|
441
451
|
if (typeof registerElements !== 'undefined') {
|
|
442
452
|
registerElements(elementsArray);
|
|
443
453
|
}
|
|
444
|
-
}, apomTreeConverter, selectorResolver);
|
|
454
|
+
}, apomTreeConverter, selectorResolver, elementModelBase + '\n' + elementModels + '\n' + modelRegistry);
|
|
445
455
|
}
|
|
446
456
|
|
|
447
457
|
/**
|
|
@@ -470,15 +480,56 @@ async function executeToolInternal(name, args) {
|
|
|
470
480
|
|
|
471
481
|
let resolved = await Promise.race([tryResolve(), timeoutPromise]);
|
|
472
482
|
|
|
473
|
-
// Auto-refresh: if looks like APOM ID but not found, re-register elements and retry
|
|
483
|
+
// Auto-refresh: if looks like APOM ID but not found, re-register elements and retry.
|
|
484
|
+
// quickRegisterElements re-injects ModelRegistry — if that itself fails (e.g. ReferenceError
|
|
485
|
+
// from a partially-loaded browser context), surface a clear "call analyzePage" message
|
|
486
|
+
// instead of leaking the raw browser error.
|
|
474
487
|
if (!resolved.found && isApomIdPattern(identifier)) {
|
|
475
|
-
|
|
488
|
+
try {
|
|
489
|
+
await quickRegisterElements(page);
|
|
490
|
+
} catch (e) {
|
|
491
|
+
if (/ModelRegistry is not defined/.test(String(e?.message || e))) {
|
|
492
|
+
throw new Error(
|
|
493
|
+
`APOM registry stale after navigation/reload. Call analyzePage() to refresh element ids before reusing "${identifier}".`
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
throw e;
|
|
497
|
+
}
|
|
476
498
|
resolved = await Promise.race([tryResolve(), timeoutPromise]);
|
|
477
499
|
}
|
|
478
500
|
|
|
479
501
|
return resolved;
|
|
480
502
|
}
|
|
481
503
|
|
|
504
|
+
/**
|
|
505
|
+
* Snapshot of all APOM ids on the page → { id: { tag, type, text } }
|
|
506
|
+
* Used by click({ autoAnalyzeAfter: true }) to compute pre/post deltas.
|
|
507
|
+
*/
|
|
508
|
+
async function getApomSnapshot(page) {
|
|
509
|
+
return await page.evaluate((apomCode, modelsCode) => {
|
|
510
|
+
if (typeof window.ModelRegistry === 'undefined') eval(modelsCode);
|
|
511
|
+
if (typeof buildAPOMTree === 'undefined') eval(apomCode);
|
|
512
|
+
const apom = buildAPOMTree(true);
|
|
513
|
+
const map = {};
|
|
514
|
+
function walk(node) {
|
|
515
|
+
if (!node || typeof node !== 'object') return;
|
|
516
|
+
if (node.id) {
|
|
517
|
+
map[node.id] = {
|
|
518
|
+
tag: node.tag,
|
|
519
|
+
type: node.type,
|
|
520
|
+
text: (node.metadata && node.metadata.text ? String(node.metadata.text).substring(0, 60) : null)
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
if (Array.isArray(node.children)) node.children.forEach(walk);
|
|
524
|
+
for (const k of Object.keys(node)) {
|
|
525
|
+
if (Array.isArray(node[k])) node[k].forEach(walk);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
walk(apom.tree);
|
|
529
|
+
return map;
|
|
530
|
+
}, apomTreeConverter, elementModelBase + '\n' + elementModels + '\n' + modelRegistry);
|
|
531
|
+
}
|
|
532
|
+
|
|
482
533
|
if (name === "click") {
|
|
483
534
|
const validatedArgs = schemas.ClickSchema.parse(args);
|
|
484
535
|
const page = await getLastOpenPage();
|
|
@@ -489,24 +540,79 @@ async function executeToolInternal(name, args) {
|
|
|
489
540
|
// Get identifier (id or selector)
|
|
490
541
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
491
542
|
|
|
543
|
+
// Resolve/query inside the active frame (P0-1). The ElementHandle from
|
|
544
|
+
// ctx.$() runs in the frame's context, so element.click()/.evaluate()
|
|
545
|
+
// in executeClickAction operate inside the iframe. Page-level diagnostics
|
|
546
|
+
// (network/screenshot) stay on `page`.
|
|
547
|
+
const ctx = await getTargetFrame(page);
|
|
548
|
+
|
|
492
549
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
493
|
-
const resolved = await resolveSelector(
|
|
550
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
494
551
|
if (!resolved.found) {
|
|
495
552
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
496
553
|
}
|
|
497
554
|
|
|
498
|
-
const element = await
|
|
555
|
+
const element = await ctx.$(resolved.selector);
|
|
499
556
|
if (!element) {
|
|
500
557
|
throw new Error(`Element not found: ${identifier}`);
|
|
501
558
|
}
|
|
502
559
|
|
|
560
|
+
// Pre-click APOM snapshot (only if delta is requested) — captures every id
|
|
561
|
+
// currently in the tree so we can diff the post-click state.
|
|
562
|
+
let preSnap = null;
|
|
563
|
+
if (validatedArgs.autoAnalyzeAfter) {
|
|
564
|
+
preSnap = await getApomSnapshot(ctx);
|
|
565
|
+
}
|
|
566
|
+
|
|
503
567
|
// Use shared click action handler
|
|
504
|
-
|
|
568
|
+
const clickResult = await executeClickAction(page, element, {
|
|
505
569
|
identifier,
|
|
506
570
|
screenshot: validatedArgs.screenshot,
|
|
507
571
|
skipNetworkWait: validatedArgs.skipNetworkWait,
|
|
508
|
-
networkWaitTimeout: validatedArgs.networkWaitTimeout
|
|
572
|
+
networkWaitTimeout: validatedArgs.networkWaitTimeout,
|
|
573
|
+
waitForSelector: validatedArgs.waitForSelector,
|
|
574
|
+
waitTimeoutMs: validatedArgs.waitTimeoutMs,
|
|
575
|
+
waitForRouteChange: validatedArgs.waitForRouteChange
|
|
509
576
|
});
|
|
577
|
+
|
|
578
|
+
// Post-click APOM delta: re-register new ids so callers can immediately
|
|
579
|
+
// use them in follow-up click/type calls without an extra analyzePage call.
|
|
580
|
+
if (validatedArgs.autoAnalyzeAfter && preSnap) {
|
|
581
|
+
try {
|
|
582
|
+
await quickRegisterElements(ctx);
|
|
583
|
+
const postSnap = await getApomSnapshot(ctx);
|
|
584
|
+
const added = Object.keys(postSnap).filter(id => !preSnap[id]);
|
|
585
|
+
const removed = Object.keys(preSnap).filter(id => !postSnap[id]);
|
|
586
|
+
|
|
587
|
+
let deltaText = '\n\n** APOM DELTA **';
|
|
588
|
+
if (added.length === 0 && removed.length === 0) {
|
|
589
|
+
deltaText += '\nNo APOM changes detected after click';
|
|
590
|
+
} else {
|
|
591
|
+
if (added.length > 0) {
|
|
592
|
+
const sample = added.slice(0, 15).map(id => {
|
|
593
|
+
const m = postSnap[id] || {};
|
|
594
|
+
const lab = m.text ? `"${m.text}"` : (m.type || m.tag || '?');
|
|
595
|
+
return `${id}:${lab}`;
|
|
596
|
+
});
|
|
597
|
+
deltaText += `\n+${added.length} appeared: ${sample.join(', ')}${added.length > 15 ? `, ... (${added.length - 15} more)` : ''}`;
|
|
598
|
+
}
|
|
599
|
+
if (removed.length > 0) {
|
|
600
|
+
deltaText += `\n-${removed.length} disappeared`;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
if (clickResult && Array.isArray(clickResult.content) && clickResult.content[0]) {
|
|
605
|
+
clickResult.content[0].text += deltaText;
|
|
606
|
+
}
|
|
607
|
+
} catch (e) {
|
|
608
|
+
// Delta is best-effort — never let it fail the actual click result
|
|
609
|
+
if (clickResult && Array.isArray(clickResult.content) && clickResult.content[0]) {
|
|
610
|
+
clickResult.content[0].text += `\n\n** APOM DELTA ** unavailable (${e.message})`;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
return clickResult;
|
|
510
616
|
};
|
|
511
617
|
|
|
512
618
|
// Execute with timeout
|
|
@@ -527,13 +633,16 @@ async function executeToolInternal(name, args) {
|
|
|
527
633
|
// Get identifier (id or selector)
|
|
528
634
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
529
635
|
|
|
636
|
+
// Resolve/query inside the active frame (P0-1)
|
|
637
|
+
const ctx = await getTargetFrame(page);
|
|
638
|
+
|
|
530
639
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
531
|
-
const resolved = await resolveSelector(
|
|
640
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
532
641
|
if (!resolved.found) {
|
|
533
642
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
534
643
|
}
|
|
535
644
|
|
|
536
|
-
const element = await
|
|
645
|
+
const element = await ctx.$(resolved.selector);
|
|
537
646
|
if (!element) {
|
|
538
647
|
throw new Error(`Element not found: ${identifier}`);
|
|
539
648
|
}
|
|
@@ -786,6 +895,24 @@ async function executeToolInternal(name, args) {
|
|
|
786
895
|
// Get identifier (id or selector)
|
|
787
896
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
788
897
|
|
|
898
|
+
// No identifier → full viewport screenshot. Mirrors processSceenshot pipeline used
|
|
899
|
+
// by element screenshots so the output (JPEG, scaled, base64) stays consistent.
|
|
900
|
+
if (!identifier) {
|
|
901
|
+
const buffer = await page.screenshot({ encoding: 'binary', fullPage: false });
|
|
902
|
+
const processed = await processScreenshot(buffer, {
|
|
903
|
+
maxWidth: validatedArgs.maxWidth !== undefined ? validatedArgs.maxWidth : 1024,
|
|
904
|
+
maxHeight: validatedArgs.maxHeight !== undefined ? validatedArgs.maxHeight : 8000,
|
|
905
|
+
quality: validatedArgs.quality || 40,
|
|
906
|
+
format: validatedArgs.format || 'jpeg',
|
|
907
|
+
});
|
|
908
|
+
return {
|
|
909
|
+
content: [
|
|
910
|
+
{ type: 'text', text: 'Viewport screenshot' },
|
|
911
|
+
{ type: 'image', data: processed.buffer.toString('base64'), mimeType: processed.mimeType }
|
|
912
|
+
]
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
|
|
789
916
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
790
917
|
const resolved = await resolveSelector(page, identifier);
|
|
791
918
|
if (!resolved.found) {
|
|
@@ -872,13 +999,16 @@ async function executeToolInternal(name, args) {
|
|
|
872
999
|
// Get identifier (id or selector)
|
|
873
1000
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
874
1001
|
|
|
1002
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1003
|
+
const ctx = await getTargetFrame(page);
|
|
1004
|
+
|
|
875
1005
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
876
|
-
const resolved = await resolveSelector(
|
|
1006
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
877
1007
|
if (!resolved.found) {
|
|
878
1008
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
879
1009
|
}
|
|
880
1010
|
|
|
881
|
-
const element = await
|
|
1011
|
+
const element = await ctx.$(resolved.selector);
|
|
882
1012
|
if (!element) {
|
|
883
1013
|
throw new Error(`Element not found: ${identifier}`);
|
|
884
1014
|
}
|
|
@@ -896,17 +1026,20 @@ async function executeToolInternal(name, args) {
|
|
|
896
1026
|
const timeout = validatedArgs.timeout || 5000;
|
|
897
1027
|
const waitForVisible = validatedArgs.visible !== false;
|
|
898
1028
|
|
|
1029
|
+
// Wait/query inside the active frame (P0-1)
|
|
1030
|
+
const ctx = await getTargetFrame(page);
|
|
1031
|
+
|
|
899
1032
|
try {
|
|
900
1033
|
if (waitForVisible) {
|
|
901
1034
|
// Wait for element to be visible
|
|
902
|
-
await
|
|
1035
|
+
await ctx.waitForSelector(validatedArgs.selector, { timeout, visible: true });
|
|
903
1036
|
} else {
|
|
904
1037
|
// Just wait for element to exist in DOM
|
|
905
|
-
await
|
|
1038
|
+
await ctx.waitForSelector(validatedArgs.selector, { timeout });
|
|
906
1039
|
}
|
|
907
1040
|
|
|
908
1041
|
// Get info about the element
|
|
909
|
-
const elementInfo = await
|
|
1042
|
+
const elementInfo = await ctx.evaluate((selector) => {
|
|
910
1043
|
const el = document.querySelector(selector);
|
|
911
1044
|
if (!el) return null;
|
|
912
1045
|
|
|
@@ -940,23 +1073,41 @@ async function executeToolInternal(name, args) {
|
|
|
940
1073
|
const page = await getLastOpenPage();
|
|
941
1074
|
const timeout = validatedArgs.timeout || 30000;
|
|
942
1075
|
|
|
1076
|
+
// Run in the active frame (P0-1) — defaults to main frame
|
|
1077
|
+
const ctx = await getTargetFrame(page);
|
|
1078
|
+
const scriptToRun = validatedArgs.script;
|
|
1079
|
+
|
|
943
1080
|
// Wrap operation in timeout
|
|
944
1081
|
const executeOperation = async () => {
|
|
945
|
-
//
|
|
946
|
-
//
|
|
947
|
-
|
|
948
|
-
|
|
1082
|
+
// Two-pass eval (P1-4): first run the snippet as-is so bare expressions
|
|
1083
|
+
// (`document.title`) keep returning their value. If that throws an
|
|
1084
|
+
// "Illegal return statement" SyntaxError (top-level `return ...`), retry
|
|
1085
|
+
// wrapped in an async IIFE. Robust, no fragile `^return`/`!function` heuristic.
|
|
1086
|
+
const result = await ctx.evaluate(async (code) => {
|
|
1087
|
+
const runEval = async (src) => {
|
|
949
1088
|
// eslint-disable-next-line no-eval
|
|
950
|
-
let evalResult = eval(
|
|
1089
|
+
let evalResult = eval(src);
|
|
951
1090
|
// Await promises (async IIFE, fetch, etc.)
|
|
952
1091
|
if (evalResult && typeof evalResult === 'object' && typeof evalResult.then === 'function') {
|
|
953
1092
|
evalResult = await evalResult;
|
|
954
1093
|
}
|
|
955
|
-
return
|
|
1094
|
+
return evalResult;
|
|
1095
|
+
};
|
|
1096
|
+
try {
|
|
1097
|
+
return { success: true, result: await runEval(code) };
|
|
956
1098
|
} catch (error) {
|
|
1099
|
+
const isIllegalReturn = error instanceof SyntaxError &&
|
|
1100
|
+
/return/i.test(error.message) && /illegal|outside|unexpected/i.test(error.message);
|
|
1101
|
+
if (isIllegalReturn) {
|
|
1102
|
+
try {
|
|
1103
|
+
return { success: true, result: await runEval(`(async () => { ${code} })()`) };
|
|
1104
|
+
} catch (wrapErr) {
|
|
1105
|
+
return { success: false, error: wrapErr.message };
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
957
1108
|
return { success: false, error: error.message };
|
|
958
1109
|
}
|
|
959
|
-
},
|
|
1110
|
+
}, scriptToRun);
|
|
960
1111
|
|
|
961
1112
|
await new Promise(resolve => setTimeout(resolve, validatedArgs.waitAfter || 500));
|
|
962
1113
|
|
|
@@ -1217,13 +1368,16 @@ async function executeToolInternal(name, args) {
|
|
|
1217
1368
|
// Get identifier (id or selector)
|
|
1218
1369
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
1219
1370
|
|
|
1371
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1372
|
+
const ctx = await getTargetFrame(page);
|
|
1373
|
+
|
|
1220
1374
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1221
|
-
const resolved = await resolveSelector(
|
|
1375
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1222
1376
|
if (!resolved.found) {
|
|
1223
1377
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1224
1378
|
}
|
|
1225
1379
|
|
|
1226
|
-
const element = await
|
|
1380
|
+
const element = await ctx.$(resolved.selector);
|
|
1227
1381
|
if (!element) {
|
|
1228
1382
|
throw new Error(`Element not found: ${identifier}`);
|
|
1229
1383
|
}
|
|
@@ -1240,13 +1394,16 @@ async function executeToolInternal(name, args) {
|
|
|
1240
1394
|
let element = null;
|
|
1241
1395
|
|
|
1242
1396
|
if (identifier) {
|
|
1397
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1398
|
+
const ctx = await getTargetFrame(page);
|
|
1399
|
+
|
|
1243
1400
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1244
|
-
const resolved = await resolveSelector(
|
|
1401
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1245
1402
|
if (!resolved.found) {
|
|
1246
1403
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1247
1404
|
}
|
|
1248
1405
|
|
|
1249
|
-
element = await
|
|
1406
|
+
element = await ctx.$(resolved.selector);
|
|
1250
1407
|
if (!element) {
|
|
1251
1408
|
throw new Error(`Element not found: ${identifier}`);
|
|
1252
1409
|
}
|
|
@@ -1267,13 +1424,16 @@ async function executeToolInternal(name, args) {
|
|
|
1267
1424
|
// Get identifier (id or selector)
|
|
1268
1425
|
const identifier = validatedArgs.id || validatedArgs.selector;
|
|
1269
1426
|
|
|
1427
|
+
// Resolve/query inside the active frame (P0-1)
|
|
1428
|
+
const ctx = await getTargetFrame(page);
|
|
1429
|
+
|
|
1270
1430
|
// Resolve selector (supports both APOM ID and CSS selector)
|
|
1271
|
-
const resolved = await resolveSelector(
|
|
1431
|
+
const resolved = await resolveSelector(ctx, identifier);
|
|
1272
1432
|
if (!resolved.found) {
|
|
1273
1433
|
throw new Error(`Element not found: ${identifier}${resolved.isPageObjectId ? ' (APOM ID)' : ' (CSS selector)'}`);
|
|
1274
1434
|
}
|
|
1275
1435
|
|
|
1276
|
-
const element = await
|
|
1436
|
+
const element = await ctx.$(resolved.selector);
|
|
1277
1437
|
if (!element) {
|
|
1278
1438
|
throw new Error(`Element not found: ${identifier}`);
|
|
1279
1439
|
}
|
|
@@ -1649,6 +1809,10 @@ async function executeToolInternal(name, args) {
|
|
|
1649
1809
|
await page.goto(validatedArgs.url, { waitUntil: validatedArgs.waitUntil || 'networkidle2' });
|
|
1650
1810
|
}
|
|
1651
1811
|
|
|
1812
|
+
// Top-level navigation invalidates any previously-selected iframe (P0-1).
|
|
1813
|
+
// Reset to main frame so subsequent tools don't target a dead frame.
|
|
1814
|
+
clearActiveFrame(page);
|
|
1815
|
+
|
|
1652
1816
|
const title = await page.title();
|
|
1653
1817
|
|
|
1654
1818
|
// Run post-navigation diagnostics (same as post-click)
|
|
@@ -1688,6 +1852,70 @@ async function executeToolInternal(name, args) {
|
|
|
1688
1852
|
};
|
|
1689
1853
|
}
|
|
1690
1854
|
|
|
1855
|
+
if (name === "listFrames") {
|
|
1856
|
+
schemas.ListFramesSchema.parse(args);
|
|
1857
|
+
const page = await getLastOpenPage();
|
|
1858
|
+
const frames = listFramesForPage(page);
|
|
1859
|
+
const matcher = getActiveFrameMatcher(page);
|
|
1860
|
+
|
|
1861
|
+
return {
|
|
1862
|
+
content: [{
|
|
1863
|
+
type: "text",
|
|
1864
|
+
text: JSON.stringify({
|
|
1865
|
+
count: frames.length,
|
|
1866
|
+
activeFrame: matcher || null, // null = main frame
|
|
1867
|
+
frames,
|
|
1868
|
+
hint: frames.length > 1
|
|
1869
|
+
? 'Use switchFrame({ frameUrl }) to target a child frame (e.g. a cross-origin iframe). switchFrame() with no args resets to main frame.'
|
|
1870
|
+
: 'Only the main frame is present.'
|
|
1871
|
+
}, null, 2)
|
|
1872
|
+
}],
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
if (name === "switchFrame") {
|
|
1877
|
+
const validatedArgs = schemas.SwitchFrameSchema.parse(args);
|
|
1878
|
+
const page = await getLastOpenPage();
|
|
1879
|
+
|
|
1880
|
+
// No args → reset to main frame
|
|
1881
|
+
if (!validatedArgs.frameUrl && !validatedArgs.frameSelector) {
|
|
1882
|
+
clearActiveFrame(page);
|
|
1883
|
+
return {
|
|
1884
|
+
content: [{
|
|
1885
|
+
type: "text",
|
|
1886
|
+
text: JSON.stringify({ active: null, message: 'Reset to main frame.' }, null, 2)
|
|
1887
|
+
}],
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// Validate the matcher resolves before storing it, so switchFrame fails loudly
|
|
1892
|
+
// instead of every later tool failing.
|
|
1893
|
+
const matcher = validatedArgs.frameSelector
|
|
1894
|
+
? { frameSelector: validatedArgs.frameSelector }
|
|
1895
|
+
: { frameUrl: validatedArgs.frameUrl };
|
|
1896
|
+
setActiveFrame(page, matcher);
|
|
1897
|
+
|
|
1898
|
+
let frame;
|
|
1899
|
+
try {
|
|
1900
|
+
frame = await getTargetFrame(page);
|
|
1901
|
+
} catch (e) {
|
|
1902
|
+
clearActiveFrame(page); // don't leave a broken active frame set
|
|
1903
|
+
throw e;
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
return {
|
|
1907
|
+
content: [{
|
|
1908
|
+
type: "text",
|
|
1909
|
+
text: JSON.stringify({
|
|
1910
|
+
active: frame.url(),
|
|
1911
|
+
matcher,
|
|
1912
|
+
frames: listFramesForPage(page),
|
|
1913
|
+
message: `Switched to frame: ${frame.url()}. Subsequent click/type/analyzePage/find/executeScript run inside it until switchFrame() resets.`
|
|
1914
|
+
}, null, 2)
|
|
1915
|
+
}],
|
|
1916
|
+
};
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1691
1919
|
// Figma tools
|
|
1692
1920
|
if (name === "getFigmaFrame") {
|
|
1693
1921
|
const validatedArgs = schemas.GetFigmaFrameSchema.parse(args);
|
|
@@ -2260,10 +2488,12 @@ Start coding now.`;
|
|
|
2260
2488
|
if (name === "smartFindElement") {
|
|
2261
2489
|
const validatedArgs = schemas.SmartFindElementSchema.parse(args);
|
|
2262
2490
|
const page = await getLastOpenPage();
|
|
2491
|
+
const ctx = await getTargetFrame(page); // search inside active frame (P0-1)
|
|
2263
2492
|
const maxResults = validatedArgs.maxResults || 5;
|
|
2493
|
+
const minConfidence = validatedArgs.minConfidence != null ? validatedArgs.minConfidence : 0.6;
|
|
2264
2494
|
|
|
2265
2495
|
// Execute smart search in page context
|
|
2266
|
-
const results = await
|
|
2496
|
+
const results = await ctx.evaluate((description, maxResults, utilsCode, selectorResolverCode) => {
|
|
2267
2497
|
// Inject utilities into page context
|
|
2268
2498
|
eval(utilsCode);
|
|
2269
2499
|
if (typeof registerElement === 'undefined') {
|
|
@@ -2273,25 +2503,38 @@ Start coding now.`;
|
|
|
2273
2503
|
// Determine element type from description
|
|
2274
2504
|
const elementType = determineElementType(description);
|
|
2275
2505
|
|
|
2276
|
-
// Build candidate selectors based on element type
|
|
2277
|
-
|
|
2506
|
+
// Build candidate selectors based on element type.
|
|
2507
|
+
// Dedup with a Set — broadened selectors overlap (a button can match
|
|
2508
|
+
// several queries) and we don't want the same element scored twice.
|
|
2509
|
+
const candidateSet = new Set();
|
|
2510
|
+
const add = (sel) => { try { document.querySelectorAll(sel).forEach(el => candidateSet.add(el)); } catch (e) {} };
|
|
2278
2511
|
|
|
2279
2512
|
if (elementType.type === 'input' || elementType.type === 'any') {
|
|
2280
|
-
|
|
2281
|
-
|
|
2513
|
+
add('input');
|
|
2514
|
+
add('textarea');
|
|
2282
2515
|
}
|
|
2283
2516
|
|
|
2284
2517
|
if (elementType.type === 'button' || elementType.type === 'any') {
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2518
|
+
add('button');
|
|
2519
|
+
add('input[type="submit"]');
|
|
2520
|
+
add('input[type="button"]');
|
|
2521
|
+
add('[role="button"]');
|
|
2522
|
+
// P1-3: broaden to non-semantic clickables that QA hit on real apps —
|
|
2523
|
+
// menu items rendered as div/span[onclick], nav links, tabs.
|
|
2524
|
+
add('[onclick]');
|
|
2525
|
+
add('[role="menuitem"]');
|
|
2526
|
+
add('[role="tab"]');
|
|
2527
|
+
add('nav a');
|
|
2528
|
+
add('[role="navigation"] a');
|
|
2529
|
+
add('[role="menu"] [role="menuitem"]');
|
|
2289
2530
|
}
|
|
2290
2531
|
|
|
2291
2532
|
if (elementType.type === 'link' || elementType.type === 'any') {
|
|
2292
|
-
|
|
2533
|
+
add('a');
|
|
2293
2534
|
}
|
|
2294
2535
|
|
|
2536
|
+
const candidates = Array.from(candidateSet);
|
|
2537
|
+
|
|
2295
2538
|
// Analyze each candidate
|
|
2296
2539
|
const analyzed = candidates.map(el => {
|
|
2297
2540
|
const context = analyzeButtonContextInPage(el);
|
|
@@ -2355,11 +2598,26 @@ Start coding now.`;
|
|
|
2355
2598
|
hints
|
|
2356
2599
|
};
|
|
2357
2600
|
|
|
2358
|
-
// Execute action if provided
|
|
2601
|
+
// Execute action if provided — but only when we're confident (P1-3).
|
|
2602
|
+
// Previously results[0] was clicked unconditionally, so a high-scoring form
|
|
2603
|
+
// submit could be auto-clicked when the user asked for something else.
|
|
2604
|
+
// Guard on minConfidence AND a clear gap over the runner-up.
|
|
2359
2605
|
if (validatedArgs.action && results.length > 0) {
|
|
2360
2606
|
const bestMatch = results[0];
|
|
2607
|
+
const runnerUp = results[1];
|
|
2608
|
+
const gapOk = !runnerUp || (bestMatch.score - runnerUp.score) >= 10;
|
|
2609
|
+
const confident = bestMatch.confidence >= minConfidence && gapOk;
|
|
2610
|
+
|
|
2611
|
+
if (!confident) {
|
|
2612
|
+
response.actionSkipped = {
|
|
2613
|
+
reason: bestMatch.confidence < minConfidence
|
|
2614
|
+
? `bestMatch confidence ${bestMatch.confidence.toFixed(2)} < minConfidence ${minConfidence}`
|
|
2615
|
+
: `ambiguous: top-2 score gap (${bestMatch.score - runnerUp.score}) too small`,
|
|
2616
|
+
hint: 'Refine the description, pass an explicit id/selector to click, or lower minConfidence.',
|
|
2617
|
+
};
|
|
2618
|
+
} else {
|
|
2361
2619
|
try {
|
|
2362
|
-
const actionResult = await executeElementAction(page, bestMatch.selector, validatedArgs.action);
|
|
2620
|
+
const actionResult = await executeElementAction(page, bestMatch.selector, validatedArgs.action, ctx);
|
|
2363
2621
|
response.actionExecuted = actionResult;
|
|
2364
2622
|
|
|
2365
2623
|
// If screenshot was captured, add it to content
|
|
@@ -2374,6 +2632,7 @@ Start coding now.`;
|
|
|
2374
2632
|
} catch (error) {
|
|
2375
2633
|
response.actionError = error.message;
|
|
2376
2634
|
}
|
|
2635
|
+
}
|
|
2377
2636
|
}
|
|
2378
2637
|
|
|
2379
2638
|
return {
|
|
@@ -2392,10 +2651,14 @@ Start coding now.`;
|
|
|
2392
2651
|
if (name === "analyzePage") {
|
|
2393
2652
|
const validatedArgs = schemas.AnalyzePageSchema.parse(args);
|
|
2394
2653
|
const page = await getLastOpenPage();
|
|
2395
|
-
|
|
2654
|
+
// Analyze the active frame (P0-1) — defaults to main frame. `frameList` is
|
|
2655
|
+
// surfaced in the output so the agent can discover/switch frames.
|
|
2656
|
+
const ctx = await getTargetFrame(page);
|
|
2657
|
+
const frameList = listFramesForPage(page);
|
|
2658
|
+
const pageUrl = ctx.url() || page.url();
|
|
2396
2659
|
|
|
2397
2660
|
// Check for non-HTML pages (JSON, plain text, XML) — return raw content instead of empty tree
|
|
2398
|
-
const contentType = await
|
|
2661
|
+
const contentType = await ctx.evaluate(() => document.contentType || '');
|
|
2399
2662
|
if (contentType && !contentType.includes('html')) {
|
|
2400
2663
|
const rawContent = await page.evaluate(() => {
|
|
2401
2664
|
// For JSON/text pages, browser wraps content in <pre> inside <body>
|
|
@@ -2421,7 +2684,7 @@ Start coding now.`;
|
|
|
2421
2684
|
}
|
|
2422
2685
|
|
|
2423
2686
|
// APOM Tree format (default) - v2 with tree structure and positioning
|
|
2424
|
-
const apomResult = await
|
|
2687
|
+
const apomResult = await ctx.evaluate(async (apomTreeConverterCode, selectorResolverCode, modelsCode, shouldRegister, includeAll, viewportOnly, portalOpts) => {
|
|
2425
2688
|
// Inject utilities if not already loaded
|
|
2426
2689
|
if (typeof buildAPOMTree === 'undefined') {
|
|
2427
2690
|
eval(apomTreeConverterCode);
|
|
@@ -2458,7 +2721,7 @@ Start coding now.`;
|
|
|
2458
2721
|
|
|
2459
2722
|
// Build APOM tree
|
|
2460
2723
|
// interactiveOnly = !includeAll (if includeAll is true, we want ALL elements)
|
|
2461
|
-
const apomData = buildAPOMTree(!includeAll, viewportOnly);
|
|
2724
|
+
const apomData = buildAPOMTree(!includeAll, viewportOnly, portalOpts);
|
|
2462
2725
|
|
|
2463
2726
|
// Register elements in selector resolver if requested
|
|
2464
2727
|
if (shouldRegister) {
|
|
@@ -2491,7 +2754,17 @@ Start coding now.`;
|
|
|
2491
2754
|
}
|
|
2492
2755
|
|
|
2493
2756
|
return apomData;
|
|
2494
|
-
}, apomTreeConverter, selectorResolver, elementModelBase + '\n' + elementModels + '\n' + modelRegistry, validatedArgs.registerElements !== false, validatedArgs.includeAll || false, validatedArgs.viewportOnly || false
|
|
2757
|
+
}, apomTreeConverter, selectorResolver, elementModelBase + '\n' + elementModels + '\n' + modelRegistry, validatedArgs.registerElements !== false, validatedArgs.includeAll || false, validatedArgs.viewportOnly || false, {
|
|
2758
|
+
include: validatedArgs.includePortals !== false,
|
|
2759
|
+
selectors: validatedArgs.portalSelectors || undefined
|
|
2760
|
+
});
|
|
2761
|
+
|
|
2762
|
+
// Surface frames so the agent can discover/switch into iframes (P0-1).
|
|
2763
|
+
// Only when there's more than the main frame — keeps single-frame output lean.
|
|
2764
|
+
if (frameList.length > 1) {
|
|
2765
|
+
apomResult.frames = frameList;
|
|
2766
|
+
apomResult.activeFrame = ctx === page.mainFrame() ? null : (ctx.url() || true);
|
|
2767
|
+
}
|
|
2495
2768
|
|
|
2496
2769
|
// Handle diff mode
|
|
2497
2770
|
if (validatedArgs.diff) {
|
|
@@ -2515,7 +2788,8 @@ Start coding now.`;
|
|
|
2515
2788
|
previousTimestamp: previousAnalysis.timestamp,
|
|
2516
2789
|
diff,
|
|
2517
2790
|
metadata: apomResult.metadata,
|
|
2518
|
-
alerts: apomResult.alerts
|
|
2791
|
+
alerts: apomResult.alerts,
|
|
2792
|
+
frames: frameList.length > 1 ? frameList : undefined
|
|
2519
2793
|
})
|
|
2520
2794
|
}]
|
|
2521
2795
|
};
|
|
@@ -2763,8 +3037,9 @@ Start coding now.`;
|
|
|
2763
3037
|
if (name === "findElementsByText") {
|
|
2764
3038
|
const validatedArgs = schemas.FindElementsByTextSchema.parse(args);
|
|
2765
3039
|
const page = await getLastOpenPage();
|
|
3040
|
+
const ctx = await getTargetFrame(page); // search inside active frame (P0-1)
|
|
2766
3041
|
|
|
2767
|
-
const elements = await
|
|
3042
|
+
const elements = await ctx.evaluate((text, exact, caseSensitive, utilsCode, selectorResolverCode) => {
|
|
2768
3043
|
eval(utilsCode);
|
|
2769
3044
|
if (typeof registerElement === 'undefined') {
|
|
2770
3045
|
eval(selectorResolverCode);
|
|
@@ -2830,11 +3105,14 @@ Start coding now.`;
|
|
|
2830
3105
|
truncated: elements.length > 20
|
|
2831
3106
|
};
|
|
2832
3107
|
|
|
2833
|
-
// Execute action if provided and elements found
|
|
3108
|
+
// Execute action if provided and elements found.
|
|
3109
|
+
// Pass `ctx` so the raw selector resolves inside the active frame (P0-1) —
|
|
3110
|
+
// the FIND ran in the frame, so the ACTION must too, otherwise page.$
|
|
3111
|
+
// looks in the main frame and "Element not found for action".
|
|
2834
3112
|
if (validatedArgs.action && elements.length > 0) {
|
|
2835
3113
|
const firstMatch = elements[0];
|
|
2836
3114
|
try {
|
|
2837
|
-
const actionResult = await executeElementAction(page, firstMatch.selector, validatedArgs.action);
|
|
3115
|
+
const actionResult = await executeElementAction(page, firstMatch.selector, validatedArgs.action, ctx);
|
|
2838
3116
|
response.actionExecuted = actionResult;
|
|
2839
3117
|
|
|
2840
3118
|
// If screenshot was captured, add it to content
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "chrometools-mcp",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.6.0",
|
|
4
4
|
"description": "MCP (Model Context Protocol) server for Chrome automation using Puppeteer. Persistent browser sessions, UI framework detection (MUI, Ant Design, etc.), Page Object support, visual testing, Figma comparison. Works seamlessly in WSL, Linux, macOS, and Windows.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|