@mmerterden/multi-agent-toolkit-mcp 3.13.1 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +192 -5
- package/README.md +80 -8
- package/README.tr.md +104 -8
- package/index.js +526 -162
- package/package.json +4 -4
- package/tools/context/index.js +34 -18
- package/tools/design-check/component-walk.js +359 -0
- package/tools/design-check/content-cardinality.js +3 -3
- package/tools/design-check/index.js +78 -11
- package/tools/design-check/report.js +152 -10
- package/tools/design-check/scan.js +1 -1
- package/tools/design-check/scenario-inventory.js +404 -50
- package/tools/design-check/visual-compare.js +17 -2
- package/tools/ios-app-store-audit/context.js +3 -3
- package/tools/ios-app-store-audit/exec.js +17 -0
- package/tools/ios-app-store-audit/index.js +0 -15
- package/tools/ios-app-store-audit/rules/code-signing.js +2 -2
- package/tools/ios-app-store-audit/rules/dead-reference.js +2 -2
- package/tools/ios-app-store-audit/rules/debug-tool-leak.js +2 -2
- package/tools/ios-app-store-audit/rules/embedded-sdk.js +3 -3
- package/tools/ios-app-store-audit/rules/extension-signing.js +2 -2
- package/tools/ios-app-store-audit/rules/ipv6-compliance.js +2 -2
- package/tools/ios-app-store-audit/rules/production-hygiene.js +2 -2
- package/tools/ios-app-store-audit/rules/provisioning-profile.js +3 -3
- package/tools/ios-app-store-audit/rules/required-reason-api.js +2 -2
- package/tools/offload/index.js +7 -3
- package/tools/policy/egress-proxy.js +268 -0
- package/tools/policy/index.js +283 -0
- package/tools/security/cvss.js +108 -0
- package/tools/security/deps.js +0 -0
- package/tools/security/index.js +115 -0
- package/tools/spawn-collect/index.js +141 -0
package/index.js
CHANGED
|
@@ -21,7 +21,6 @@ import { join, dirname, basename, isAbsolute, resolve, sep } from "path";
|
|
|
21
21
|
import { homedir, tmpdir } from "os";
|
|
22
22
|
import { createHash } from "crypto";
|
|
23
23
|
import { createRequire } from "node:module";
|
|
24
|
-
import * as ctxIndex from "./tools/context/index.js";
|
|
25
24
|
|
|
26
25
|
// This file is ESM ("type": "module"), so `require` does not exist here. The
|
|
27
26
|
// optional readability helpers are resolved through createRequire: absent, the
|
|
@@ -46,11 +45,15 @@ import {
|
|
|
46
45
|
shutdownAllLsp,
|
|
47
46
|
} from "./tools/code-intel/index.js";
|
|
48
47
|
import { PASS_TOOLS, handlePass, PASS_READ_ONLY, PASS_OUTPUT_SCHEMAS } from "./tools/pass-kit/index.js";
|
|
48
|
+
import { SECURITY_TOOLS, handleSecurity, SECURITY_OUTPUT_SCHEMAS } from "./tools/security/index.js";
|
|
49
49
|
import { parseLaunchOutput } from "./tools/launch-time/index.js";
|
|
50
50
|
import { parseLeaksOutput, parseMeminfoOutput, diffMeminfo } from "./tools/memory/index.js";
|
|
51
51
|
import { auditIosTree, auditAndroidDump, parseAuditResults } from "./tools/a11y/index.js";
|
|
52
52
|
import { interactiveElements } from "./tools/ui-inspect/index.js";
|
|
53
53
|
import { selectCrashReports } from "./tools/crash-logs/index.js";
|
|
54
|
+
import { urlPolicy, urlPolicyError, urlAllowList, indexDenyList, indexDenied as pathDenied, strictLaunchOptions } from "./tools/policy/index.js";
|
|
55
|
+
import { startEgressProxy } from "./tools/policy/egress-proxy.js";
|
|
56
|
+
import { spawnCollect, killAllSpawned } from "./tools/spawn-collect/index.js";
|
|
54
57
|
import {
|
|
55
58
|
offloadLargeText,
|
|
56
59
|
queryOffloadedOutput,
|
|
@@ -66,6 +69,14 @@ if (!existsSync(SCREENSHOT_DIR)) mkdirSync(SCREENSHOT_DIR, { recursive: true });
|
|
|
66
69
|
// ever removed them.
|
|
67
70
|
pruneWorkDir(SCREENSHOT_DIR);
|
|
68
71
|
|
|
72
|
+
// Read once at startup; see tools/policy for what strict refuses. Unset, both
|
|
73
|
+
// checks return early and nothing is refused.
|
|
74
|
+
const URL_POLICY = urlPolicy();
|
|
75
|
+
const URL_ALLOW = URL_POLICY === "strict" ? urlAllowList() : [];
|
|
76
|
+
const INDEX_DENY = indexDenyList({ policy: URL_POLICY });
|
|
77
|
+
const checkUrl = (url) => urlPolicyError(url, { policy: URL_POLICY, allow: URL_ALLOW });
|
|
78
|
+
const indexDenied = (path) => pathDenied(path, INDEX_DENY);
|
|
79
|
+
|
|
69
80
|
// ── Helpers ──
|
|
70
81
|
|
|
71
82
|
// Marker prefix for command failures. The CallTool dispatch turns any result
|
|
@@ -156,67 +167,6 @@ function runAsync(cmd, opts = {}) {
|
|
|
156
167
|
});
|
|
157
168
|
}
|
|
158
169
|
|
|
159
|
-
// spawn-based collector for the long-runners that also want incremental output
|
|
160
|
-
// (progress heartbeats read the last line). Interleaves stdout+stderr the way a
|
|
161
|
-
// terminal would, honors an AbortSignal, and never rejects.
|
|
162
|
-
//
|
|
163
|
-
// Output is capped at the 64MB the execSync it replaced enforced via maxBuffer:
|
|
164
|
-
// unbounded `output += chunk` on a verbose xcodebuild can exceed V8's max
|
|
165
|
-
// string length, and that throw fires inside a stream 'data' handler - outside
|
|
166
|
-
// the CallTool try/catch - killing the whole stdio server. The oldest chunks
|
|
167
|
-
// are dropped; errors and test verdicts land at the tail of a build log.
|
|
168
|
-
const SPAWN_OUTPUT_CAP = 64 * 1024 * 1024;
|
|
169
|
-
|
|
170
|
-
function spawnCollect(cmd, { timeout = 600000, signal, env, onLine } = {}) {
|
|
171
|
-
return new Promise((resolve) => {
|
|
172
|
-
const child = spawn("/bin/sh", ["-c", cmd], { env });
|
|
173
|
-
const chunks = [];
|
|
174
|
-
let size = 0;
|
|
175
|
-
let truncated = false;
|
|
176
|
-
let timedOut = false;
|
|
177
|
-
let aborted = false;
|
|
178
|
-
const timer = setTimeout(() => { timedOut = true; child.kill("SIGTERM"); }, timeout);
|
|
179
|
-
const onAbort = () => { aborted = true; child.kill("SIGTERM"); };
|
|
180
|
-
if (signal) {
|
|
181
|
-
if (signal.aborted) onAbort();
|
|
182
|
-
else signal.addEventListener("abort", onAbort, { once: true });
|
|
183
|
-
}
|
|
184
|
-
const collect = (chunk) => {
|
|
185
|
-
const s = String(chunk);
|
|
186
|
-
chunks.push(s);
|
|
187
|
-
size += s.length;
|
|
188
|
-
while (size > SPAWN_OUTPUT_CAP && chunks.length > 1) {
|
|
189
|
-
size -= chunks[0].length;
|
|
190
|
-
chunks.shift();
|
|
191
|
-
truncated = true;
|
|
192
|
-
}
|
|
193
|
-
if (size > SPAWN_OUTPUT_CAP) {
|
|
194
|
-
chunks[0] = chunks[0].slice(size - SPAWN_OUTPUT_CAP);
|
|
195
|
-
size = SPAWN_OUTPUT_CAP;
|
|
196
|
-
truncated = true;
|
|
197
|
-
}
|
|
198
|
-
if (onLine) {
|
|
199
|
-
const lines = s.split("\n");
|
|
200
|
-
for (let i = lines.length - 1; i >= 0; i--) {
|
|
201
|
-
const line = lines[i].trim();
|
|
202
|
-
if (line) { onLine(line); break; }
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
};
|
|
206
|
-
child.stdout.on("data", collect);
|
|
207
|
-
child.stderr.on("data", collect);
|
|
208
|
-
child.on("error", (e) => {
|
|
209
|
-
clearTimeout(timer);
|
|
210
|
-
resolve({ code: 127, output: `${chunks.join("")}\n${e.message}`, truncated, timedOut, aborted });
|
|
211
|
-
});
|
|
212
|
-
child.on("close", (code) => {
|
|
213
|
-
clearTimeout(timer);
|
|
214
|
-
if (signal) signal.removeEventListener("abort", onAbort);
|
|
215
|
-
resolve({ code: code ?? 1, output: chunks.join(""), truncated, timedOut, aborted });
|
|
216
|
-
});
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
|
|
220
170
|
function startHeartbeat(ctx, label, detail) {
|
|
221
171
|
if (!ctx?.progress) return () => {};
|
|
222
172
|
const started = Date.now();
|
|
@@ -244,11 +194,31 @@ function safeRelOutput(name, baseDir) {
|
|
|
244
194
|
return full;
|
|
245
195
|
}
|
|
246
196
|
|
|
197
|
+
// Bundle IDs / package names: reverse-DNS characters only (alphanumeric, dots,
|
|
198
|
+
// underscores, hyphens). Absent (undefined/null) passes through for the callers
|
|
199
|
+
// that treat it as optional, but a PRESENT value must name something: an empty
|
|
200
|
+
// string makes `simctl privacy <d> reset all <id>` reset every app on the
|
|
201
|
+
// device, and a leading dash reaches the CLI as an option instead of an
|
|
202
|
+
// operand. A slash is refused because `defaults write <id>` reads a value
|
|
203
|
+
// containing one as a plist path, so a bundle id of /tmp/x writes a file. The
|
|
204
|
+
// one place a slash is legitimate, an Android package/activity component, has
|
|
205
|
+
// its own validator.
|
|
247
206
|
function sanitizeId(id) {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
if (!/^[a-zA-Z0-9._
|
|
251
|
-
return
|
|
207
|
+
if (id === undefined || id === null) return id;
|
|
208
|
+
const s = String(id);
|
|
209
|
+
if (!/^[a-zA-Z0-9._][a-zA-Z0-9._-]*$/.test(s)) throw new Error(`Invalid bundle id or package name: ${JSON.stringify(s)}`);
|
|
210
|
+
return s;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// An Android activity: a class name, relative (.Main) or qualified, optionally
|
|
214
|
+
// prefixed with its package as package/activity. At most one slash, never at
|
|
215
|
+
// either end.
|
|
216
|
+
function androidActivity(activity) {
|
|
217
|
+
const s = String(activity ?? "");
|
|
218
|
+
if (!/^[a-zA-Z0-9._][a-zA-Z0-9._-]*(\/[a-zA-Z0-9._][a-zA-Z0-9._-]*)?$/.test(s)) {
|
|
219
|
+
throw new Error(`Invalid activity: ${JSON.stringify(s)}`);
|
|
220
|
+
}
|
|
221
|
+
return s;
|
|
252
222
|
}
|
|
253
223
|
|
|
254
224
|
// Single-quote a value for POSIX sh. Every command here is built as a string and
|
|
@@ -431,6 +401,17 @@ const HAS_ADB = hasCommand("adb");
|
|
|
431
401
|
// two simctl recorders on one simulator fight over the io channel.
|
|
432
402
|
const RECORDINGS = new Map();
|
|
433
403
|
|
|
404
|
+
// Resolves once the child has either started or failed to. A spawn failure
|
|
405
|
+
// (the binary is gone, not executable) arrives as an 'error' event, and an
|
|
406
|
+
// 'error' nobody listens for is thrown as an uncaught exception - which takes
|
|
407
|
+
// the whole stdio server down one tick after the start call answered "started".
|
|
408
|
+
function spawnStarted(child) {
|
|
409
|
+
return new Promise((resolve) => {
|
|
410
|
+
child.once("spawn", () => resolve(null));
|
|
411
|
+
child.once("error", (e) => resolve(e));
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
|
|
434
415
|
function fileResult(text, ...paths) {
|
|
435
416
|
return {
|
|
436
417
|
type: "file",
|
|
@@ -659,6 +640,8 @@ async function handleIOS(name, args, ctx = {}) {
|
|
|
659
640
|
const recParent = dirname(f);
|
|
660
641
|
if (!existsSync(recParent)) return `${ERROR_PREFIX}directory does not exist: ${recParent}`;
|
|
661
642
|
const child = spawn("xcrun", ["simctl", "io", d, "recordVideo", `--codec=${args.codec || "hevc"}`, "--force", f], { stdio: "ignore" });
|
|
643
|
+
const spawnError = await spawnStarted(child);
|
|
644
|
+
if (spawnError) return `${ERROR_PREFIX}could not start xcrun simctl io recordVideo: ${spawnError.message}`;
|
|
662
645
|
child.once("close", () => { if (RECORDINGS.get(key)?.child === child) RECORDINGS.delete(key); });
|
|
663
646
|
RECORDINGS.set(key, { child, path: f });
|
|
664
647
|
return `Recording started on ${d} (pid ${child.pid}) -> ${f}\nCall ios_record_video with action:"stop" to finish.`;
|
|
@@ -1121,7 +1104,7 @@ function adbFlag(id) { return id ? `-s ${deviceSerial(id)}` : ""; }
|
|
|
1121
1104
|
// carries the package (com.x/.Main) is used as it is.
|
|
1122
1105
|
function androidComponent(packageName, activity) {
|
|
1123
1106
|
const pkg = sanitizeId(packageName);
|
|
1124
|
-
const act =
|
|
1107
|
+
const act = androidActivity(activity);
|
|
1125
1108
|
return act.includes("/") ? act : `${pkg}/${act}`;
|
|
1126
1109
|
}
|
|
1127
1110
|
|
|
@@ -1203,7 +1186,14 @@ async function handleAndroid(name, args, ctx = {}) {
|
|
|
1203
1186
|
case "android_key_event": return run(`adb ${df} shell input keyevent ${token(args.keycode, "keycode")}`) || `Key ${args.keycode}`;
|
|
1204
1187
|
case "android_launch_app": return args.activity ? run(`adb ${df} shell am start -n ${androidComponent(args.package_name, args.activity)}`) : run(`adb ${df} shell monkey -p ${sanitizeId(args.package_name)} -c android.intent.category.LAUNCHER 1`) || `Launched`;
|
|
1205
1188
|
case "android_stop_app": return run(`adb ${df} shell am force-stop ${sanitizeId(args.package_name)}`) || "Stopped";
|
|
1206
|
-
case "android_list_packages": {
|
|
1189
|
+
case "android_list_packages": {
|
|
1190
|
+
const out = run(`adb ${df} shell pm list packages`);
|
|
1191
|
+
// The filter applies to package lines, not to a failure: filtering the
|
|
1192
|
+
// ERROR text by a package name would answer with an empty success.
|
|
1193
|
+
if (isFailure(out) || !args.filter) return out;
|
|
1194
|
+
const wanted = String(args.filter).toLowerCase();
|
|
1195
|
+
return out.split("\n").filter((l) => l.toLowerCase().includes(wanted)).join("\n");
|
|
1196
|
+
}
|
|
1207
1197
|
case "android_go_home": return run(`adb ${df} shell input keyevent 3`) || "Home";
|
|
1208
1198
|
case "android_go_back": return run(`adb ${df} shell input keyevent 4`) || "Back";
|
|
1209
1199
|
case "android_get_ui_tree": {
|
|
@@ -1314,6 +1304,8 @@ async function handleAndroid(name, args, ctx = {}) {
|
|
|
1314
1304
|
if (!existsSync(parent)) return `${ERROR_PREFIX}directory does not exist: ${parent}`;
|
|
1315
1305
|
const argv = [...(args.device_id ? ["-s", deviceSerial(args.device_id)] : []), "shell", "screenrecord", "--time-limit", String(dur), remotePath];
|
|
1316
1306
|
const child = spawn("adb", argv, { stdio: "ignore" });
|
|
1307
|
+
const spawnError = await spawnStarted(child);
|
|
1308
|
+
if (spawnError) return `${ERROR_PREFIX}could not start adb screenrecord: ${spawnError.message}`;
|
|
1317
1309
|
const entry = { child, remotePath, path: localPath, exited: false };
|
|
1318
1310
|
child.once("close", () => { if (RECORDINGS.get(key) === entry) entry.exited = true; });
|
|
1319
1311
|
RECORDINGS.set(key, entry);
|
|
@@ -1485,13 +1477,121 @@ async function handleAndroid(name, args, ctx = {}) {
|
|
|
1485
1477
|
let _browser = null;
|
|
1486
1478
|
let _page = null;
|
|
1487
1479
|
let _engine = null;
|
|
1480
|
+
let _userAgent = null;
|
|
1488
1481
|
|
|
1489
1482
|
// A request for a different engine closes the current browser and relaunches;
|
|
1490
1483
|
// a partial launch (newContext/newPage threw) is closed rather than left as a
|
|
1491
1484
|
// browser with no page that the next call would launch beside.
|
|
1492
|
-
|
|
1485
|
+
//
|
|
1486
|
+
// The user agent belongs to the browser context: set there, it reaches every
|
|
1487
|
+
// request and navigator.userAgent alike. A header override would change only
|
|
1488
|
+
// the former. So a web_goto naming a DIFFERENT user_agent starts a fresh
|
|
1489
|
+
// context, and with it drops the previous one's cookies, storage and tabs; one
|
|
1490
|
+
// that omits it keeps whatever context is open.
|
|
1491
|
+
// ── Strict URL policy inside the browser ─────────────────────────────
|
|
1492
|
+
//
|
|
1493
|
+
// Checking the URL handed to web_goto is not enough: a script, a click, a
|
|
1494
|
+
// redirect, a popup, a fetch or an <img> reaches a host without passing
|
|
1495
|
+
// through it. Under the strict policy every context gets a route that runs the
|
|
1496
|
+
// same check on every request the context makes - navigations, subresources,
|
|
1497
|
+
// popups, WebSockets - and aborts the refused ones before they leave the
|
|
1498
|
+
// browser. Service workers are blocked, because their requests bypass routing.
|
|
1499
|
+
// Routing disables the HTTP cache, which is why it is not installed by default.
|
|
1500
|
+
const REFUSAL_CAP = 50;
|
|
1501
|
+
let _refusals = [];
|
|
1502
|
+
let _refusalSeq = 0;
|
|
1503
|
+
|
|
1504
|
+
function recordRefusal(url, reason, navigation) {
|
|
1505
|
+
_refusals.push({ seq: ++_refusalSeq, url: String(url).slice(0, 300), reason, navigation });
|
|
1506
|
+
if (_refusals.length > REFUSAL_CAP) _refusals.shift();
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
const refusalsSince = (seq) => _refusals.filter((r) => r.seq > seq);
|
|
1510
|
+
|
|
1511
|
+
// data:, blob: and about: never reach the network.
|
|
1512
|
+
const LOCAL_SCHEME = /^(data|blob|about):/i;
|
|
1513
|
+
|
|
1514
|
+
async function guardContext(ctx) {
|
|
1515
|
+
await ctx.route("**/*", async (route) => {
|
|
1516
|
+
const req = route.request();
|
|
1517
|
+
const url = req.url();
|
|
1518
|
+
const refused = LOCAL_SCHEME.test(url) ? null : await checkUrl(url);
|
|
1519
|
+
if (refused) {
|
|
1520
|
+
recordRefusal(url, refused, req.isNavigationRequest());
|
|
1521
|
+
return route.abort("blockedbyclient").catch(() => {});
|
|
1522
|
+
}
|
|
1523
|
+
return route.fallback().catch(() => {});
|
|
1524
|
+
});
|
|
1525
|
+
await ctx.routeWebSocket(/.*/, async (ws) => {
|
|
1526
|
+
const url = ws.url();
|
|
1527
|
+
const refused = await checkUrl(url.replace(/^ws(s?):/i, "http$1:"));
|
|
1528
|
+
if (refused) {
|
|
1529
|
+
recordRefusal(url, refused, false);
|
|
1530
|
+
await ws.close({ code: 1008, reason: "refused by the URL policy" }).catch(() => {});
|
|
1531
|
+
return;
|
|
1532
|
+
}
|
|
1533
|
+
ws.connectToServer();
|
|
1534
|
+
});
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
// The route above never sees the hops of a redirect chain; the proxy does.
|
|
1538
|
+
// Started with the first strict browser and kept for the life of the process
|
|
1539
|
+
// (it is unref'd and listens on 127.0.0.1 only).
|
|
1540
|
+
let _egressProxy = null;
|
|
1541
|
+
async function egressProxy() {
|
|
1542
|
+
_egressProxy ??= startEgressProxy({
|
|
1543
|
+
check: (url, lookup) => urlPolicyError(url, { policy: URL_POLICY, allow: URL_ALLOW, lookup }),
|
|
1544
|
+
onRefuse: (url, reason) => recordRefusal(url, reason, null),
|
|
1545
|
+
}).catch((e) => { _egressProxy = null; throw e; });
|
|
1546
|
+
return _egressProxy;
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
async function launchOptions(engineName) {
|
|
1550
|
+
if (URL_POLICY !== "strict") return { headless: true };
|
|
1551
|
+
const proxy = await egressProxy();
|
|
1552
|
+
return strictLaunchOptions(engineName, proxy.url);
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
// WebKit has no launch switch for WebRTC, so under the strict policy its pages
|
|
1556
|
+
// get no peer-connection constructors at all.
|
|
1557
|
+
const NO_PEER_CONNECTION = `for (const k of ["RTCPeerConnection", "webkitRTCPeerConnection", "RTCDataChannel"]) { try { delete window[k]; } catch {} }`;
|
|
1558
|
+
|
|
1559
|
+
async function newContext(agent) {
|
|
1560
|
+
const strict = URL_POLICY === "strict";
|
|
1561
|
+
const ctx = await _browser.newContext({
|
|
1562
|
+
...(agent ? { userAgent: agent } : {}),
|
|
1563
|
+
...(strict ? { serviceWorkers: "block" } : {}),
|
|
1564
|
+
});
|
|
1565
|
+
if (strict) {
|
|
1566
|
+
try {
|
|
1567
|
+
if (_browser.browserType().name() === "webkit") await ctx.addInitScript(NO_PEER_CONNECTION);
|
|
1568
|
+
await guardContext(ctx);
|
|
1569
|
+
} catch (e) {
|
|
1570
|
+
await ctx.close().catch(() => {});
|
|
1571
|
+
throw e;
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
return ctx;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
async function ensureBrowser(browserType, userAgent) {
|
|
1493
1578
|
const wanted = browserType || _engine || "chromium";
|
|
1494
|
-
|
|
1579
|
+
const agent = userAgent === undefined || userAgent === null || userAgent === "" ? _userAgent : String(userAgent);
|
|
1580
|
+
if (_browser && _page && _engine === wanted && agent === _userAgent) return _page;
|
|
1581
|
+
if (_browser && _page && _engine === wanted) {
|
|
1582
|
+
try { await _page.context().close(); } catch {}
|
|
1583
|
+
_page = null;
|
|
1584
|
+
try {
|
|
1585
|
+
const ctx = await newContext(agent);
|
|
1586
|
+
_page = await ctx.newPage();
|
|
1587
|
+
observePage(_page);
|
|
1588
|
+
} catch (e) {
|
|
1589
|
+
await closeBrowser();
|
|
1590
|
+
throw e;
|
|
1591
|
+
}
|
|
1592
|
+
_userAgent = agent;
|
|
1593
|
+
return _page;
|
|
1594
|
+
}
|
|
1495
1595
|
if (_browser || _page) await closeBrowser();
|
|
1496
1596
|
let pw;
|
|
1497
1597
|
try {
|
|
@@ -1502,9 +1602,9 @@ async function ensureBrowser(browserType) {
|
|
|
1502
1602
|
const engines = { chromium: pw.chromium, webkit: pw.webkit, firefox: pw.firefox };
|
|
1503
1603
|
const engine = engines[wanted];
|
|
1504
1604
|
if (!engine) throw new Error(`unknown browser engine: ${wanted}`);
|
|
1505
|
-
_browser = await engine.launch(
|
|
1605
|
+
_browser = await engine.launch(await launchOptions(wanted));
|
|
1506
1606
|
try {
|
|
1507
|
-
const ctx = await
|
|
1607
|
+
const ctx = await newContext(agent);
|
|
1508
1608
|
_page = await ctx.newPage();
|
|
1509
1609
|
observePage(_page);
|
|
1510
1610
|
} catch (e) {
|
|
@@ -1512,6 +1612,7 @@ async function ensureBrowser(browserType) {
|
|
|
1512
1612
|
throw e;
|
|
1513
1613
|
}
|
|
1514
1614
|
_engine = wanted;
|
|
1615
|
+
_userAgent = agent;
|
|
1515
1616
|
return _page;
|
|
1516
1617
|
}
|
|
1517
1618
|
|
|
@@ -1521,6 +1622,7 @@ async function closeBrowser() {
|
|
|
1521
1622
|
_page = null;
|
|
1522
1623
|
_browser = null;
|
|
1523
1624
|
_engine = null;
|
|
1625
|
+
_userAgent = null;
|
|
1524
1626
|
}
|
|
1525
1627
|
|
|
1526
1628
|
// ── Page observation ─────────────────────────────────────────────────
|
|
@@ -1766,6 +1868,10 @@ function turndownBundlePath(resolved) {
|
|
|
1766
1868
|
function blankPageError(page, tool) {
|
|
1767
1869
|
const url = page.url();
|
|
1768
1870
|
if (url && url !== "about:blank" && !url.startsWith("chrome-error://")) return null;
|
|
1871
|
+
// Under strict, a navigation the policy aborted leaves the browser's error
|
|
1872
|
+
// page behind; naming the refusal says why better than "nothing opened".
|
|
1873
|
+
const refused = url.startsWith("chrome-error://") ? [..._refusals].reverse().find((r) => r.navigation) : null;
|
|
1874
|
+
if (refused) return `ERROR: ${tool} has no page to read: the last navigation, to ${refused.url}, was refused. ${refused.reason}`;
|
|
1769
1875
|
return `ERROR: ${tool} reads the page that is currently open, and nothing has been opened yet (${url || "no url"}). Call web_goto first.`;
|
|
1770
1876
|
}
|
|
1771
1877
|
|
|
@@ -1867,7 +1973,7 @@ function locate(page, args) {
|
|
|
1867
1973
|
}
|
|
1868
1974
|
|
|
1869
1975
|
const WEB_TOOLS = [
|
|
1870
|
-
{ name: "web_goto", description: "Open a URL in a headless browser (Playwright). Reuses a single browser instance across calls.", inputSchema: { type: "object", properties: { url: { type: "string" }, browser: { type: "string", enum: ["chromium", "webkit", "firefox"], description: "Default chromium; use webkit for Safari-like behavior" }, wait_until: { type: "string", enum: ["load", "domcontentloaded", "networkidle"] }, timeout_ms: { type: "number", description: "Navigation timeout. Default 30000." }, viewport: { type: "string", description: "WIDTHxHEIGHT, e.g. 390x844 for a phone-sized layout." }, locale: { type: "string", description: "Accept-Language for this navigation, e.g. tr-TR." }, user_agent: { type: "string" } }, required: ["url"] } },
|
|
1976
|
+
{ name: "web_goto", description: "Open a URL in a headless browser (Playwright). Reuses a single browser instance across calls.", inputSchema: { type: "object", properties: { url: { type: "string" }, browser: { type: "string", enum: ["chromium", "webkit", "firefox"], description: "Default chromium; use webkit for Safari-like behavior" }, wait_until: { type: "string", enum: ["load", "domcontentloaded", "networkidle"] }, timeout_ms: { type: "number", description: "Navigation timeout. Default 30000." }, viewport: { type: "string", description: "WIDTHxHEIGHT, e.g. 390x844 for a phone-sized layout." }, locale: { type: "string", description: "Accept-Language for this navigation, e.g. tr-TR." }, user_agent: { type: "string", description: "User-Agent for the browser context, seen by requests and navigator.userAgent alike. A value different from the current one starts a fresh context, discarding its cookies, storage and tabs; omit it to keep the current context." } }, required: ["url"] } },
|
|
1871
1977
|
{ name: "web_screenshot", description: "Capture a screenshot of the current page. Returns a base64 PNG by default; pass `path` to write the file and return only its location.", inputSchema: { type: "object", properties: { full_page: { type: "boolean" }, path: { type: "string", description: "Absolute file path to write the PNG to. The parent directory must already exist. Returns the path instead of the image." } } } },
|
|
1872
1978
|
{ name: "web_click", description: "Click an element by CSS selector, text, or a [ref=eN] from web_snapshot. Auto-waits for the element.", inputSchema: { type: "object", properties: { selector: { type: "string", description: "CSS selector, or 'text=...' / 'role=...'" }, ref: { type: "string", description: "A ref from the most recent web_snapshot, e.g. e7. Use this instead of selector." }, timeout_ms: { type: "number" } } } },
|
|
1873
1979
|
{ name: "web_type", description: "Type text into an input matched by a selector or a [ref=eN] from web_snapshot.", inputSchema: { type: "object", properties: { selector: { type: "string" }, ref: { type: "string", description: "A ref from the most recent web_snapshot." }, text: { type: "string" }, clear_first: { type: "boolean" } }, required: ["text"] } },
|
|
@@ -1888,9 +1994,121 @@ const WEB_TOOLS = [
|
|
|
1888
1994
|
{ name: "web_select_option", description: "Choose an option in a <select>, by value or by visible label.", inputSchema: { type: "object", properties: { selector: { type: "string" }, ref: { type: "string" }, value: { type: "string" }, label: { type: "string" } } } },
|
|
1889
1995
|
];
|
|
1890
1996
|
|
|
1997
|
+
// The policy verdict on where the page is now. Only a network or file URL is
|
|
1998
|
+
// judged: about:blank and the browser's own error page hold nothing a refused
|
|
1999
|
+
// host served.
|
|
2000
|
+
async function landedRefusal(page) {
|
|
2001
|
+
const url = page.url();
|
|
2002
|
+
if (!/^(https?|wss?|ftp|file):/i.test(url)) return null;
|
|
2003
|
+
return checkUrl(url);
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
// The crawl loop proper. handleWeb routes the crawl's User-Agent around it and
|
|
2007
|
+
// removes the route afterwards, whatever this returns or throws.
|
|
2008
|
+
async function crawlFrom(page, { start, origin, maxPages, maxDepth, delay, respectRobots, wantIndex, indexDb, crawlDir, ctxIndex }) {
|
|
2009
|
+
let indexed = 0;
|
|
2010
|
+
const disallowed = respectRobots ? await robotsDisallow(page, origin) : [];
|
|
2011
|
+
const seen = new Set([start]);
|
|
2012
|
+
const queue = [{ url: start, depth: 0 }];
|
|
2013
|
+
const out = [];
|
|
2014
|
+
while (queue.length && out.length < maxPages) {
|
|
2015
|
+
const { url, depth } = queue.shift();
|
|
2016
|
+
if (disallowed.some((rule) => url.startsWith(origin + rule))) continue;
|
|
2017
|
+
const refused = await checkUrl(url);
|
|
2018
|
+
if (refused) {
|
|
2019
|
+
out.push(`--- ${url}\nERROR: ${refused}`);
|
|
2020
|
+
continue;
|
|
2021
|
+
}
|
|
2022
|
+
let gotoError = null;
|
|
2023
|
+
try {
|
|
2024
|
+
await page.goto(url, { waitUntil: "load", timeout: 15000 });
|
|
2025
|
+
} catch (e) {
|
|
2026
|
+
gotoError = e;
|
|
2027
|
+
}
|
|
2028
|
+
// A redirect can land somewhere the queued URL did not name. Such a page is
|
|
2029
|
+
// neither read, indexed nor followed.
|
|
2030
|
+
const landed = await landedRefusal(page);
|
|
2031
|
+
if (landed) {
|
|
2032
|
+
out.push(`--- ${url}\nERROR: redirected to a refused address. ${landed}`);
|
|
2033
|
+
await page.goto("about:blank").catch(() => {});
|
|
2034
|
+
continue;
|
|
2035
|
+
}
|
|
2036
|
+
if (gotoError) {
|
|
2037
|
+
out.push(`--- ${url}\nERROR: ${gotoError.message}`);
|
|
2038
|
+
continue;
|
|
2039
|
+
}
|
|
2040
|
+
const doc = await extractReadable(page);
|
|
2041
|
+
out.push(`--- ${url}\n${doc.markdown}`);
|
|
2042
|
+
// A crawl that cannot be searched afterwards is a wall of text: twenty
|
|
2043
|
+
// pages arrive at once and the caller has no way to ask which of them
|
|
2044
|
+
// answers the question. Each page is written beside the index and
|
|
2045
|
+
// indexed under its URL, so context_search ranks the crawl the same way
|
|
2046
|
+
// it ranks an offloaded payload. Failure here never fails the crawl -
|
|
2047
|
+
// the pages are already in the reply.
|
|
2048
|
+
if (wantIndex && indexDb) {
|
|
2049
|
+
try {
|
|
2050
|
+
const slug = url.replace(/[^a-z0-9]+/gi, "-").slice(0, 120);
|
|
2051
|
+
const file = join(crawlDir, `${slug || "page"}.md`);
|
|
2052
|
+
writeFileSync(file, `# ${url}\n\n${doc.markdown}\n`);
|
|
2053
|
+
ctxIndex.indexFile(indexDb, file);
|
|
2054
|
+
indexed += 1;
|
|
2055
|
+
} catch {
|
|
2056
|
+
/* the page is in the reply; the index is the bonus */
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
if (depth < maxDepth) {
|
|
2060
|
+
for (const link of await sameOriginLinks(page)) {
|
|
2061
|
+
if (seen.has(link) || seen.size >= maxPages * 4) continue;
|
|
2062
|
+
seen.add(link);
|
|
2063
|
+
queue.push({ url: link, depth: depth + 1 });
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
// One request at a time, with a pause. A crawler that opens a site in
|
|
2067
|
+
// parallel is a load test nobody asked for.
|
|
2068
|
+
if (queue.length && out.length < maxPages) await page.waitForTimeout(delay);
|
|
2069
|
+
}
|
|
2070
|
+
resetRefs();
|
|
2071
|
+
const indexNote = indexed
|
|
2072
|
+
? ` - ${indexed} indexed, searchable with context_search`
|
|
2073
|
+
: wantIndex && indexDb === null
|
|
2074
|
+
? " - the index could not be opened, so these pages are not searchable"
|
|
2075
|
+
: "";
|
|
2076
|
+
return `${out.length} page(s)${indexNote}\n\n${fenceUntrusted(`${out.length} page(s) from ${origin && origin !== "null" ? origin : start}`, out.join("\n\n"))}`;
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
// web_crawl announces itself on every request it makes, so a site operator
|
|
2080
|
+
// reading the access log can tell the crawl from a visitor and find its owner.
|
|
2081
|
+
const crawlUserAgent = () => `multi-agent-toolkit-mcp/${PKG.version} (+https://github.com/mmerterden/multi-agent-toolkit-mcp)`;
|
|
2082
|
+
|
|
2083
|
+
// Non-negative integer when given, the default when absent. An explicit 0 is a
|
|
2084
|
+
// value (max_depth 0 means the start page only, delay_ms 0 means no pause), so
|
|
2085
|
+
// it cannot be read as "not given".
|
|
2086
|
+
function countOr(value, fallback) {
|
|
2087
|
+
if (value === undefined || value === null || value === "") return fallback;
|
|
2088
|
+
const n = Math.trunc(Number(value));
|
|
2089
|
+
return Number.isFinite(n) && n >= 0 ? n : fallback;
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
const PAGE_READING_TOOLS = new Set([
|
|
2093
|
+
"web_get_text", "web_extract", "web_snapshot", "web_screenshot", "web_eval",
|
|
2094
|
+
"web_map", "web_storage_state", "web_select_option", "web_type", "web_click", "web_press_key", "web_wait_for",
|
|
2095
|
+
]);
|
|
2096
|
+
|
|
1891
2097
|
async function handleWeb(name, args) {
|
|
1892
2098
|
if (name === "web_close") { await closeBrowser(); resetRefs(); return "Browser closed"; }
|
|
1893
|
-
|
|
2099
|
+
// Before the browser launches: a refused URL costs nothing.
|
|
2100
|
+
if (name === "web_goto") {
|
|
2101
|
+
const refused = await checkUrl(args.url);
|
|
2102
|
+
if (refused) return `${ERROR_PREFIX}${refused}`;
|
|
2103
|
+
}
|
|
2104
|
+
const page = await ensureBrowser(args.browser, name === "web_goto" ? args.user_agent : undefined);
|
|
2105
|
+
// The route guard stops a refused page from loading; this is the second
|
|
2106
|
+
// line, for a page that got there anyway (a DNS answer that changed between
|
|
2107
|
+
// two lookups): nothing is read from a page whose address is refused now.
|
|
2108
|
+
if (URL_POLICY === "strict" && PAGE_READING_TOOLS.has(name)) {
|
|
2109
|
+
const refused = await landedRefusal(page);
|
|
2110
|
+
if (refused) return `${ERROR_PREFIX}the current page is at a refused address. ${refused}`;
|
|
2111
|
+
}
|
|
1894
2112
|
switch (name) {
|
|
1895
2113
|
case "web_goto": {
|
|
1896
2114
|
if (args.viewport) {
|
|
@@ -1910,10 +2128,33 @@ async function handleWeb(name, args) {
|
|
|
1910
2128
|
// Every ref from before this navigation points at a page that is gone,
|
|
1911
2129
|
// so they are dropped rather than left to resolve by accident.
|
|
1912
2130
|
resetRefs();
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
2131
|
+
const seq = _refusalSeq;
|
|
2132
|
+
let gotoError = null;
|
|
2133
|
+
try {
|
|
2134
|
+
await page.goto(args.url, {
|
|
2135
|
+
waitUntil: args.wait_until || "load",
|
|
2136
|
+
timeout: args.timeout_ms || 30000,
|
|
2137
|
+
});
|
|
2138
|
+
} catch (e) {
|
|
2139
|
+
gotoError = e;
|
|
2140
|
+
}
|
|
2141
|
+
// A redirect can land somewhere the requested URL did not name, and a
|
|
2142
|
+
// goto that threw (a timeout after the redirect) may still have landed.
|
|
2143
|
+
const landed = await landedRefusal(page);
|
|
2144
|
+
if (landed) {
|
|
2145
|
+
await page.goto("about:blank").catch(() => {});
|
|
2146
|
+
resetRefs();
|
|
2147
|
+
return `${ERROR_PREFIX}${args.url} redirected to a refused address. ${landed}`;
|
|
2148
|
+
}
|
|
2149
|
+
if (gotoError) {
|
|
2150
|
+
const refused = refusalsSince(seq).find((r) => r.navigation);
|
|
2151
|
+
if (refused) {
|
|
2152
|
+
await page.goto("about:blank").catch(() => {});
|
|
2153
|
+
resetRefs();
|
|
2154
|
+
return `${ERROR_PREFIX}${args.url} was refused on the way to ${refused.url}. ${refused.reason}`;
|
|
2155
|
+
}
|
|
2156
|
+
throw gotoError;
|
|
2157
|
+
}
|
|
1917
2158
|
return `Opened ${args.url} (title: "${await page.title()}")`;
|
|
1918
2159
|
}
|
|
1919
2160
|
case "web_screenshot": {
|
|
@@ -2063,74 +2304,44 @@ async function handleWeb(name, args) {
|
|
|
2063
2304
|
const blank = blankPageError(page, "web_crawl");
|
|
2064
2305
|
if (blank) return blank;
|
|
2065
2306
|
const maxPages = Math.min(Number(args.max_pages) || 20, 200);
|
|
2066
|
-
const maxDepth =
|
|
2067
|
-
const delay =
|
|
2307
|
+
const maxDepth = countOr(args.max_depth, 2);
|
|
2308
|
+
const delay = countOr(args.delay_ms, 250);
|
|
2068
2309
|
const respectRobots = args.respect_robots !== false;
|
|
2069
2310
|
const start = page.url();
|
|
2311
|
+
// The start page can be reached without web_goto (a click, a script),
|
|
2312
|
+
// so it is checked here as well as every page the crawl opens.
|
|
2313
|
+
const refusedStart = await checkUrl(start);
|
|
2314
|
+
if (refusedStart) return `${ERROR_PREFIX}${refusedStart}`;
|
|
2070
2315
|
const origin = new URL(start).origin;
|
|
2071
2316
|
const wantIndex = args.index !== false;
|
|
2072
|
-
let indexed = 0;
|
|
2073
2317
|
let indexDb = null;
|
|
2074
2318
|
let crawlDir = null;
|
|
2319
|
+
let ctxIndex = null;
|
|
2075
2320
|
if (wantIndex) {
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
crawlDir = join(ctxIndex.INDEX_DIR, "crawl");
|
|
2079
|
-
if (!existsSync(crawlDir)) mkdirSync(crawlDir, { recursive: true });
|
|
2080
|
-
} catch {
|
|
2081
|
-
indexDb = null;
|
|
2082
|
-
}
|
|
2083
|
-
}
|
|
2084
|
-
const disallowed = respectRobots ? await robotsDisallow(page, origin) : [];
|
|
2085
|
-
const seen = new Set([start]);
|
|
2086
|
-
const queue = [{ url: start, depth: 0 }];
|
|
2087
|
-
const out = [];
|
|
2088
|
-
while (queue.length && out.length < maxPages) {
|
|
2089
|
-
const { url, depth } = queue.shift();
|
|
2090
|
-
if (disallowed.some((rule) => url.startsWith(origin + rule))) continue;
|
|
2091
|
-
try {
|
|
2092
|
-
await page.goto(url, { waitUntil: "load", timeout: 15000 });
|
|
2093
|
-
} catch (e) {
|
|
2094
|
-
out.push(`--- ${url}\nERROR: ${e.message}`);
|
|
2095
|
-
continue;
|
|
2096
|
-
}
|
|
2097
|
-
const doc = await extractReadable(page);
|
|
2098
|
-
out.push(`--- ${url}\n${doc.markdown}`);
|
|
2099
|
-
// A crawl that cannot be searched afterwards is a wall of text: twenty
|
|
2100
|
-
// pages arrive at once and the caller has no way to ask which of them
|
|
2101
|
-
// answers the question. Each page is written beside the index and
|
|
2102
|
-
// indexed under its URL, so context_search ranks the crawl the same way
|
|
2103
|
-
// it ranks an offloaded payload. Failure here never fails the crawl -
|
|
2104
|
-
// the pages are already in the reply.
|
|
2105
|
-
if (wantIndex && indexDb) {
|
|
2321
|
+
const opened = await openContextDb();
|
|
2322
|
+
if (!opened.error) {
|
|
2106
2323
|
try {
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
indexed += 1;
|
|
2324
|
+
ctxIndex = opened.ctx;
|
|
2325
|
+
crawlDir = join(ctxIndex.INDEX_DIR, "crawl");
|
|
2326
|
+
if (!existsSync(crawlDir)) mkdirSync(crawlDir, { recursive: true });
|
|
2327
|
+
indexDb = opened.db;
|
|
2112
2328
|
} catch {
|
|
2113
|
-
|
|
2329
|
+
indexDb = null;
|
|
2114
2330
|
}
|
|
2115
2331
|
}
|
|
2116
|
-
if (depth < maxDepth) {
|
|
2117
|
-
for (const link of await sameOriginLinks(page)) {
|
|
2118
|
-
if (seen.has(link) || seen.size >= maxPages * 4) continue;
|
|
2119
|
-
seen.add(link);
|
|
2120
|
-
queue.push({ url: link, depth: depth + 1 });
|
|
2121
|
-
}
|
|
2122
|
-
}
|
|
2123
|
-
// One request at a time, with a pause. A crawler that opens a site in
|
|
2124
|
-
// parallel is a load test nobody asked for.
|
|
2125
|
-
if (queue.length && out.length < maxPages) await page.waitForTimeout(delay);
|
|
2126
2332
|
}
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2333
|
+
// A route rather than a header override: a context created with a
|
|
2334
|
+
// user_agent wins over extra headers, and the crawl has to identify
|
|
2335
|
+
// itself whatever the session was set to.
|
|
2336
|
+
// fallback, not continue: continue would send the request straight to the
|
|
2337
|
+
// network and skip the context's URL-policy route behind it.
|
|
2338
|
+
const identify = (route) => route.fallback({ headers: { ...route.request().headers(), "user-agent": crawlUserAgent() } }).catch(() => {});
|
|
2339
|
+
await page.route("**/*", identify);
|
|
2340
|
+
try {
|
|
2341
|
+
return await crawlFrom(page, { start, origin, maxPages, maxDepth, delay, respectRobots, wantIndex, indexDb, crawlDir, ctxIndex });
|
|
2342
|
+
} finally {
|
|
2343
|
+
await page.unroute("**/*", identify).catch(() => {});
|
|
2344
|
+
}
|
|
2134
2345
|
}
|
|
2135
2346
|
|
|
2136
2347
|
case "web_press_key": {
|
|
@@ -2318,33 +2529,66 @@ const CONTEXT_TOOLS = [
|
|
|
2318
2529
|
},
|
|
2319
2530
|
];
|
|
2320
2531
|
|
|
2321
|
-
|
|
2322
|
-
|
|
2532
|
+
// node:sqlite is unflagged from Node 22.13, and a static import of it fails the
|
|
2533
|
+
// whole module graph, so every tool would go down with the three that need it.
|
|
2534
|
+
// The context module is imported on first use instead; when it cannot load,
|
|
2535
|
+
// context_* and crawl indexing answer with the reason and nothing else changes.
|
|
2536
|
+
let contextModule = null;
|
|
2537
|
+
function loadContext() {
|
|
2538
|
+
contextModule ??= import("./tools/context/index.js").then(
|
|
2539
|
+
(mod) => ({ mod }),
|
|
2540
|
+
(e) => ({ error: `the context index needs node:sqlite, available unflagged from Node 22.13 (this is ${process.version}): ${e.message}` }),
|
|
2541
|
+
);
|
|
2542
|
+
return contextModule;
|
|
2543
|
+
}
|
|
2544
|
+
|
|
2545
|
+
// One handle for the life of the server. A DatabaseSync per call keeps every
|
|
2546
|
+
// earlier handle open until the garbage collector happens to run.
|
|
2547
|
+
let contextDb = null;
|
|
2548
|
+
async function openContextDb() {
|
|
2549
|
+
const ctx = await loadContext();
|
|
2550
|
+
if (ctx.error) return { error: ctx.error };
|
|
2323
2551
|
try {
|
|
2324
|
-
|
|
2552
|
+
contextDb ??= ctx.mod.openIndex();
|
|
2325
2553
|
} catch (e) {
|
|
2326
|
-
return
|
|
2554
|
+
return { error: `could not open the index: ${e.message}` };
|
|
2327
2555
|
}
|
|
2556
|
+
return { ctx: ctx.mod, db: contextDb };
|
|
2557
|
+
}
|
|
2558
|
+
|
|
2559
|
+
function closeContextDb() {
|
|
2560
|
+
try { contextDb?.close(); } catch {}
|
|
2561
|
+
contextDb = null;
|
|
2562
|
+
}
|
|
2563
|
+
|
|
2564
|
+
async function handleContext(name, args) {
|
|
2565
|
+
const opened = await openContextDb();
|
|
2566
|
+
if (opened.error) return `${ERROR_PREFIX}${opened.error}`;
|
|
2567
|
+
const { ctx, db } = opened;
|
|
2328
2568
|
switch (name) {
|
|
2329
2569
|
case "context_index": {
|
|
2330
|
-
const
|
|
2570
|
+
const denied = indexDenied(String(args.path));
|
|
2571
|
+
if (denied) return `${ERROR_PREFIX}${denied}`;
|
|
2572
|
+
const out = ctx.indexFile(db, String(args.path));
|
|
2331
2573
|
if (out.reason === "no such file") return `${ERROR_PREFIX}no such file: ${args.path}`;
|
|
2332
2574
|
if (!out.indexed) return `Already indexed and unchanged: ${out.chunks} passage(s)`;
|
|
2333
2575
|
return `Indexed ${args.path}: ${out.chunks} passage(s)`;
|
|
2334
2576
|
}
|
|
2335
2577
|
case "context_search": {
|
|
2336
|
-
const hits =
|
|
2578
|
+
const hits = ctx.search(db, String(args.query), {
|
|
2337
2579
|
limit: Number(args.limit) || 5,
|
|
2338
2580
|
path: args.path ? String(args.path) : null,
|
|
2339
|
-
});
|
|
2581
|
+
}).filter((h) => !indexDenied(h.path));
|
|
2340
2582
|
if (!hits.length) return "(no match in the index; run context_index on the file first)";
|
|
2341
2583
|
return hits
|
|
2342
2584
|
.map((h) => `[id ${h.id}] ${h.path}:${h.firstLine}-${h.lastLine}\n${h.snippet}`)
|
|
2343
2585
|
.join("\n\n");
|
|
2344
2586
|
}
|
|
2345
2587
|
case "context_get": {
|
|
2346
|
-
const row =
|
|
2588
|
+
const row = ctx.getChunk(db, args.id);
|
|
2347
2589
|
if (!row) return `${ERROR_PREFIX}no passage with id ${args.id}`;
|
|
2590
|
+
const denied = indexDenied(row.path);
|
|
2591
|
+
if (denied) return `${ERROR_PREFIX}${denied}`;
|
|
2348
2592
|
return `${row.path} lines ${row.first_line}-${row.last_line}\n\n${row.body}`;
|
|
2349
2593
|
}
|
|
2350
2594
|
default:
|
|
@@ -2375,21 +2619,51 @@ const RESEARCH_TOOLS = [
|
|
|
2375
2619
|
},
|
|
2376
2620
|
];
|
|
2377
2621
|
|
|
2622
|
+
const RESEARCH_PROVIDERS = ["brave", "perplexity"];
|
|
2623
|
+
const RESEARCH_KEYS = { brave: "BRAVE_API_KEY", perplexity: "PERPLEXITY_API_KEY" };
|
|
2624
|
+
// Brave's web search answers at most 20 results per request.
|
|
2625
|
+
const RESEARCH_MAX_RESULTS = 20;
|
|
2626
|
+
|
|
2627
|
+
// The provider decides both the endpoint and the key, so it is resolved once,
|
|
2628
|
+
// normalized, and checked against the same list the schema declares. A value
|
|
2629
|
+
// from RESEARCH_PROVIDER never passes through the schema, and deriving the key
|
|
2630
|
+
// and the endpoint separately lets a near miss ("Brave") send one provider's
|
|
2631
|
+
// key to the other.
|
|
2632
|
+
function researchProvider(args) {
|
|
2633
|
+
const raw = args.provider || process.env.RESEARCH_PROVIDER || "brave";
|
|
2634
|
+
const provider = String(raw).trim().toLowerCase();
|
|
2635
|
+
if (RESEARCH_PROVIDERS.includes(provider)) return { provider };
|
|
2636
|
+
return { error: `${ERROR_PREFIX}unknown research provider "${String(raw).trim()}"; expected one of ${RESEARCH_PROVIDERS.join(", ")}` };
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2378
2639
|
function researchKey(provider) {
|
|
2379
|
-
const name = provider
|
|
2380
|
-
const value = process.env[name];
|
|
2640
|
+
const name = RESEARCH_KEYS[provider];
|
|
2641
|
+
const value = name ? process.env[name] : undefined;
|
|
2381
2642
|
// The NAME is safe to say; the value never is. A caller who has not set it
|
|
2382
2643
|
// needs to know which variable to set.
|
|
2383
2644
|
return value ? { value, name } : { error: `${ERROR_PREFIX}${name} is not set in this environment` };
|
|
2384
2645
|
}
|
|
2385
2646
|
|
|
2647
|
+
function researchLimit(value) {
|
|
2648
|
+
const n = Math.trunc(Number(value));
|
|
2649
|
+
return Number.isFinite(n) && n > 0 ? Math.min(n, RESEARCH_MAX_RESULTS) : 5;
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// One shape for every provider: title, url and snippet, one block per result.
|
|
2653
|
+
function researchRows(rows) {
|
|
2654
|
+
if (!rows.length) return "(no results)";
|
|
2655
|
+
return rows.map((r) => `${r.title ?? ""}\n${r.url ?? ""}\n${r.snippet ?? ""}`).join("\n\n");
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2386
2658
|
async function handleResearch(name, args) {
|
|
2387
2659
|
const timeout = AbortSignal.timeout(20000);
|
|
2388
2660
|
if (name === "research_search") {
|
|
2389
|
-
const
|
|
2661
|
+
const chosen = researchProvider(args);
|
|
2662
|
+
if (chosen.error) return chosen.error;
|
|
2663
|
+
const { provider } = chosen;
|
|
2390
2664
|
const key = researchKey(provider);
|
|
2391
2665
|
if (key.error) return key.error;
|
|
2392
|
-
const limit =
|
|
2666
|
+
const limit = researchLimit(args.limit);
|
|
2393
2667
|
try {
|
|
2394
2668
|
if (provider === "brave") {
|
|
2395
2669
|
const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(args.query)}&count=${limit}`;
|
|
@@ -2400,8 +2674,7 @@ async function handleResearch(name, args) {
|
|
|
2400
2674
|
if (!res.ok) return `${ERROR_PREFIX}brave returned ${res.status}`;
|
|
2401
2675
|
const body = await res.json();
|
|
2402
2676
|
const rows = (body?.web?.results || []).slice(0, limit);
|
|
2403
|
-
|
|
2404
|
-
return rows.map((r) => `${r.title}\n${r.url}\n${r.description ?? ""}`).join("\n\n");
|
|
2677
|
+
return researchRows(rows.map((r) => ({ title: r.title, url: r.url, snippet: r.description })));
|
|
2405
2678
|
}
|
|
2406
2679
|
const res = await fetch("https://api.perplexity.ai/chat/completions", {
|
|
2407
2680
|
method: "POST",
|
|
@@ -2414,10 +2687,14 @@ async function handleResearch(name, args) {
|
|
|
2414
2687
|
});
|
|
2415
2688
|
if (!res.ok) return `${ERROR_PREFIX}perplexity returned ${res.status}`;
|
|
2416
2689
|
const body = await res.json();
|
|
2417
|
-
|
|
2418
|
-
|
|
2690
|
+
// search_results carries titles and snippets; an older response has only
|
|
2691
|
+
// the citation URLs, which still fill the url line.
|
|
2692
|
+
const rows = Array.isArray(body?.search_results) && body.search_results.length
|
|
2693
|
+
? body.search_results.map((r) => ({ title: r.title, url: r.url, snippet: r.snippet ?? "" }))
|
|
2694
|
+
: (body?.citations || []).map((c) => ({ title: "", url: c, snippet: "" }));
|
|
2695
|
+
return researchRows(rows.slice(0, limit));
|
|
2419
2696
|
} catch (e) {
|
|
2420
|
-
return `${ERROR_PREFIX}${String(e.message || e).slice(0, 200)}`;
|
|
2697
|
+
return `${ERROR_PREFIX}${provider}: ${String(e.message || e).slice(0, 200)}`;
|
|
2421
2698
|
}
|
|
2422
2699
|
}
|
|
2423
2700
|
if (name === "research_ask") {
|
|
@@ -2459,6 +2736,8 @@ const MEDIA_TOOLS = [
|
|
|
2459
2736
|
},
|
|
2460
2737
|
];
|
|
2461
2738
|
|
|
2739
|
+
const FRAME_NAME = /^frame-\d{3,}\.png$/;
|
|
2740
|
+
|
|
2462
2741
|
async function handleMedia(name, args) {
|
|
2463
2742
|
if (name !== "media_frames") return null;
|
|
2464
2743
|
if (!existsSync(String(args.path))) return `${ERROR_PREFIX}no such file: ${args.path}`;
|
|
@@ -2471,11 +2750,17 @@ async function handleMedia(name, args) {
|
|
|
2471
2750
|
const threshold = Number(args.threshold) > 0 ? Number(args.threshold) : 0.25;
|
|
2472
2751
|
const outDir = args.out_dir ? String(args.out_dir) : join(dirname(String(args.path)), `${basename(String(args.path), ".mp4")}-frames`);
|
|
2473
2752
|
if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
|
|
2753
|
+
// Frames from an earlier run into the same directory would be counted as
|
|
2754
|
+
// this run's, and ffmpeg refuses to overwrite them without -y. Only the
|
|
2755
|
+
// names this tool writes are removed.
|
|
2756
|
+
for (const f of readdirSync(outDir)) {
|
|
2757
|
+
if (FRAME_NAME.test(f)) unlinkSync(join(outDir, f));
|
|
2758
|
+
}
|
|
2474
2759
|
// The scene filter is what drops near-duplicates: it emits a frame only when
|
|
2475
2760
|
// enough of the picture changed, which is the difference between twelve
|
|
2476
2761
|
// useful screens and six hundred copies of the same one.
|
|
2477
2762
|
const args2 = [
|
|
2478
|
-
"-hide_banner", "-loglevel", "error",
|
|
2763
|
+
"-y", "-hide_banner", "-loglevel", "error",
|
|
2479
2764
|
"-i", String(args.path),
|
|
2480
2765
|
"-vf", `select='gt(scene,${threshold})',showinfo`,
|
|
2481
2766
|
"-vsync", "vfr",
|
|
@@ -2484,7 +2769,7 @@ async function handleMedia(name, args) {
|
|
|
2484
2769
|
];
|
|
2485
2770
|
const r = spawnSync("ffmpeg", args2, { encoding: "utf8", timeout: 120000 });
|
|
2486
2771
|
if (r.status !== 0) return `${ERROR_PREFIX}ffmpeg failed: ${String(r.stderr || "").slice(0, 200)}`;
|
|
2487
|
-
const frames = readdirSync(outDir).filter((f) =>
|
|
2772
|
+
const frames = readdirSync(outDir).filter((f) => FRAME_NAME.test(f)).sort();
|
|
2488
2773
|
if (!frames.length) {
|
|
2489
2774
|
return `No scene changes above ${threshold} in ${args.path}. A lower threshold keeps more frames.`;
|
|
2490
2775
|
}
|
|
@@ -2499,10 +2784,10 @@ async function handleMedia(name, args) {
|
|
|
2499
2784
|
//
|
|
2500
2785
|
// Unset means everything, which is the behaviour every existing consumer
|
|
2501
2786
|
// already has. The value is a comma-separated list of families: ios, android,
|
|
2502
|
-
// web, design, code, pass, agent, context, research, media. An unknown name is
|
|
2787
|
+
// web, design, code, pass, agent, context, research, media, security. An unknown name is
|
|
2503
2788
|
// reported on stderr rather than silently ignored, because a typo that quietly
|
|
2504
2789
|
// disables a family is worse than a noisy one.
|
|
2505
|
-
const ALL_CAPS = ["ios", "android", "web", "design", "code", "pass", "agent", "context", "research", "media"];
|
|
2790
|
+
const ALL_CAPS = ["ios", "android", "web", "design", "code", "pass", "agent", "context", "research", "media", "security"];
|
|
2506
2791
|
|
|
2507
2792
|
function enabledCaps() {
|
|
2508
2793
|
const raw = (process.env.MCP_TOOLKIT_CAPS || "").trim();
|
|
@@ -2543,6 +2828,7 @@ const ALL_TOOLS = [
|
|
|
2543
2828
|
...DESIGN_TOOLS,
|
|
2544
2829
|
...CODE_TOOLS,
|
|
2545
2830
|
...PASS_TOOLS,
|
|
2831
|
+
...SECURITY_TOOLS,
|
|
2546
2832
|
];
|
|
2547
2833
|
|
|
2548
2834
|
// Name -> inputSchema, so the CallTool boundary can enforce the declared shape.
|
|
@@ -2554,7 +2840,7 @@ const TOOL_SCHEMAS = new Map(ALL_TOOLS.map((t) => [t.name, t.inputSchema || {}])
|
|
|
2554
2840
|
// number, which is the root cause of the shell-injection class. This enforces
|
|
2555
2841
|
// the declared contract at the one boundary every tool passes through. Zero
|
|
2556
2842
|
// dependencies (no ajv): supports type (string/number/integer/boolean/array/
|
|
2557
|
-
// object), required,
|
|
2843
|
+
// object), required, enum and additionalProperties:false at any depth.
|
|
2558
2844
|
function schemaTypeOk(value, type) {
|
|
2559
2845
|
switch (type) {
|
|
2560
2846
|
// A scalar is a fine string (paths/schemes have always arrived as strings,
|
|
@@ -2611,6 +2897,25 @@ function checkConstraints(key, value, spec) {
|
|
|
2611
2897
|
return `argument '${key}' must be <= ${spec.maximum}`;
|
|
2612
2898
|
}
|
|
2613
2899
|
}
|
|
2900
|
+
// Declared object fields are enforced at every depth, not only at the top
|
|
2901
|
+
// level: a target list whose items declare additionalProperties:false must
|
|
2902
|
+
// refuse a misspelt field inside an item the same way a tool refuses a
|
|
2903
|
+
// misspelt argument.
|
|
2904
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value) && (spec.properties || spec.required)) {
|
|
2905
|
+
for (const req of spec.required || []) {
|
|
2906
|
+
if (value[req] === undefined || value[req] === null) return `argument '${key}' is missing required field: ${req}`;
|
|
2907
|
+
}
|
|
2908
|
+
const props = spec.properties || {};
|
|
2909
|
+
if (spec.additionalProperties === false) {
|
|
2910
|
+
const extra = Object.keys(value).find((k) => !Object.hasOwn(props, k));
|
|
2911
|
+
if (extra) return `argument '${key}' has unexpected field: ${extra}`;
|
|
2912
|
+
}
|
|
2913
|
+
for (const [k, sub] of Object.entries(props)) {
|
|
2914
|
+
if (value[k] === undefined || value[k] === null) continue;
|
|
2915
|
+
const subErr = checkConstraints(`${key}.${k}`, value[k], sub);
|
|
2916
|
+
if (subErr) return subErr;
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2614
2919
|
if (Array.isArray(value)) {
|
|
2615
2920
|
if (Number.isFinite(spec.minItems) && value.length < spec.minItems) {
|
|
2616
2921
|
return `argument '${key}' must have at least ${spec.minItems} item(s)`;
|
|
@@ -2650,6 +2955,12 @@ function validateArgs(name, args) {
|
|
|
2650
2955
|
return `missing required argument: ${req}`;
|
|
2651
2956
|
}
|
|
2652
2957
|
}
|
|
2958
|
+
// Only where a schema says so. Most tools declare no additionalProperties and
|
|
2959
|
+
// keep accepting extras, as they always have.
|
|
2960
|
+
if (schema.additionalProperties === false) {
|
|
2961
|
+
const extra = Object.keys(args).find((k) => !Object.hasOwn(props, k));
|
|
2962
|
+
if (extra) return `unexpected argument: ${extra}`;
|
|
2963
|
+
}
|
|
2653
2964
|
for (const [key, spec] of Object.entries(props)) {
|
|
2654
2965
|
if (args[key] === undefined || args[key] === null) continue;
|
|
2655
2966
|
const err = checkConstraints(key, args[key], spec);
|
|
@@ -2671,6 +2982,7 @@ const READ_ONLY_TOOLS = new Set([
|
|
|
2671
2982
|
"android_list_crashes",
|
|
2672
2983
|
"web_screenshot", "web_get_text",
|
|
2673
2984
|
"design_mock_detect", "design_scenario_inventory", "design_ui_geometry", "design_visual_compare",
|
|
2985
|
+
"design_component_variants",
|
|
2674
2986
|
// Seven of the eight. Every code_* tool reads source and answers; the one
|
|
2675
2987
|
// that is not here stops a cached server.
|
|
2676
2988
|
...CODE_READ_ONLY,
|
|
@@ -2805,6 +3117,7 @@ const OUTPUT_SCHEMAS = {
|
|
|
2805
3117
|
// made all 78 unavailable in Claude Code - see the note further down.
|
|
2806
3118
|
...CODE_OUTPUT_SCHEMAS,
|
|
2807
3119
|
...PASS_OUTPUT_SCHEMAS,
|
|
3120
|
+
...SECURITY_OUTPUT_SCHEMAS,
|
|
2808
3121
|
ios_accessibility_audit: ACCESSIBILITY_AUDIT_SCHEMA,
|
|
2809
3122
|
android_accessibility_audit: ACCESSIBILITY_AUDIT_SCHEMA,
|
|
2810
3123
|
|
|
@@ -2977,7 +3290,13 @@ if (SERVED_TOOLS.length !== ANNOTATED_TOOLS.length) {
|
|
|
2977
3290
|
}
|
|
2978
3291
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: SERVED_TOOLS }));
|
|
2979
3292
|
|
|
2980
|
-
|
|
3293
|
+
// Never written to the offload directory. The query tool reads that directory,
|
|
3294
|
+
// and web_storage_state returns session cookies and localStorage: a file on
|
|
3295
|
+
// disk would outlive the session it authenticates, and the reply is JSON the
|
|
3296
|
+
// caller restores from, which a head+tail window would break.
|
|
3297
|
+
const OFFLOAD_EXEMPT = new Set(["agent_query_output", "web_storage_state"]);
|
|
3298
|
+
|
|
3299
|
+
async function handleCallTool(request, extra) {
|
|
2981
3300
|
const { name, arguments: args } = request.params;
|
|
2982
3301
|
// Long-running tools (xcodebuild, export, validate, install) report progress
|
|
2983
3302
|
// when the client sent a progressToken, and abandon their child process when
|
|
@@ -3019,6 +3338,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
|
|
|
3019
3338
|
// aborted call cancels the in-flight LSP request instead of orphaning it.
|
|
3020
3339
|
else if (name.startsWith("code_")) result = await handleCode(name, args || {}, ctx);
|
|
3021
3340
|
else if (name.startsWith("pass_")) result = await handlePass(name, args || {});
|
|
3341
|
+
else if (name.startsWith("security_")) result = await handleSecurity(name, args || {});
|
|
3022
3342
|
else return { content: [{ type: "text", text: `Unknown tool: ${name}` }], isError: true };
|
|
3023
3343
|
// Same reason as dispatchStep: a handler returns null only from its
|
|
3024
3344
|
// `default:` arm, so an unrecognised name that happens to carry a known
|
|
@@ -3064,9 +3384,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
|
|
|
3064
3384
|
return { content: [{ type: "text", text: String(result) }], isError: true };
|
|
3065
3385
|
}
|
|
3066
3386
|
// Prose payloads only: a tool with an outputSchema answers with JSON the
|
|
3067
|
-
// caller parses, and a head+tail summary would break that parse.
|
|
3068
|
-
|
|
3069
|
-
if (!OUTPUT_SCHEMAS[name] && name !== "agent_query_output") {
|
|
3387
|
+
// caller parses, and a head+tail summary would break that parse.
|
|
3388
|
+
if (!OUTPUT_SCHEMAS[name] && !OFFLOAD_EXEMPT.has(name)) {
|
|
3070
3389
|
const { text, offloaded } = offloadLargeText(name, String(result));
|
|
3071
3390
|
if (offloaded) return { content: [{ type: "text", text }] };
|
|
3072
3391
|
return withStructured(name, text);
|
|
@@ -3078,6 +3397,18 @@ server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
|
|
|
3078
3397
|
isError: true,
|
|
3079
3398
|
};
|
|
3080
3399
|
}
|
|
3400
|
+
}
|
|
3401
|
+
|
|
3402
|
+
// Counted so a host that closes stdin still gets the answers it is owed before
|
|
3403
|
+
// the server exits.
|
|
3404
|
+
let inFlight = 0;
|
|
3405
|
+
server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
|
|
3406
|
+
inFlight += 1;
|
|
3407
|
+
try {
|
|
3408
|
+
return await handleCallTool(request, extra);
|
|
3409
|
+
} finally {
|
|
3410
|
+
inFlight -= 1;
|
|
3411
|
+
}
|
|
3081
3412
|
});
|
|
3082
3413
|
|
|
3083
3414
|
// Recorder children must not outlive the server: `simctl io recordVideo` has no
|
|
@@ -3089,8 +3420,41 @@ function stopRecorders() {
|
|
|
3089
3420
|
}
|
|
3090
3421
|
}
|
|
3091
3422
|
|
|
3092
|
-
|
|
3093
|
-
|
|
3423
|
+
// One path for every way the server is told to go: children first, then the
|
|
3424
|
+
// browser (bounded, a wedged browser must not keep the process alive), then
|
|
3425
|
+
// the index handle.
|
|
3426
|
+
const BROWSER_CLOSE_BUDGET_MS = 3000;
|
|
3427
|
+
let shuttingDown = null;
|
|
3428
|
+
function shutdown() {
|
|
3429
|
+
shuttingDown ??= (async () => {
|
|
3430
|
+
stopRecorders();
|
|
3431
|
+
shutdownAllLsp();
|
|
3432
|
+
killAllSpawned("SIGTERM");
|
|
3433
|
+
await Promise.race([closeBrowser(), new Promise((r) => setTimeout(r, BROWSER_CLOSE_BUDGET_MS).unref())]);
|
|
3434
|
+
closeContextDb();
|
|
3435
|
+
process.exit(0);
|
|
3436
|
+
})();
|
|
3437
|
+
return shuttingDown;
|
|
3438
|
+
}
|
|
3439
|
+
process.on("SIGTERM", shutdown);
|
|
3440
|
+
process.on("SIGINT", shutdown);
|
|
3441
|
+
|
|
3442
|
+
// A host that closes stdin is gone, and nobody is left to send action:"stop"
|
|
3443
|
+
// or close the browser. An open browser, an LSP server or a recorder would keep
|
|
3444
|
+
// this process alive indefinitely. Recorders stop at once so their files are
|
|
3445
|
+
// finalised; calls already in flight get up to STDIN_END_GRACE_MS to answer,
|
|
3446
|
+
// and then the same shutdown as SIGTERM runs.
|
|
3447
|
+
const STDIN_END_GRACE_MS = 5000;
|
|
3448
|
+
process.stdin.once("end", async () => {
|
|
3449
|
+
stopRecorders();
|
|
3450
|
+
const deadline = Date.now() + STDIN_END_GRACE_MS;
|
|
3451
|
+
while (inFlight > 0 && Date.now() < deadline) await new Promise((r) => setTimeout(r, 50));
|
|
3452
|
+
// The SDK writes a response after its handler settles; let that write reach
|
|
3453
|
+
// stdout before exiting.
|
|
3454
|
+
await new Promise((r) => setImmediate(r));
|
|
3455
|
+
await new Promise((r) => process.stdout.write("", () => r()));
|
|
3456
|
+
await shutdown();
|
|
3457
|
+
});
|
|
3094
3458
|
|
|
3095
3459
|
const transport = new StdioServerTransport();
|
|
3096
3460
|
await server.connect(transport);
|