mcp-scraper 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -29
- package/dist/bin/api-server.cjs +159 -322
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +43 -1754
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -4
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +48 -329
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +8 -92
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +83 -1801
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +19 -26
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-7I6PA2UQ.js +7 -0
- package/dist/chunk-7I6PA2UQ.js.map +1 -0
- package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
- package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
- package/dist/{chunk-ROZEDHAA.js → chunk-FU3WERZ7.js} +20 -27
- package/dist/chunk-FU3WERZ7.js.map +1 -0
- package/dist/{chunk-TK4D2BZ5.js → chunk-VQFVA6RH.js} +17 -698
- package/dist/chunk-VQFVA6RH.js.map +1 -0
- package/dist/{server-YSKBD2VX.js → server-7CGBJWJY.js} +836 -114
- package/dist/server-7CGBJWJY.js.map +1 -0
- package/docs/mcp-tool-craft-lint.generated.md +2 -5
- package/docs/mcp-tool-manifest.generated.json +18 -110
- package/package.json +1 -1
- package/dist/chunk-DUEW4EOO.js +0 -214
- package/dist/chunk-DUEW4EOO.js.map +0 -1
- package/dist/chunk-IHXHQL53.js +0 -7
- package/dist/chunk-IHXHQL53.js.map +0 -1
- package/dist/chunk-NXRWFOEZ.js +0 -816
- package/dist/chunk-NXRWFOEZ.js.map +0 -1
- package/dist/chunk-ROZEDHAA.js.map +0 -1
- package/dist/chunk-TK4D2BZ5.js.map +0 -1
- package/dist/server-YSKBD2VX.js.map +0 -1
|
@@ -1,32 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
-
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
-
for (let key of __getOwnPropNames(from))
|
|
12
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
-
}
|
|
15
|
-
return to;
|
|
16
|
-
};
|
|
17
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
18
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
19
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
20
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
21
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
22
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
23
|
-
mod
|
|
24
|
-
));
|
|
25
3
|
|
|
26
4
|
// bin/mcp-scraper-combined-stdio-server.ts
|
|
27
|
-
var
|
|
28
|
-
var
|
|
29
|
-
var
|
|
5
|
+
var import_node_fs4 = require("fs");
|
|
6
|
+
var import_node_os4 = require("os");
|
|
7
|
+
var import_node_path5 = require("path");
|
|
30
8
|
var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31
9
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
32
10
|
|
|
@@ -274,41 +252,9 @@ var HttpMcpToolExecutor = class {
|
|
|
274
252
|
|
|
275
253
|
// src/mcp/browser-agent-mcp-server.ts
|
|
276
254
|
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
277
|
-
var
|
|
278
|
-
var
|
|
279
|
-
var
|
|
280
|
-
|
|
281
|
-
// src/lib/chrome-profiles.ts
|
|
282
|
-
var import_promises = require("fs/promises");
|
|
283
|
-
var import_node_os = require("os");
|
|
284
|
-
var import_node_path = require("path");
|
|
285
|
-
function defaultChromeUserDataDir() {
|
|
286
|
-
if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
|
|
287
|
-
if (process.platform === "win32") {
|
|
288
|
-
return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
|
|
289
|
-
}
|
|
290
|
-
return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
|
|
291
|
-
}
|
|
292
|
-
function recommendedKernelProfileName(value) {
|
|
293
|
-
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
294
|
-
}
|
|
295
|
-
async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
|
|
296
|
-
const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
|
|
297
|
-
const localState = JSON.parse(raw);
|
|
298
|
-
const wanted = emailFilter?.trim().toLowerCase();
|
|
299
|
-
return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
|
|
300
|
-
const email = (info.user_name ?? "").trim();
|
|
301
|
-
const displayName = (info.name ?? info.shortcut_name ?? "").trim();
|
|
302
|
-
const labelSource = email || displayName || directory;
|
|
303
|
-
return {
|
|
304
|
-
email,
|
|
305
|
-
displayName,
|
|
306
|
-
chromeProfileDirectory: directory,
|
|
307
|
-
chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
|
|
308
|
-
recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
|
|
309
|
-
};
|
|
310
|
-
}).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
|
|
311
|
-
}
|
|
255
|
+
var import_node_fs = require("fs");
|
|
256
|
+
var import_node_os2 = require("os");
|
|
257
|
+
var import_node_path2 = require("path");
|
|
312
258
|
|
|
313
259
|
// src/lib/browser-service-env.ts
|
|
314
260
|
function browserServiceProfileName() {
|
|
@@ -323,1363 +269,8 @@ function browserServiceProfileSaveChanges() {
|
|
|
323
269
|
return void 0;
|
|
324
270
|
}
|
|
325
271
|
|
|
326
|
-
// src/lib/local-browser-profiles.ts
|
|
327
|
-
var import_node_fs = require("fs");
|
|
328
|
-
var import_promises2 = require("fs/promises");
|
|
329
|
-
var import_node_os2 = require("os");
|
|
330
|
-
var import_node_path2 = require("path");
|
|
331
|
-
var EXCLUDED_PROFILE_NAMES = [
|
|
332
|
-
"Cache",
|
|
333
|
-
"Code Cache",
|
|
334
|
-
"Crashpad",
|
|
335
|
-
"DawnCache",
|
|
336
|
-
"DawnGraphiteCache",
|
|
337
|
-
"DawnWebGPUCache",
|
|
338
|
-
"GPUCache",
|
|
339
|
-
"GrShaderCache",
|
|
340
|
-
"ShaderCache",
|
|
341
|
-
"Shared Dictionary"
|
|
342
|
-
];
|
|
343
|
-
function defaultImportedBrowserProfilesDir() {
|
|
344
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
345
|
-
}
|
|
346
|
-
function defaultChromeExecutablePath() {
|
|
347
|
-
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
348
|
-
if (envPath) return envPath;
|
|
349
|
-
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
350
|
-
(0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
351
|
-
(0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
352
|
-
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
353
|
-
return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
|
|
354
|
-
}
|
|
355
|
-
function localBrowserModeEnabled() {
|
|
356
|
-
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
357
|
-
}
|
|
358
|
-
function importedProfileName(value) {
|
|
359
|
-
return recommendedKernelProfileName(value);
|
|
360
|
-
}
|
|
361
|
-
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
362
|
-
return (0, import_node_path2.join)(outputDir, importedProfileName(name));
|
|
363
|
-
}
|
|
364
|
-
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
365
|
-
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
366
|
-
}
|
|
367
|
-
function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
368
|
-
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
|
|
369
|
-
}
|
|
370
|
-
async function readExistingManifest(path) {
|
|
371
|
-
try {
|
|
372
|
-
return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
|
|
373
|
-
} catch {
|
|
374
|
-
return null;
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
function isExcludedProfilePath(sourceRoot, sourcePath) {
|
|
378
|
-
const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
|
|
379
|
-
if (!rel) return false;
|
|
380
|
-
const parts = rel.split(import_node_path2.sep).filter(Boolean);
|
|
381
|
-
const last = parts[parts.length - 1] ?? "";
|
|
382
|
-
if (last.startsWith("Singleton")) return true;
|
|
383
|
-
if (last === "LOCK" || last.endsWith(".lock")) return true;
|
|
384
|
-
if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
|
|
385
|
-
if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
|
|
386
|
-
return false;
|
|
387
|
-
}
|
|
388
|
-
async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
|
|
389
|
-
await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
|
|
390
|
-
recursive: true,
|
|
391
|
-
force: true,
|
|
392
|
-
errorOnExist: false,
|
|
393
|
-
dereference: false,
|
|
394
|
-
filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
|
|
395
|
-
});
|
|
396
|
-
}
|
|
397
|
-
function selectProfile(profiles, options) {
|
|
398
|
-
const requestedDirectory = options.chromeProfileDirectory?.trim();
|
|
399
|
-
const requestedEmail = options.email?.trim().toLowerCase();
|
|
400
|
-
const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
|
|
401
|
-
if (exactDirectory) return exactDirectory;
|
|
402
|
-
const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
|
|
403
|
-
if (exactEmail) return exactEmail;
|
|
404
|
-
const first = profiles[0];
|
|
405
|
-
if (first) return first;
|
|
406
|
-
const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
|
|
407
|
-
throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
|
|
408
|
-
}
|
|
409
|
-
async function ensureProfilePathExists(profile) {
|
|
410
|
-
const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
|
|
411
|
-
if (!details?.isDirectory()) {
|
|
412
|
-
throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
|
|
416
|
-
const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
|
|
417
|
-
const localState = JSON.parse(raw);
|
|
418
|
-
const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
|
|
419
|
-
localState.profile = {
|
|
420
|
-
...localState.profile ?? {},
|
|
421
|
-
info_cache: { Default: selectedInfo },
|
|
422
|
-
last_used: "Default",
|
|
423
|
-
last_active_profiles: ["Default"]
|
|
424
|
-
};
|
|
425
|
-
await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
|
|
426
|
-
`);
|
|
427
|
-
}
|
|
428
|
-
async function importChromeProfile(options = {}) {
|
|
429
|
-
const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
|
|
430
|
-
const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
|
|
431
|
-
const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
|
|
432
|
-
await ensureProfilePathExists(sourceProfile);
|
|
433
|
-
const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
|
|
434
|
-
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
435
|
-
const profileRoot = importedProfileRoot(name, outputDir);
|
|
436
|
-
const manifestPath = importedProfileManifestPath(name, outputDir);
|
|
437
|
-
const userDataDir = importedProfileUserDataDir(name, outputDir);
|
|
438
|
-
const existingManifest = await readExistingManifest(manifestPath);
|
|
439
|
-
const existing = (0, import_node_fs.existsSync)(profileRoot);
|
|
440
|
-
if (existing && !options.overwrite) {
|
|
441
|
-
throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
|
|
442
|
-
}
|
|
443
|
-
if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
|
|
444
|
-
await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
|
|
445
|
-
await writeNormalizedLocalState(
|
|
446
|
-
(0, import_node_path2.join)(sourceUserDataDir, "Local State"),
|
|
447
|
-
(0, import_node_path2.join)(userDataDir, "Local State"),
|
|
448
|
-
sourceProfile.chromeProfileDirectory
|
|
449
|
-
);
|
|
450
|
-
await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
|
|
451
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
452
|
-
const manifest = {
|
|
453
|
-
layoutVersion: 1,
|
|
454
|
-
name,
|
|
455
|
-
profileRoot,
|
|
456
|
-
userDataDir,
|
|
457
|
-
browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
|
|
458
|
-
sourceUserDataDir,
|
|
459
|
-
sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
|
|
460
|
-
sourceProfilePath: sourceProfile.chromeProfilePath,
|
|
461
|
-
email: sourceProfile.email,
|
|
462
|
-
displayName: sourceProfile.displayName,
|
|
463
|
-
createdAt: existingManifest?.createdAt ?? now,
|
|
464
|
-
updatedAt: now,
|
|
465
|
-
copyMode: "chrome-profile-clone"
|
|
466
|
-
};
|
|
467
|
-
await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
468
|
-
`);
|
|
469
|
-
return {
|
|
470
|
-
profile: manifest,
|
|
471
|
-
sourceProfile,
|
|
472
|
-
excludedNames: EXCLUDED_PROFILE_NAMES
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
476
|
-
const profileName = importedProfileName(name);
|
|
477
|
-
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
478
|
-
const manifest = await readExistingManifest(manifestPath);
|
|
479
|
-
if (!manifest) {
|
|
480
|
-
throw new Error(
|
|
481
|
-
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
482
|
-
);
|
|
483
|
-
}
|
|
484
|
-
return manifest;
|
|
485
|
-
}
|
|
486
|
-
async function syncImportedChromeProfile(name, options = {}) {
|
|
487
|
-
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
488
|
-
const manifest = await loadImportedBrowserProfile(name, outputDir);
|
|
489
|
-
return importChromeProfile({
|
|
490
|
-
name: manifest.name,
|
|
491
|
-
sourceUserDataDir: manifest.sourceUserDataDir,
|
|
492
|
-
chromeProfileDirectory: manifest.sourceProfileDirectory,
|
|
493
|
-
outputDir,
|
|
494
|
-
overwrite: true,
|
|
495
|
-
browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
|
|
496
|
-
});
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
// src/services/browser-agent/local-browser-agent-service.ts
|
|
500
|
-
var import_node_crypto = require("crypto");
|
|
501
|
-
var import_node_fs3 = require("fs");
|
|
502
|
-
var import_node_path4 = require("path");
|
|
503
|
-
var import_playwright_extra = require("playwright-extra");
|
|
504
|
-
var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
|
|
505
|
-
|
|
506
|
-
// src/services/fanout/cdp-capture.ts
|
|
507
|
-
var ANSWER_RE = /\/backend-api\/(f\/)?conversation$|\/chat_conversations\/[^/]+\/(completion|retry_completion)$|\/completion$/i;
|
|
508
|
-
var FanoutCdpCapture = class {
|
|
509
|
-
client = null;
|
|
510
|
-
reqUrl = /* @__PURE__ */ new Map();
|
|
511
|
-
ctById = /* @__PURE__ */ new Map();
|
|
512
|
-
answerIds = /* @__PURE__ */ new Set();
|
|
513
|
-
bodies = [];
|
|
514
|
-
pending = [];
|
|
515
|
-
answerStreams = 0;
|
|
516
|
-
async attach(page) {
|
|
517
|
-
if (this.client) return;
|
|
518
|
-
const client = await page.context().newCDPSession(page);
|
|
519
|
-
this.client = client;
|
|
520
|
-
await client.send("Network.enable", { maxTotalBufferSize: 2e8, maxResourceBufferSize: 2e8 });
|
|
521
|
-
client.on("Network.requestWillBeSent", (e) => {
|
|
522
|
-
const id = e.requestId;
|
|
523
|
-
const req = e.request;
|
|
524
|
-
if (id) this.reqUrl.set(id, req?.url || "");
|
|
525
|
-
});
|
|
526
|
-
client.on("Network.responseReceived", (e) => {
|
|
527
|
-
const id = e.requestId;
|
|
528
|
-
const res = e.response;
|
|
529
|
-
const url = res?.url || this.reqUrl.get(id) || "";
|
|
530
|
-
const headers = res?.headers || {};
|
|
531
|
-
const ct = String(headers["content-type"] || headers["Content-Type"] || "");
|
|
532
|
-
if (id) this.ctById.set(id, ct);
|
|
533
|
-
if (id && (this.isAnswerUrl(url) || /event-stream/i.test(ct))) this.answerIds.add(id);
|
|
534
|
-
});
|
|
535
|
-
client.on("Network.loadingFinished", (e) => this.grab(e.requestId));
|
|
536
|
-
}
|
|
537
|
-
reset() {
|
|
538
|
-
this.bodies = [];
|
|
539
|
-
this.answerStreams = 0;
|
|
540
|
-
this.answerIds.clear();
|
|
541
|
-
this.pending = [];
|
|
542
|
-
}
|
|
543
|
-
answerStreamCount() {
|
|
544
|
-
return this.answerStreams;
|
|
545
|
-
}
|
|
546
|
-
async drain() {
|
|
547
|
-
await Promise.all(this.pending);
|
|
548
|
-
}
|
|
549
|
-
bestAnswerText() {
|
|
550
|
-
const eventStreams = this.bodies.filter((b) => /event-stream/i.test(b.ct));
|
|
551
|
-
const answerStreams = eventStreams.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
|
|
552
|
-
if (answerStreams.length) return answerStreams.map((b) => b.text).join("\n");
|
|
553
|
-
const answerJson = this.bodies.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
|
|
554
|
-
if (answerJson.length) return answerJson[0].text;
|
|
555
|
-
if (eventStreams.length) return eventStreams.sort((a, b) => b.text.length - a.text.length)[0].text;
|
|
556
|
-
return "";
|
|
557
|
-
}
|
|
558
|
-
async detach() {
|
|
559
|
-
try {
|
|
560
|
-
await this.client?.detach();
|
|
561
|
-
} catch {
|
|
562
|
-
this.client = null;
|
|
563
|
-
}
|
|
564
|
-
this.client = null;
|
|
565
|
-
}
|
|
566
|
-
isAnswerUrl(url) {
|
|
567
|
-
let path = "";
|
|
568
|
-
try {
|
|
569
|
-
path = new URL(url).pathname;
|
|
570
|
-
} catch {
|
|
571
|
-
path = String(url || "").split("?")[0];
|
|
572
|
-
}
|
|
573
|
-
return ANSWER_RE.test(path);
|
|
574
|
-
}
|
|
575
|
-
grab(requestId) {
|
|
576
|
-
if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
|
|
577
|
-
const url = this.reqUrl.get(requestId) || "";
|
|
578
|
-
const ct = this.ctById.get(requestId) || "";
|
|
579
|
-
const client = this.client;
|
|
580
|
-
const p = client.send("Network.getResponseBody", { requestId }).then((r) => {
|
|
581
|
-
const text = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
|
|
582
|
-
if (text) {
|
|
583
|
-
this.bodies.push({ url, ct, text });
|
|
584
|
-
if (this.isAnswerUrl(url)) this.answerStreams++;
|
|
585
|
-
}
|
|
586
|
-
}).catch(() => void 0);
|
|
587
|
-
this.pending.push(p);
|
|
588
|
-
}
|
|
589
|
-
};
|
|
590
|
-
|
|
591
|
-
// src/services/fanout/parse-helpers.ts
|
|
592
|
-
var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
|
|
593
|
-
var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
|
|
594
|
-
function domainOf(u) {
|
|
595
|
-
try {
|
|
596
|
-
return new URL(u).hostname.replace(/^www\./, "");
|
|
597
|
-
} catch {
|
|
598
|
-
return "";
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
function normUrl(u) {
|
|
602
|
-
try {
|
|
603
|
-
const x = new URL(u);
|
|
604
|
-
x.hash = "";
|
|
605
|
-
for (const k of [...x.searchParams.keys()]) {
|
|
606
|
-
if (TRACK.test(k)) x.searchParams.delete(k);
|
|
607
|
-
}
|
|
608
|
-
return x.toString().replace(/\/$/, "").toLowerCase();
|
|
609
|
-
} catch {
|
|
610
|
-
return String(u || "").toLowerCase();
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
function isExternal(u) {
|
|
614
|
-
const h = domainOf(u);
|
|
615
|
-
return Boolean(h) && !INTERNAL.test(h);
|
|
616
|
-
}
|
|
617
|
-
function unesc(s) {
|
|
618
|
-
try {
|
|
619
|
-
return JSON.parse('"' + s + '"');
|
|
620
|
-
} catch {
|
|
621
|
-
return String(s).replace(/\\(.)/g, "$1");
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
function stripCitationMarkers(s) {
|
|
625
|
-
return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
|
|
626
|
-
}
|
|
627
|
-
function sliceBalanced(text, openIdx) {
|
|
628
|
-
const open = text[openIdx];
|
|
629
|
-
const close = open === "[" ? "]" : "}";
|
|
630
|
-
let depth = 0;
|
|
631
|
-
let inStr = false;
|
|
632
|
-
let esc2 = false;
|
|
633
|
-
for (let i = openIdx; i < text.length; i++) {
|
|
634
|
-
const ch = text[i];
|
|
635
|
-
if (inStr) {
|
|
636
|
-
if (esc2) esc2 = false;
|
|
637
|
-
else if (ch === "\\") esc2 = true;
|
|
638
|
-
else if (ch === '"') inStr = false;
|
|
639
|
-
continue;
|
|
640
|
-
}
|
|
641
|
-
if (ch === '"') {
|
|
642
|
-
inStr = true;
|
|
643
|
-
continue;
|
|
644
|
-
}
|
|
645
|
-
if (ch === open) depth++;
|
|
646
|
-
else if (ch === close) {
|
|
647
|
-
depth--;
|
|
648
|
-
if (depth === 0) return text.slice(openIdx, i + 1);
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
return null;
|
|
652
|
-
}
|
|
653
|
-
function findJsonValues(text, field) {
|
|
654
|
-
const out = [];
|
|
655
|
-
const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
|
|
656
|
-
let m;
|
|
657
|
-
while (m = re.exec(text)) {
|
|
658
|
-
const openIdx = m.index + m[0].length - 1;
|
|
659
|
-
const slice = sliceBalanced(text, openIdx);
|
|
660
|
-
if (slice) {
|
|
661
|
-
try {
|
|
662
|
-
out.push(JSON.parse(slice));
|
|
663
|
-
} catch {
|
|
664
|
-
continue;
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
return out;
|
|
669
|
-
}
|
|
670
|
-
function stringsInArrayBody(arrBody) {
|
|
671
|
-
const out = [];
|
|
672
|
-
const re = /"((?:[^"\\]|\\.){2,300}?)"/g;
|
|
673
|
-
let m;
|
|
674
|
-
while (m = re.exec(arrBody)) out.push(unesc(m[1]));
|
|
675
|
-
return out;
|
|
676
|
-
}
|
|
677
|
-
function cleanQuery(q) {
|
|
678
|
-
const t = (q || "").trim();
|
|
679
|
-
if (t.length < 2 || t.length > 300) return null;
|
|
680
|
-
return t;
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
// src/services/fanout/adapters/chatgpt.ts
|
|
684
|
-
function roundOf(refId) {
|
|
685
|
-
const m = /^turn(\d+)/.exec(refId || "");
|
|
686
|
-
return m ? Number(m[1]) : null;
|
|
687
|
-
}
|
|
688
|
-
function normRef(r) {
|
|
689
|
-
if (typeof r === "string") {
|
|
690
|
-
const m = /turn(\d+)(search|news|ref|video|image|product)(\d+)/.exec(r);
|
|
691
|
-
return m ? `turn${m[1]}${m[2]}${m[3]}` : null;
|
|
692
|
-
}
|
|
693
|
-
if (r && typeof r === "object") {
|
|
694
|
-
const o = r;
|
|
695
|
-
if (o.turn_index != null) {
|
|
696
|
-
return `turn${o.turn_index}${o.ref_type || "search"}${o.ref_index != null ? o.ref_index : ""}`;
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
|
-
return null;
|
|
700
|
-
}
|
|
701
|
-
function refsInText(s) {
|
|
702
|
-
return (String(s || "").match(/turn\d+(?:search|news|ref|video|image|product)\d+/g) || []).map(normRef).filter((x) => Boolean(x));
|
|
703
|
-
}
|
|
704
|
-
function collectInto(arr, v) {
|
|
705
|
-
if (Array.isArray(v)) arr.push(...v);
|
|
706
|
-
else if (v && typeof v === "object") arr.push(v);
|
|
707
|
-
}
|
|
708
|
-
function reconstructFromSsePatches(raw) {
|
|
709
|
-
const contentReferences = [];
|
|
710
|
-
const searchResultGroups = [];
|
|
711
|
-
const citations = [];
|
|
712
|
-
const citedUrls = /* @__PURE__ */ new Set();
|
|
713
|
-
const state = { found: false };
|
|
714
|
-
const handleOp = (op) => {
|
|
715
|
-
if (!op || typeof op !== "object") return;
|
|
716
|
-
const o = op;
|
|
717
|
-
const v = o.v;
|
|
718
|
-
if (o.o === "patch" && Array.isArray(v)) {
|
|
719
|
-
state.found = true;
|
|
720
|
-
v.forEach(handleOp);
|
|
721
|
-
return;
|
|
722
|
-
}
|
|
723
|
-
const path = typeof o.p === "string" ? o.p : "";
|
|
724
|
-
if ((path === "" || path === "/message") && v && typeof v === "object") {
|
|
725
|
-
const root = v;
|
|
726
|
-
const msg = root.message || root;
|
|
727
|
-
const md = msg && msg.metadata;
|
|
728
|
-
if (md) {
|
|
729
|
-
collectInto(contentReferences, md.content_references);
|
|
730
|
-
collectInto(searchResultGroups, md.search_result_groups);
|
|
731
|
-
collectInto(citations, md.citations);
|
|
732
|
-
state.found = true;
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
|
-
if (!path) return;
|
|
736
|
-
if (/\/safe_urls$/.test(path) && Array.isArray(v)) {
|
|
737
|
-
for (const u of v) if (typeof u === "string") citedUrls.add(u);
|
|
738
|
-
state.found = true;
|
|
739
|
-
} else if (path.includes("content_references")) {
|
|
740
|
-
collectInto(contentReferences, v);
|
|
741
|
-
state.found = true;
|
|
742
|
-
} else if (path.includes("search_result_groups")) {
|
|
743
|
-
collectInto(searchResultGroups, v);
|
|
744
|
-
state.found = true;
|
|
745
|
-
} else if (path.includes("citations")) {
|
|
746
|
-
collectInto(citations, v);
|
|
747
|
-
state.found = true;
|
|
748
|
-
}
|
|
749
|
-
};
|
|
750
|
-
for (const line of raw.split("\n")) {
|
|
751
|
-
const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
|
|
752
|
-
if (!s || s === "[DONE]" || s[0] !== "{" && s[0] !== "[") continue;
|
|
753
|
-
let parsed;
|
|
754
|
-
try {
|
|
755
|
-
parsed = JSON.parse(s);
|
|
756
|
-
} catch {
|
|
757
|
-
continue;
|
|
758
|
-
}
|
|
759
|
-
const ops = Array.isArray(parsed) ? parsed : [parsed];
|
|
760
|
-
ops.forEach(handleOp);
|
|
761
|
-
}
|
|
762
|
-
if (!state.found) return raw;
|
|
763
|
-
const citedRefs = [...citedUrls].map((url) => ({ type: "grouped_webpages", items: [{ url }] }));
|
|
764
|
-
const synthetic = JSON.stringify({
|
|
765
|
-
content_references: [...contentReferences, ...citedRefs],
|
|
766
|
-
search_result_groups: searchResultGroups,
|
|
767
|
-
citations
|
|
768
|
-
});
|
|
769
|
-
return raw + "\n" + synthetic;
|
|
770
|
-
}
|
|
771
|
-
var chatgptAdapter = {
|
|
772
|
-
platform: "ChatGPT",
|
|
773
|
-
matchesHost(hostname) {
|
|
774
|
-
return /(^|\.)chatgpt\.com$|(^|\.)openai\.com$/i.test(hostname);
|
|
775
|
-
},
|
|
776
|
-
parse(rawInput, prompt) {
|
|
777
|
-
const raw = reconstructFromSsePatches(rawInput);
|
|
778
|
-
const sources = /* @__PURE__ */ new Map();
|
|
779
|
-
const snippets = /* @__PURE__ */ new Map();
|
|
780
|
-
const queries = /* @__PURE__ */ new Set();
|
|
781
|
-
const unmapped = /* @__PURE__ */ new Set();
|
|
782
|
-
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
783
|
-
const ensure = (url) => {
|
|
784
|
-
if (!isExternal(url)) return null;
|
|
785
|
-
const k = normUrl(url);
|
|
786
|
-
let s = sources.get(k);
|
|
787
|
-
if (!s) {
|
|
788
|
-
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
|
|
789
|
-
sources.set(k, s);
|
|
790
|
-
}
|
|
791
|
-
return s;
|
|
792
|
-
};
|
|
793
|
-
const addSnippet = (url, title, text) => {
|
|
794
|
-
const clean = stripCitationMarkers(text);
|
|
795
|
-
if (!isExternal(url) || clean.length < 2) return;
|
|
796
|
-
snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean.slice(0, 400) });
|
|
797
|
-
};
|
|
798
|
-
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
799
|
-
for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
|
|
800
|
-
for (const q of stringsInArrayBody(m[1])) {
|
|
801
|
-
const c = cleanQuery(q);
|
|
802
|
-
if (c) queries.add(c);
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
|
-
for (const m of raw.matchAll(/"(?:search_queries|queries|search_query)"\s*:\s*\[([^\]]*)\]/g)) {
|
|
806
|
-
for (const q of stringsInArrayBody(m[1])) {
|
|
807
|
-
const c = cleanQuery(q);
|
|
808
|
-
if (c) queries.add(c);
|
|
809
|
-
}
|
|
810
|
-
}
|
|
811
|
-
for (const v of findJsonValues(raw, "search_result_groups")) {
|
|
812
|
-
if (!Array.isArray(v)) continue;
|
|
813
|
-
for (const g of v) {
|
|
814
|
-
const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
|
|
815
|
-
for (const e of entries) {
|
|
816
|
-
const url = typeof e.url === "string" ? e.url : "";
|
|
817
|
-
const s = ensure(url);
|
|
818
|
-
if (!s) continue;
|
|
819
|
-
if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
|
|
820
|
-
const ref = normRef(e.ref_id);
|
|
821
|
-
if (ref && s.round == null) s.round = roundOf(ref);
|
|
822
|
-
if (typeof e.snippet === "string" && e.snippet) addSnippet(url, s.title, e.snippet);
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"(?:[\s\S]{0,120}?"(?:title|name)"\s*:\s*"((?:[^"\\]|\\.){0,200})")?/g)) {
|
|
827
|
-
const s = ensure(m[1]);
|
|
828
|
-
if (s && !s.title && m[2]) s.title = unesc(m[2]).slice(0, 200);
|
|
829
|
-
}
|
|
830
|
-
let spanTotal = 0;
|
|
831
|
-
for (const cr of findJsonValues(raw, "content_references")) {
|
|
832
|
-
if (!Array.isArray(cr)) continue;
|
|
833
|
-
for (const ref of cr) {
|
|
834
|
-
const type = typeof ref.type === "string" ? ref.type : "";
|
|
835
|
-
const isBundle = /sources_footnote|nav_list|alt_text/.test(type);
|
|
836
|
-
const refs = refsInText(typeof ref.matched_text === "string" ? ref.matched_text : "");
|
|
837
|
-
if (!isBundle) spanTotal++;
|
|
838
|
-
const ord = typeof ref.start_idx === "number" ? ref.start_idx : isBundle ? Infinity : spanTotal;
|
|
839
|
-
const items = Array.isArray(ref.items) ? ref.items : [];
|
|
840
|
-
items.forEach((it, idx) => {
|
|
841
|
-
const refForItem = refs.length === items.length ? refs[idx] : refs[0];
|
|
842
|
-
const urls = [it.url].concat((Array.isArray(it.supporting_websites) ? it.supporting_websites : []).map((s) => s?.url)).filter((u) => typeof u === "string");
|
|
843
|
-
for (const u of urls) {
|
|
844
|
-
const s = ensure(u);
|
|
845
|
-
if (!s) continue;
|
|
846
|
-
if (!isBundle) {
|
|
847
|
-
s.cited = true;
|
|
848
|
-
s.timesCited++;
|
|
849
|
-
if (ord < s.firstOrder) s.firstOrder = ord;
|
|
850
|
-
} else if (!s.cited) {
|
|
851
|
-
s.cited = true;
|
|
852
|
-
}
|
|
853
|
-
if (refForItem && s.round == null) s.round = roundOf(refForItem);
|
|
854
|
-
}
|
|
855
|
-
if (typeof it.url === "string" && typeof it.snippet === "string") addSnippet(it.url, typeof it.title === "string" ? it.title : "", it.snippet);
|
|
856
|
-
});
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
for (const v of findJsonValues(raw, "citations")) {
|
|
860
|
-
if (!Array.isArray(v)) continue;
|
|
861
|
-
v.forEach((c, i) => {
|
|
862
|
-
const md = c && typeof c === "object" ? c.metadata : null;
|
|
863
|
-
if (!md || typeof md.url !== "string") return;
|
|
864
|
-
const s = ensure(md.url);
|
|
865
|
-
if (!s) return;
|
|
866
|
-
s.cited = true;
|
|
867
|
-
s.timesCited++;
|
|
868
|
-
const ord = typeof c.start_ix === "number" ? c.start_ix : i;
|
|
869
|
-
if (ord < s.firstOrder) s.firstOrder = ord;
|
|
870
|
-
if (typeof md.text === "string") addSnippet(md.url, typeof md.title === "string" ? md.title : "", md.text);
|
|
871
|
-
});
|
|
872
|
-
}
|
|
873
|
-
for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
|
|
874
|
-
const url = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
|
|
875
|
-
const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
|
|
876
|
-
if (url) {
|
|
877
|
-
ensure(url);
|
|
878
|
-
addSnippet(url, title ? unesc(title) : "", unesc(m[2]));
|
|
879
|
-
}
|
|
880
|
-
}
|
|
881
|
-
const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
|
|
882
|
-
const slugMatches = modelMatches.length ? modelMatches : [...raw.matchAll(/"model_slug"\s*:\s*"([^"]+)"/g)];
|
|
883
|
-
if (slugMatches.length) meta.model = slugMatches[slugMatches.length - 1][1];
|
|
884
|
-
const finish = raw.match(/"finish_details"\s*:\s*\{[^}]*"type"\s*:\s*"([^"]+)"/);
|
|
885
|
-
if (finish) meta.finishType = finish[1];
|
|
886
|
-
const searchRounds = (raw.match(/"recipient"\s*:\s*"(?:web|browser)"[\s\S]{0,400}?"content_type"\s*:\s*"code"[\s\S]{0,200}?search\(/g) || []).length;
|
|
887
|
-
const sourceRounds = /* @__PURE__ */ new Set();
|
|
888
|
-
for (const s of sources.values()) if (s.round != null) sourceRounds.add(s.round);
|
|
889
|
-
meta.rounds = Math.max(searchRounds, sourceRounds.size);
|
|
890
|
-
const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited || a.firstOrder - b.firstOrder : a.cited ? -1 : 1).map((s) => ({
|
|
891
|
-
url: s.url,
|
|
892
|
-
domain: s.domain,
|
|
893
|
-
title: s.title || s.domain,
|
|
894
|
-
cited: s.cited,
|
|
895
|
-
timesCited: s.timesCited,
|
|
896
|
-
snippet: snippets.get(normUrl(s.url))?.text || "",
|
|
897
|
-
round: s.round
|
|
898
|
-
}));
|
|
899
|
-
return {
|
|
900
|
-
subQueries: [...queries],
|
|
901
|
-
sources: ordered,
|
|
902
|
-
snippets: [...snippets.values()],
|
|
903
|
-
meta,
|
|
904
|
-
unmappedKeys: [...unmapped].slice(0, 50)
|
|
905
|
-
};
|
|
906
|
-
}
|
|
907
|
-
};
|
|
908
|
-
|
|
909
|
-
// src/services/fanout/adapters/claude.ts
|
|
910
|
-
function isFavicon(u) {
|
|
911
|
-
return /\/s2\/favicons/i.test(u) || /favicon/i.test(u);
|
|
912
|
-
}
|
|
913
|
-
var claudeAdapter = {
|
|
914
|
-
platform: "Claude",
|
|
915
|
-
matchesHost(hostname) {
|
|
916
|
-
return /(^|\.)claude\.ai$/i.test(hostname);
|
|
917
|
-
},
|
|
918
|
-
parse(raw, prompt) {
|
|
919
|
-
const sources = /* @__PURE__ */ new Map();
|
|
920
|
-
const snippets = /* @__PURE__ */ new Map();
|
|
921
|
-
const queries = /* @__PURE__ */ new Set();
|
|
922
|
-
const unmapped = /* @__PURE__ */ new Set();
|
|
923
|
-
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
924
|
-
const ensure = (url) => {
|
|
925
|
-
if (!url || !isExternal(url) || isFavicon(url)) return null;
|
|
926
|
-
const k = normUrl(url);
|
|
927
|
-
let s = sources.get(k);
|
|
928
|
-
if (!s) {
|
|
929
|
-
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
|
|
930
|
-
sources.set(k, s);
|
|
931
|
-
}
|
|
932
|
-
return s;
|
|
933
|
-
};
|
|
934
|
-
const toolInputs = /* @__PURE__ */ new Map();
|
|
935
|
-
const searchBlocks = /* @__PURE__ */ new Set();
|
|
936
|
-
const finalizeTool = (index) => {
|
|
937
|
-
const t = toolInputs.get(index);
|
|
938
|
-
if (!t || !/web_search|web_fetch|search|browse/i.test(t.name)) return;
|
|
939
|
-
try {
|
|
940
|
-
const input = JSON.parse(t.buf);
|
|
941
|
-
if (typeof input.query === "string") {
|
|
942
|
-
const q = cleanQuery(input.query);
|
|
943
|
-
if (q) queries.add(q);
|
|
944
|
-
}
|
|
945
|
-
} catch {
|
|
946
|
-
const m = t.buf.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)"/);
|
|
947
|
-
if (m) {
|
|
948
|
-
const q = cleanQuery(unesc(m[1]));
|
|
949
|
-
if (q) queries.add(q);
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
};
|
|
953
|
-
for (const line of raw.split("\n")) {
|
|
954
|
-
const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
|
|
955
|
-
if (!s || s[0] !== "{" && s[0] !== "[") continue;
|
|
956
|
-
let ev;
|
|
957
|
-
try {
|
|
958
|
-
ev = JSON.parse(s);
|
|
959
|
-
} catch {
|
|
960
|
-
continue;
|
|
961
|
-
}
|
|
962
|
-
const type = ev.type;
|
|
963
|
-
if (type === "content_block_start") {
|
|
964
|
-
const cb = ev.content_block || {};
|
|
965
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
966
|
-
if (cb.type === "tool_use" && index >= 0) {
|
|
967
|
-
const name = typeof cb.name === "string" ? cb.name : "";
|
|
968
|
-
toolInputs.set(index, { name, buf: "" });
|
|
969
|
-
if (/web_search|web_fetch|search|browse/i.test(name)) searchBlocks.add(index);
|
|
970
|
-
}
|
|
971
|
-
} else if (type === "content_block_delta") {
|
|
972
|
-
const d = ev.delta || {};
|
|
973
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
974
|
-
if (d.type === "input_json_delta" && typeof d.partial_json === "string") {
|
|
975
|
-
const t = toolInputs.get(index);
|
|
976
|
-
if (t) t.buf += d.partial_json;
|
|
977
|
-
} else if (d.type === "citation_start_delta") {
|
|
978
|
-
const c = d.citation || {};
|
|
979
|
-
if (typeof c.url === "string") {
|
|
980
|
-
const src = ensure(c.url);
|
|
981
|
-
if (src) {
|
|
982
|
-
src.cited = true;
|
|
983
|
-
src.timesCited++;
|
|
984
|
-
if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
|
|
985
|
-
if (typeof c.cited_text === "string" && c.cited_text) {
|
|
986
|
-
snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
} else if (type === "content_block_stop") {
|
|
992
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
993
|
-
if (index >= 0) finalizeTool(index);
|
|
994
|
-
} else if (type === "message_delta" || type === "message_stop" || type === "message_start") {
|
|
995
|
-
const d = ev.delta || ev.message || {};
|
|
996
|
-
if (typeof d.stop_reason === "string" && !meta.finishType) meta.finishType = d.stop_reason;
|
|
997
|
-
if (typeof d.model === "string" && !meta.model) meta.model = d.model;
|
|
998
|
-
}
|
|
999
|
-
}
|
|
1000
|
-
for (const index of searchBlocks) finalizeTool(index);
|
|
1001
|
-
for (const m of raw.matchAll(/"title"\s*:\s*"((?:[^"\\]|\\.){1,200})"\s*,\s*"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) {
|
|
1002
|
-
const src = ensure(m[2]);
|
|
1003
|
-
if (src && !src.title) src.title = unesc(m[1]).slice(0, 200);
|
|
1004
|
-
}
|
|
1005
|
-
for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) ensure(m[1]);
|
|
1006
|
-
for (const m of raw.matchAll(/\\"title\\"\s*:\s*\\"((?:[^"\\]|\\.){1,160}?)\\"[\s\S]{0,160}?\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) {
|
|
1007
|
-
const src = ensure(m[2]);
|
|
1008
|
-
if (src && !src.title) src.title = unesc(unesc(m[1])).slice(0, 200);
|
|
1009
|
-
}
|
|
1010
|
-
for (const m of raw.matchAll(/\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) ensure(m[1]);
|
|
1011
|
-
if (!meta.model) {
|
|
1012
|
-
const mm = raw.match(/"model"\s*:\s*"(claude[^"]+)"/);
|
|
1013
|
-
if (mm) meta.model = mm[1];
|
|
1014
|
-
}
|
|
1015
|
-
meta.rounds = queries.size > 0 ? searchBlocks.size : 0;
|
|
1016
|
-
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
1017
|
-
const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited : a.cited ? -1 : 1).map((s) => ({
|
|
1018
|
-
url: s.url,
|
|
1019
|
-
domain: s.domain,
|
|
1020
|
-
title: s.title || s.domain,
|
|
1021
|
-
cited: s.cited,
|
|
1022
|
-
timesCited: s.timesCited,
|
|
1023
|
-
snippet: snippets.get(normUrl(s.url))?.text || "",
|
|
1024
|
-
round: s.round
|
|
1025
|
-
}));
|
|
1026
|
-
return {
|
|
1027
|
-
subQueries: [...queries],
|
|
1028
|
-
sources: ordered,
|
|
1029
|
-
snippets: [...snippets.values()],
|
|
1030
|
-
meta,
|
|
1031
|
-
unmappedKeys: [...unmapped].slice(0, 50)
|
|
1032
|
-
};
|
|
1033
|
-
}
|
|
1034
|
-
};
|
|
1035
|
-
|
|
1036
|
-
// src/services/fanout/adapters/index.ts
|
|
1037
|
-
var FANOUT_ADAPTERS = [chatgptAdapter, claudeAdapter];
|
|
1038
|
-
function adapterForHost(hostname) {
|
|
1039
|
-
return FANOUT_ADAPTERS.find((a) => a.matchesHost(hostname)) ?? null;
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
// src/services/fanout/build.ts
|
|
1043
|
-
function buildFanoutResult(ctx) {
|
|
1044
|
-
const { platform, parsed, ready, rawBytes } = ctx;
|
|
1045
|
-
const browsedUrls = parsed.sources;
|
|
1046
|
-
const citedUrls = browsedUrls.filter((s) => s.cited);
|
|
1047
|
-
const browsedOnly = browsedUrls.filter((s) => !s.cited);
|
|
1048
|
-
const prompt = ctx.prompt || parsed.meta.title || "";
|
|
1049
|
-
const result = {
|
|
1050
|
-
platform,
|
|
1051
|
-
capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1052
|
-
prompt,
|
|
1053
|
-
meta: parsed.meta,
|
|
1054
|
-
subQueries: parsed.subQueries,
|
|
1055
|
-
browsedUrls,
|
|
1056
|
-
citedUrls,
|
|
1057
|
-
browsedOnly,
|
|
1058
|
-
snippets: parsed.snippets,
|
|
1059
|
-
counts: {
|
|
1060
|
-
subQueries: parsed.subQueries.length,
|
|
1061
|
-
browsed: browsedUrls.length,
|
|
1062
|
-
cited: citedUrls.length,
|
|
1063
|
-
browsedOnly: browsedOnly.length
|
|
1064
|
-
}
|
|
1065
|
-
};
|
|
1066
|
-
if (parsed.subQueries.length === 0 && browsedUrls.length === 0) {
|
|
1067
|
-
result.debug = {
|
|
1068
|
-
interceptorReady: ready,
|
|
1069
|
-
rawBytes,
|
|
1070
|
-
unmappedKeys: parsed.unmappedKeys,
|
|
1071
|
-
note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. The session must be opened with MCP_SCRAPER_BROWSER_MODE=local and navigated to chatgpt.com or claude.ai after opening, so the hook installs before the site scripts run."
|
|
1072
|
-
};
|
|
1073
|
-
}
|
|
1074
|
-
return result;
|
|
1075
|
-
}
|
|
1076
|
-
|
|
1077
|
-
// src/services/fanout/classify.ts
|
|
1078
|
-
var SOCIAL = /(^|\.)(youtube\.com|youtu\.be|instagram\.com|tiktok\.com|facebook\.com|fb\.com|x\.com|twitter\.com|linkedin\.com|pinterest\.com|threads\.net|vimeo\.com)$/i;
|
|
1079
|
-
var ENCYCLOPEDIA = /(^|\.)(wikipedia\.org|wikimedia\.org|britannica\.com|merriam-webster\.com|dictionary\.com|fandom\.com|investopedia\.com)$/i;
|
|
1080
|
-
var REVIEW = /(^|\.)(yelp\.com|trustpilot\.com|g2\.com|capterra\.com|angi\.com|angieslist\.com|bbb\.org|consumeraffairs\.com|bestcompany\.com|sitejabber\.com|clutch\.co|bestpickreports\.com|homeguide\.com|porch\.com|ontoplist\.com|thumbtack\.com)$/i;
|
|
1081
|
-
var NEWS = /(^|\.)(reuters\.com|apnews\.com|nytimes\.com|wsj\.com|washingtonpost\.com|bloomberg\.com|cnbc\.com|forbes\.com|bbc\.com|theguardian\.com|cnn\.com|npr\.org|usatoday\.com|expressnews\.com|mysanantonio\.com|beaumontenterprise\.com)$/i;
|
|
1082
|
-
function classifyDomain(domain, firstPartyDomain) {
|
|
1083
|
-
const d = domain.toLowerCase().replace(/^www\./, "");
|
|
1084
|
-
if (firstPartyDomain) {
|
|
1085
|
-
const fp = firstPartyDomain.toLowerCase().replace(/^www\./, "");
|
|
1086
|
-
if (fp && (d === fp || d.endsWith("." + fp))) return "First-party/vendor";
|
|
1087
|
-
}
|
|
1088
|
-
if (/(^|\.)reddit\.com$/i.test(d)) return "Reddit";
|
|
1089
|
-
if (SOCIAL.test(d)) return "Social/video";
|
|
1090
|
-
if (ENCYCLOPEDIA.test(d)) return "Encyclopedia";
|
|
1091
|
-
if (REVIEW.test(d)) return "Review site";
|
|
1092
|
-
if (NEWS.test(d) || /(^|\.)news\./i.test(d)) return "News/media";
|
|
1093
|
-
if (/^docs?\.|^developer\.|^developers\.|readthedocs|\.dev$/i.test(d)) return "Docs";
|
|
1094
|
-
return "Blog";
|
|
1095
|
-
}
|
|
1096
|
-
function emptyCategoryCounts() {
|
|
1097
|
-
return { "First-party/vendor": 0, "News/media": 0, Reddit: 0, "Social/video": 0, Encyclopedia: 0, "Review site": 0, Docs: 0, Blog: 0 };
|
|
1098
|
-
}
|
|
1099
|
-
function enrichFanout(result, opts = {}) {
|
|
1100
|
-
const addSiteType = (s) => ({ ...s, siteType: classifyDomain(s.domain, opts.firstPartyDomain) });
|
|
1101
|
-
const browsedUrls = result.browsedUrls.map(addSiteType);
|
|
1102
|
-
const citedUrls = result.citedUrls.map(addSiteType);
|
|
1103
|
-
const browsedOnly = result.browsedOnly.map(addSiteType);
|
|
1104
|
-
const byCategory = emptyCategoryCounts();
|
|
1105
|
-
const domainAgg = /* @__PURE__ */ new Map();
|
|
1106
|
-
for (const s of browsedUrls) {
|
|
1107
|
-
byCategory[s.siteType]++;
|
|
1108
|
-
const cur = domainAgg.get(s.domain);
|
|
1109
|
-
if (!cur) domainAgg.set(s.domain, { domain: s.domain, count: 1, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType });
|
|
1110
|
-
else {
|
|
1111
|
-
cur.count++;
|
|
1112
|
-
cur.cited = cur.cited || s.cited;
|
|
1113
|
-
cur.timesCited += s.timesCited;
|
|
1114
|
-
}
|
|
1115
|
-
}
|
|
1116
|
-
const topSites = [...domainAgg.values()].sort((a, b) => b.count - a.count || b.timesCited - a.timesCited);
|
|
1117
|
-
const citationOrder = citedUrls.slice().sort((a, b) => b.timesCited - a.timesCited).map((s, i) => ({ rank: i + 1, domain: s.domain, url: s.url, timesCited: s.timesCited }));
|
|
1118
|
-
return {
|
|
1119
|
-
platform: result.platform,
|
|
1120
|
-
capturedAt: result.capturedAt,
|
|
1121
|
-
prompt: result.prompt,
|
|
1122
|
-
meta: result.meta,
|
|
1123
|
-
queries: result.subQueries,
|
|
1124
|
-
browsedUrls,
|
|
1125
|
-
citedUrls,
|
|
1126
|
-
browsedOnly,
|
|
1127
|
-
snippets: result.snippets,
|
|
1128
|
-
counts: result.counts,
|
|
1129
|
-
aggregates: { topSites, citationOrder, byCategory },
|
|
1130
|
-
firstPartyDomain: opts.firstPartyDomain ? opts.firstPartyDomain.toLowerCase().replace(/^www\./, "") : null,
|
|
1131
|
-
...result.debug ? { debug: result.debug } : {}
|
|
1132
|
-
};
|
|
1133
|
-
}
|
|
1134
|
-
|
|
1135
|
-
// src/services/fanout/export.ts
|
|
1136
|
-
var import_node_fs2 = require("fs");
|
|
1137
|
-
var import_node_os3 = require("os");
|
|
1138
|
-
var import_node_path3 = require("path");
|
|
1139
|
-
var import_papaparse = __toESM(require("papaparse"), 1);
|
|
1140
|
-
function outputBaseDir() {
|
|
1141
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
|
|
1142
|
-
}
|
|
1143
|
-
function safe(value) {
|
|
1144
|
-
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
1145
|
-
}
|
|
1146
|
-
function writeTable(path, rows, delimiter) {
|
|
1147
|
-
(0, import_node_fs2.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
1148
|
-
}
|
|
1149
|
-
function exportFanout(enriched) {
|
|
1150
|
-
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
1151
|
-
const dir = (0, import_node_path3.join)(outputBaseDir(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
1152
|
-
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
|
1153
|
-
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
1154
|
-
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
1155
|
-
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
1156
|
-
return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
|
|
1157
|
-
});
|
|
1158
|
-
const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
1159
|
-
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
1160
|
-
const paths = {
|
|
1161
|
-
dir,
|
|
1162
|
-
json: (0, import_node_path3.join)(dir, "fanout.json"),
|
|
1163
|
-
queriesCsv: (0, import_node_path3.join)(dir, "queries.csv"),
|
|
1164
|
-
queriesTsv: (0, import_node_path3.join)(dir, "queries.tsv"),
|
|
1165
|
-
citationsCsv: (0, import_node_path3.join)(dir, "citations.csv"),
|
|
1166
|
-
sourcesCsv: (0, import_node_path3.join)(dir, "sources.csv"),
|
|
1167
|
-
domainsCsv: (0, import_node_path3.join)(dir, "domains.csv"),
|
|
1168
|
-
report: (0, import_node_path3.join)(dir, "report.html")
|
|
1169
|
-
};
|
|
1170
|
-
(0, import_node_fs2.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
|
|
1171
|
-
writeTable(paths.queriesCsv, queryRows, ",");
|
|
1172
|
-
writeTable(paths.queriesTsv, queryRows, " ");
|
|
1173
|
-
writeTable(paths.citationsCsv, citationRows, ",");
|
|
1174
|
-
writeTable(paths.sourcesCsv, sourceRows, ",");
|
|
1175
|
-
writeTable(paths.domainsCsv, domainRows, ",");
|
|
1176
|
-
(0, import_node_fs2.writeFileSync)(paths.report, renderReportHtml(enriched));
|
|
1177
|
-
return paths;
|
|
1178
|
-
}
|
|
1179
|
-
function esc(s) {
|
|
1180
|
-
return String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
1181
|
-
}
|
|
1182
|
-
function renderReportHtml(enriched) {
|
|
1183
|
-
const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
|
|
1184
|
-
return `<!doctype html>
|
|
1185
|
-
<html lang="en">
|
|
1186
|
-
<head>
|
|
1187
|
-
<meta charset="utf-8" />
|
|
1188
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
1189
|
-
<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
|
|
1190
|
-
<style>
|
|
1191
|
-
:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
|
|
1192
|
-
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
|
|
1193
|
-
.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
|
|
1194
|
-
h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
|
|
1195
|
-
.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
|
|
1196
|
-
.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
|
|
1197
|
-
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
1198
|
-
.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
|
|
1199
|
-
.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
|
|
1200
|
-
.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
|
|
1201
|
-
.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
|
|
1202
|
-
.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
|
|
1203
|
-
.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
|
|
1204
|
-
.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
|
|
1205
|
-
.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
|
|
1206
|
-
.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
|
|
1207
|
-
.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
|
|
1208
|
-
.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
|
|
1209
|
-
.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
|
|
1210
|
-
.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
|
|
1211
|
-
.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
|
|
1212
|
-
.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
|
|
1213
|
-
.hide{display:none}
|
|
1214
|
-
</style>
|
|
1215
|
-
</head>
|
|
1216
|
-
<body><div class="wrap">
|
|
1217
|
-
<h1>AI Search Fan-Out</h1>
|
|
1218
|
-
<p class="sub" id="sub"></p>
|
|
1219
|
-
<div class="stat" id="stat"></div>
|
|
1220
|
-
<div class="cards" id="cards"></div>
|
|
1221
|
-
<div id="charts"></div>
|
|
1222
|
-
<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
|
|
1223
|
-
<div class="chips" id="chips"></div>
|
|
1224
|
-
<div id="list"></div>
|
|
1225
|
-
</div>
|
|
1226
|
-
<script>
|
|
1227
|
-
const D=${data};
|
|
1228
|
-
const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
|
|
1229
|
-
document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
|
|
1230
|
-
document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
|
|
1231
|
-
const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
|
|
1232
|
-
const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
|
|
1233
|
-
function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
|
|
1234
|
-
const charts=document.getElementById('charts');
|
|
1235
|
-
charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
|
|
1236
|
-
let tab='q',filter=null;
|
|
1237
|
-
const chips=document.getElementById('chips'),list=document.getElementById('list');
|
|
1238
|
-
function render(){
|
|
1239
|
-
chips.innerHTML='';list.innerHTML='';
|
|
1240
|
-
if(tab==='q'){
|
|
1241
|
-
D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
|
|
1242
|
-
} else {
|
|
1243
|
-
const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
|
|
1244
|
-
Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
|
|
1245
|
-
D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}
|
|
1249
|
-
document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
|
|
1250
|
-
render();
|
|
1251
|
-
</script>
|
|
1252
|
-
</body></html>`;
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
// src/services/fanout/run-capture.ts
|
|
1256
|
-
var COMPOSER_SELECTORS = [
|
|
1257
|
-
"#prompt-textarea",
|
|
1258
|
-
"div#prompt-textarea",
|
|
1259
|
-
'textarea[data-testid="prompt-textarea"]',
|
|
1260
|
-
'div.ProseMirror[contenteditable="true"]',
|
|
1261
|
-
'div[contenteditable="true"]',
|
|
1262
|
-
"textarea"
|
|
1263
|
-
];
|
|
1264
|
-
async function sendPrompt(page, text) {
|
|
1265
|
-
for (const selector of COMPOSER_SELECTORS) {
|
|
1266
|
-
const locator = page.locator(selector).first();
|
|
1267
|
-
try {
|
|
1268
|
-
await locator.waitFor({ state: "visible", timeout: 12e3 });
|
|
1269
|
-
} catch {
|
|
1270
|
-
continue;
|
|
1271
|
-
}
|
|
1272
|
-
await locator.click({ timeout: 4e3 }).catch(() => void 0);
|
|
1273
|
-
await locator.focus().catch(() => void 0);
|
|
1274
|
-
await page.keyboard.type(text, { delay: 12 });
|
|
1275
|
-
await page.waitForTimeout(300).catch(() => void 0);
|
|
1276
|
-
await page.keyboard.press("Enter");
|
|
1277
|
-
return true;
|
|
1278
|
-
}
|
|
1279
|
-
return false;
|
|
1280
|
-
}
|
|
1281
|
-
async function waitForAnswer(page, cap, baseline, waitMs) {
|
|
1282
|
-
const start = Date.now();
|
|
1283
|
-
while (Date.now() - start < waitMs && cap.answerStreamCount() <= baseline) {
|
|
1284
|
-
await page.waitForTimeout(400).catch(() => void 0);
|
|
1285
|
-
}
|
|
1286
|
-
await page.waitForTimeout(1e3).catch(() => void 0);
|
|
1287
|
-
}
|
|
1288
|
-
async function runFanoutCapture(page, input) {
|
|
1289
|
-
const cap = new FanoutCdpCapture();
|
|
1290
|
-
await cap.attach(page);
|
|
1291
|
-
try {
|
|
1292
|
-
const baseline = cap.answerStreamCount();
|
|
1293
|
-
if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
|
|
1294
|
-
const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
|
|
1295
|
-
await waitForAnswer(page, cap, baseline, waitMs);
|
|
1296
|
-
await cap.drain();
|
|
1297
|
-
let hostname = "";
|
|
1298
|
-
try {
|
|
1299
|
-
hostname = new URL(page.url()).hostname;
|
|
1300
|
-
} catch {
|
|
1301
|
-
hostname = "";
|
|
1302
|
-
}
|
|
1303
|
-
const adapter = adapterForHost(hostname);
|
|
1304
|
-
if (!adapter) {
|
|
1305
|
-
throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
|
|
1306
|
-
}
|
|
1307
|
-
const text = cap.bestAnswerText();
|
|
1308
|
-
const parsed = adapter.parse(text, input.prompt || "");
|
|
1309
|
-
const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text.length });
|
|
1310
|
-
const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
|
|
1311
|
-
const exports2 = input.export ? exportFanout(enriched) : null;
|
|
1312
|
-
return { result: enriched, exports: exports2 };
|
|
1313
|
-
} finally {
|
|
1314
|
-
await cap.detach().catch(() => void 0);
|
|
1315
|
-
}
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
// src/services/browser-agent/local-browser-agent-service.ts
|
|
1319
|
-
import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
|
|
1320
|
-
function directProfileDir() {
|
|
1321
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
|
|
1322
|
-
}
|
|
1323
|
-
function configuredProfileName(profile) {
|
|
1324
|
-
return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directProfileDir() ? (0, import_node_path4.basename)(directProfileDir()) : "default");
|
|
1325
|
-
}
|
|
1326
|
-
function launchExecutablePath(profile) {
|
|
1327
|
-
const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || profile.browserExecutablePath || defaultChromeExecutablePath();
|
|
1328
|
-
return configured && (0, import_node_fs3.existsSync)(configured) ? configured : void 0;
|
|
1329
|
-
}
|
|
1330
|
-
function ignoreDefaultArgsFor(executablePath) {
|
|
1331
|
-
if (!executablePath) return void 0;
|
|
1332
|
-
if (process.platform !== "darwin") return void 0;
|
|
1333
|
-
if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
|
|
1334
|
-
return ["--password-store=basic", "--use-mock-keychain"];
|
|
1335
|
-
}
|
|
1336
|
-
function normalizeKeyCombo(value) {
|
|
1337
|
-
return value.split("+").map((part) => {
|
|
1338
|
-
const trimmed = part.trim();
|
|
1339
|
-
const lower = trimmed.toLowerCase();
|
|
1340
|
-
if (lower === "ctrl" || lower === "control") return "Control";
|
|
1341
|
-
if (lower === "cmd" || lower === "command" || lower === "meta") return "Meta";
|
|
1342
|
-
if (lower === "option" || lower === "alt") return "Alt";
|
|
1343
|
-
if (lower === "return") return "Enter";
|
|
1344
|
-
if (lower === "esc") return "Escape";
|
|
1345
|
-
if (trimmed.length === 1) return trimmed.toUpperCase();
|
|
1346
|
-
return trimmed;
|
|
1347
|
-
}).join("+");
|
|
1348
|
-
}
|
|
1349
|
-
function sessionSummary(session) {
|
|
1350
|
-
return {
|
|
1351
|
-
session_id: session.sessionId,
|
|
1352
|
-
status: session.status,
|
|
1353
|
-
label: session.label,
|
|
1354
|
-
profile: session.profileName,
|
|
1355
|
-
user_data_dir: session.userDataDir,
|
|
1356
|
-
created_at: session.createdAt,
|
|
1357
|
-
last_action_at: session.lastActionAt,
|
|
1358
|
-
closed_at: session.closedAt,
|
|
1359
|
-
watch_url: `local://browser-session/${session.sessionId}`,
|
|
1360
|
-
url: session.status === "open" ? session.page.url() : null
|
|
1361
|
-
};
|
|
1362
|
-
}
|
|
1363
|
-
var LocalBrowserAgentService = class {
|
|
1364
|
-
contexts = /* @__PURE__ */ new Map();
|
|
1365
|
-
sessions = /* @__PURE__ */ new Map();
|
|
1366
|
-
async open(input) {
|
|
1367
|
-
const profile = await this.resolveProfile(input.profile);
|
|
1368
|
-
const contextState = await this.ensureContext(profile);
|
|
1369
|
-
const pages = contextState.context.pages();
|
|
1370
|
-
const page = pages.length === 1 && !this.hasOpenSessionForContext(contextState.key) ? pages[0] : await contextState.context.newPage();
|
|
1371
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1372
|
-
const sessionId = `lbs_${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
1373
|
-
const session = {
|
|
1374
|
-
sessionId,
|
|
1375
|
-
label: input.label ?? null,
|
|
1376
|
-
page,
|
|
1377
|
-
contextKey: contextState.key,
|
|
1378
|
-
profileName: profile.name,
|
|
1379
|
-
userDataDir: profile.userDataDir,
|
|
1380
|
-
createdAt: now,
|
|
1381
|
-
lastActionAt: now,
|
|
1382
|
-
closedAt: null,
|
|
1383
|
-
status: "open"
|
|
1384
|
-
};
|
|
1385
|
-
const timeoutSeconds = input.timeout_seconds ?? 600;
|
|
1386
|
-
if (timeoutSeconds > 0) {
|
|
1387
|
-
session.timeout = setTimeout(() => {
|
|
1388
|
-
void this.close(sessionId);
|
|
1389
|
-
}, timeoutSeconds * 1e3);
|
|
1390
|
-
session.timeout.unref();
|
|
1391
|
-
}
|
|
1392
|
-
this.sessions.set(sessionId, session);
|
|
1393
|
-
if (input.url) await this.goto(sessionId, input.url);
|
|
1394
|
-
return {
|
|
1395
|
-
session_id: sessionId,
|
|
1396
|
-
watch_url: `local://browser-session/${sessionId}`,
|
|
1397
|
-
live_view_url: null,
|
|
1398
|
-
profile: profile.name,
|
|
1399
|
-
user_data_dir: profile.userDataDir,
|
|
1400
|
-
url: input.url ?? null,
|
|
1401
|
-
mode: "local"
|
|
1402
|
-
};
|
|
1403
|
-
}
|
|
1404
|
-
async screenshot(sessionId) {
|
|
1405
|
-
const session = this.requireOpenSession(sessionId);
|
|
1406
|
-
await this.waitForPage(session.page);
|
|
1407
|
-
const image = await session.page.screenshot({ type: "png", fullPage: false });
|
|
1408
|
-
const page = await this.read(sessionId);
|
|
1409
|
-
return {
|
|
1410
|
-
...page,
|
|
1411
|
-
image_base64: image.toString("base64"),
|
|
1412
|
-
mime_type: "image/png"
|
|
1413
|
-
};
|
|
1414
|
-
}
|
|
1415
|
-
async read(sessionId) {
|
|
1416
|
-
const session = this.requireOpenSession(sessionId);
|
|
1417
|
-
await this.waitForPage(session.page);
|
|
1418
|
-
this.touch(session);
|
|
1419
|
-
const title = await session.page.title().catch(() => "");
|
|
1420
|
-
const snapshot = await session.page.evaluate(() => {
|
|
1421
|
-
const cssEscape = (value) => {
|
|
1422
|
-
const escapeFn = globalThis.CSS?.escape;
|
|
1423
|
-
return escapeFn ? escapeFn(value) : value.replace(/["\\#.:>+~[\]()]/g, "\\$&");
|
|
1424
|
-
};
|
|
1425
|
-
const textFor = (element) => {
|
|
1426
|
-
const html = element;
|
|
1427
|
-
return (element.getAttribute("aria-label") || element.getAttribute("title") || html.placeholder || html.value || html.innerText || element.textContent || "").replace(/\s+/g, " ").trim();
|
|
1428
|
-
};
|
|
1429
|
-
const selectorFor = (element) => {
|
|
1430
|
-
const id = element.id;
|
|
1431
|
-
if (id) return `#${cssEscape(id)}`;
|
|
1432
|
-
const name = element.getAttribute("name");
|
|
1433
|
-
const tag = element.tagName.toLowerCase();
|
|
1434
|
-
if (name) return `${tag}[name="${name.replace(/"/g, '\\"')}"]`;
|
|
1435
|
-
const parts = [];
|
|
1436
|
-
let current = element;
|
|
1437
|
-
while (current && current !== document.body && parts.length < 4) {
|
|
1438
|
-
const currentTag = current.tagName.toLowerCase();
|
|
1439
|
-
const parent = current.parentElement;
|
|
1440
|
-
if (!parent) {
|
|
1441
|
-
parts.unshift(currentTag);
|
|
1442
|
-
break;
|
|
1443
|
-
}
|
|
1444
|
-
const children = Array.from(parent.children);
|
|
1445
|
-
const siblings = children.filter((child) => child.tagName === current.tagName);
|
|
1446
|
-
const index = siblings.indexOf(current) + 1;
|
|
1447
|
-
parts.unshift(siblings.length > 1 ? `${currentTag}:nth-of-type(${index})` : currentTag);
|
|
1448
|
-
current = parent;
|
|
1449
|
-
}
|
|
1450
|
-
return parts.join(" > ") || tag;
|
|
1451
|
-
};
|
|
1452
|
-
const candidates = Array.from(document.querySelectorAll(
|
|
1453
|
-
'a, button, input, textarea, select, summary, [role], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'
|
|
1454
|
-
));
|
|
1455
|
-
const elements = candidates.map((element, index) => {
|
|
1456
|
-
const rect = element.getBoundingClientRect();
|
|
1457
|
-
const style = window.getComputedStyle(element);
|
|
1458
|
-
const visible = rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && Number(style.opacity || "1") > 0;
|
|
1459
|
-
if (!visible) return null;
|
|
1460
|
-
const label = textFor(element);
|
|
1461
|
-
return {
|
|
1462
|
-
index,
|
|
1463
|
-
tag: element.tagName.toLowerCase(),
|
|
1464
|
-
role: element.getAttribute("role") || null,
|
|
1465
|
-
text: label.slice(0, 180),
|
|
1466
|
-
selector: selectorFor(element),
|
|
1467
|
-
x: Math.round(rect.left + rect.width / 2),
|
|
1468
|
-
y: Math.round(rect.top + rect.height / 2),
|
|
1469
|
-
left: Math.round(rect.left),
|
|
1470
|
-
top: Math.round(rect.top),
|
|
1471
|
-
width: Math.round(rect.width),
|
|
1472
|
-
height: Math.round(rect.height)
|
|
1473
|
-
};
|
|
1474
|
-
}).filter(Boolean).slice(0, 120);
|
|
1475
|
-
return {
|
|
1476
|
-
text: (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").slice(0, 3e4),
|
|
1477
|
-
elements,
|
|
1478
|
-
viewport: {
|
|
1479
|
-
width: window.innerWidth,
|
|
1480
|
-
height: window.innerHeight,
|
|
1481
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
1482
|
-
scrollX: window.scrollX,
|
|
1483
|
-
scrollY: window.scrollY
|
|
1484
|
-
}
|
|
1485
|
-
};
|
|
1486
|
-
});
|
|
1487
|
-
return {
|
|
1488
|
-
url: session.page.url(),
|
|
1489
|
-
title,
|
|
1490
|
-
text: snapshot.text,
|
|
1491
|
-
elements: snapshot.elements,
|
|
1492
|
-
viewport: snapshot.viewport
|
|
1493
|
-
};
|
|
1494
|
-
}
|
|
1495
|
-
async locate(sessionId, targets) {
|
|
1496
|
-
const session = this.requireOpenSession(sessionId);
|
|
1497
|
-
this.touch(session);
|
|
1498
|
-
const located = [];
|
|
1499
|
-
for (const target of targets) {
|
|
1500
|
-
try {
|
|
1501
|
-
const locator = target.selector ? session.page.locator(target.selector) : session.page.getByText(target.text ?? "", { exact: target.match === "exact" });
|
|
1502
|
-
const selected = locator.nth(target.index ?? 0);
|
|
1503
|
-
await selected.waitFor({ state: "visible", timeout: 3e3 }).catch(() => void 0);
|
|
1504
|
-
const box = await selected.boundingBox();
|
|
1505
|
-
const text = await selected.textContent({ timeout: 1e3 }).catch(() => null);
|
|
1506
|
-
located.push({
|
|
1507
|
-
input: target,
|
|
1508
|
-
found: Boolean(box),
|
|
1509
|
-
element: box ? {
|
|
1510
|
-
left: Math.round(box.x),
|
|
1511
|
-
top: Math.round(box.y),
|
|
1512
|
-
width: Math.round(box.width),
|
|
1513
|
-
height: Math.round(box.height),
|
|
1514
|
-
x: Math.round(box.x + box.width / 2),
|
|
1515
|
-
y: Math.round(box.y + box.height / 2),
|
|
1516
|
-
text: (text ?? "").replace(/\s+/g, " ").trim().slice(0, 220)
|
|
1517
|
-
} : null
|
|
1518
|
-
});
|
|
1519
|
-
} catch (err) {
|
|
1520
|
-
located.push({
|
|
1521
|
-
input: target,
|
|
1522
|
-
found: false,
|
|
1523
|
-
element: null,
|
|
1524
|
-
error: err instanceof Error ? err.message : String(err)
|
|
1525
|
-
});
|
|
1526
|
-
}
|
|
1527
|
-
}
|
|
1528
|
-
const read = await this.read(sessionId);
|
|
1529
|
-
return {
|
|
1530
|
-
url: read.url,
|
|
1531
|
-
title: read.title,
|
|
1532
|
-
viewport: read.viewport,
|
|
1533
|
-
replay: null,
|
|
1534
|
-
targets: located
|
|
1535
|
-
};
|
|
1536
|
-
}
|
|
1537
|
-
async goto(sessionId, url) {
|
|
1538
|
-
const session = this.requireOpenSession(sessionId);
|
|
1539
|
-
this.touch(session);
|
|
1540
|
-
const response = await session.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 }).catch((err) => {
|
|
1541
|
-
throw new Error(err instanceof Error ? err.message : String(err));
|
|
1542
|
-
});
|
|
1543
|
-
await this.waitForPage(session.page);
|
|
1544
|
-
return {
|
|
1545
|
-
ok: true,
|
|
1546
|
-
url: session.page.url(),
|
|
1547
|
-
status: response?.status() ?? null,
|
|
1548
|
-
mode: "local"
|
|
1549
|
-
};
|
|
1550
|
-
}
|
|
1551
|
-
async click(sessionId, input) {
|
|
1552
|
-
const session = this.requireOpenSession(sessionId);
|
|
1553
|
-
this.touch(session);
|
|
1554
|
-
await session.page.mouse.click(input.x, input.y, {
|
|
1555
|
-
button: input.button ?? "left",
|
|
1556
|
-
clickCount: input.num_clicks ?? 1
|
|
1557
|
-
});
|
|
1558
|
-
await this.waitForPage(session.page);
|
|
1559
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1560
|
-
}
|
|
1561
|
-
async type(sessionId, input) {
|
|
1562
|
-
const session = this.requireOpenSession(sessionId);
|
|
1563
|
-
this.touch(session);
|
|
1564
|
-
await session.page.keyboard.type(input.text, { delay: input.delay });
|
|
1565
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1566
|
-
}
|
|
1567
|
-
async scroll(sessionId, input) {
|
|
1568
|
-
const session = this.requireOpenSession(sessionId);
|
|
1569
|
-
this.touch(session);
|
|
1570
|
-
if (typeof input.x === "number" && typeof input.y === "number") {
|
|
1571
|
-
await session.page.mouse.move(input.x, input.y);
|
|
1572
|
-
}
|
|
1573
|
-
await session.page.mouse.wheel((input.delta_x ?? 0) * 120, (input.delta_y ?? 5) * 120);
|
|
1574
|
-
await session.page.waitForTimeout(150).catch(() => void 0);
|
|
1575
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1576
|
-
}
|
|
1577
|
-
async press(sessionId, input) {
|
|
1578
|
-
const session = this.requireOpenSession(sessionId);
|
|
1579
|
-
this.touch(session);
|
|
1580
|
-
for (const key of input.keys) {
|
|
1581
|
-
await session.page.keyboard.press(normalizeKeyCombo(key));
|
|
1582
|
-
}
|
|
1583
|
-
await this.waitForPage(session.page);
|
|
1584
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1585
|
-
}
|
|
1586
|
-
async close(sessionId) {
|
|
1587
|
-
const session = this.sessions.get(sessionId);
|
|
1588
|
-
if (!session || session.status === "closed") {
|
|
1589
|
-
return { ok: true, closed: true, already_closed: true, mode: "local" };
|
|
1590
|
-
}
|
|
1591
|
-
if (session.timeout) clearTimeout(session.timeout);
|
|
1592
|
-
session.status = "closed";
|
|
1593
|
-
session.closedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1594
|
-
await session.page.close().catch(() => void 0);
|
|
1595
|
-
await this.closeContextIfUnused(session.contextKey);
|
|
1596
|
-
return { ok: true, closed: true, mode: "local" };
|
|
1597
|
-
}
|
|
1598
|
-
list(includeClosed = false) {
|
|
1599
|
-
return Array.from(this.sessions.values()).filter((session) => includeClosed || session.status === "open").map(sessionSummary);
|
|
1600
|
-
}
|
|
1601
|
-
async captureFanout(sessionId, input) {
|
|
1602
|
-
const session = this.requireOpenSession(sessionId);
|
|
1603
|
-
this.touch(session);
|
|
1604
|
-
return runFanoutCapture(session.page, input);
|
|
1605
|
-
}
|
|
1606
|
-
async screenshotOnly(sessionId) {
|
|
1607
|
-
const session = this.requireOpenSession(sessionId);
|
|
1608
|
-
await this.waitForPage(session.page);
|
|
1609
|
-
const img = await session.page.screenshot({ type: "png", fullPage: false });
|
|
1610
|
-
return img.toString("base64");
|
|
1611
|
-
}
|
|
1612
|
-
async resolveProfile(profile) {
|
|
1613
|
-
const directDir = directProfileDir();
|
|
1614
|
-
const requestedName = configuredProfileName(profile);
|
|
1615
|
-
if (directDir && (!profile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
|
|
1616
|
-
return {
|
|
1617
|
-
name: requestedName,
|
|
1618
|
-
userDataDir: directDir,
|
|
1619
|
-
browserExecutablePath: process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || defaultChromeExecutablePath() || null,
|
|
1620
|
-
manifest: null
|
|
1621
|
-
};
|
|
1622
|
-
}
|
|
1623
|
-
const manifest = await loadImportedBrowserProfile(requestedName);
|
|
1624
|
-
return {
|
|
1625
|
-
name: manifest.name,
|
|
1626
|
-
userDataDir: manifest.userDataDir,
|
|
1627
|
-
browserExecutablePath: manifest.browserExecutablePath,
|
|
1628
|
-
manifest
|
|
1629
|
-
};
|
|
1630
|
-
}
|
|
1631
|
-
async ensureContext(profile) {
|
|
1632
|
-
const existing = this.contexts.get(profile.userDataDir);
|
|
1633
|
-
if (existing) return existing;
|
|
1634
|
-
const executablePath = launchExecutablePath(profile);
|
|
1635
|
-
const context = await import_playwright_extra.chromium.launchPersistentContext(profile.userDataDir, {
|
|
1636
|
-
headless: false,
|
|
1637
|
-
viewport: { width: 1440, height: 900 },
|
|
1638
|
-
acceptDownloads: true,
|
|
1639
|
-
...executablePath ? { executablePath } : {},
|
|
1640
|
-
...ignoreDefaultArgsFor(executablePath) ? { ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath) } : {}
|
|
1641
|
-
});
|
|
1642
|
-
const state = { key: profile.userDataDir, context, profile };
|
|
1643
|
-
context.on("close", () => {
|
|
1644
|
-
this.contexts.delete(profile.userDataDir);
|
|
1645
|
-
const closedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1646
|
-
for (const session of this.sessions.values()) {
|
|
1647
|
-
if (session.contextKey === profile.userDataDir && session.status === "open") {
|
|
1648
|
-
session.status = "closed";
|
|
1649
|
-
session.closedAt = closedAt;
|
|
1650
|
-
if (session.timeout) clearTimeout(session.timeout);
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
});
|
|
1654
|
-
this.contexts.set(profile.userDataDir, state);
|
|
1655
|
-
return state;
|
|
1656
|
-
}
|
|
1657
|
-
requireOpenSession(sessionId) {
|
|
1658
|
-
const session = this.sessions.get(sessionId);
|
|
1659
|
-
if (!session || session.status !== "open") throw new Error(`Local browser session not found or closed: ${sessionId}`);
|
|
1660
|
-
return session;
|
|
1661
|
-
}
|
|
1662
|
-
touch(session) {
|
|
1663
|
-
session.lastActionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1664
|
-
}
|
|
1665
|
-
hasOpenSessionForContext(contextKey) {
|
|
1666
|
-
return Array.from(this.sessions.values()).some((session) => session.contextKey === contextKey && session.status === "open");
|
|
1667
|
-
}
|
|
1668
|
-
async closeContextIfUnused(contextKey) {
|
|
1669
|
-
if (this.hasOpenSessionForContext(contextKey)) return;
|
|
1670
|
-
const state = this.contexts.get(contextKey);
|
|
1671
|
-
if (!state) return;
|
|
1672
|
-
this.contexts.delete(contextKey);
|
|
1673
|
-
await state.context.close().catch(() => void 0);
|
|
1674
|
-
}
|
|
1675
|
-
async waitForPage(page) {
|
|
1676
|
-
await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
|
|
1677
|
-
await page.waitForTimeout(200).catch(() => void 0);
|
|
1678
|
-
}
|
|
1679
|
-
};
|
|
1680
|
-
|
|
1681
272
|
// src/version.ts
|
|
1682
|
-
var PACKAGE_VERSION = "0.3.
|
|
273
|
+
var PACKAGE_VERSION = "0.3.8";
|
|
1683
274
|
|
|
1684
275
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
1685
276
|
var import_zod = require("zod");
|
|
@@ -1699,35 +290,16 @@ var BrowserOpenInputSchema = {
|
|
|
1699
290
|
label: import_zod.z.string().optional().describe("Optional human label for this session, shown in the watch console."),
|
|
1700
291
|
url: import_zod.z.string().url().optional().describe("Optional URL to navigate to immediately after opening."),
|
|
1701
292
|
profile: import_zod.z.string().optional().describe("Optional saved hosted profile name to load a logged-in session for a site."),
|
|
1702
|
-
save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and
|
|
293
|
+
save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and browser storage back to the named profile when the session is closed. Use this for profile setup or intentional auth refreshes; avoid parallel sessions writing to the same profile."),
|
|
1703
294
|
timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy browser session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
|
|
1704
295
|
};
|
|
1705
|
-
var BrowserProfileListInputSchema = {
|
|
1706
|
-
email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to filter for, for example seo@example.com."),
|
|
1707
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory. Defaults to the current OS Chrome profile root.")
|
|
1708
|
-
};
|
|
1709
296
|
var BrowserProfileOnboardInputSchema = {
|
|
1710
297
|
email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
|
|
1711
298
|
profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
|
|
1712
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional Chrome metadata directory for deriving a profile name only. This is not a local browser setup path."),
|
|
1713
299
|
url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
|
|
1714
300
|
label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
|
|
1715
301
|
timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
|
|
1716
302
|
};
|
|
1717
|
-
var BrowserProfileImportInputSchema = {
|
|
1718
|
-
email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to select from Chrome Local State."),
|
|
1719
|
-
name: import_zod.z.string().optional().describe("Name for the managed MCP Scraper browser profile. Defaults to a slug derived from the selected Chrome account."),
|
|
1720
|
-
chrome_profile_directory: import_zod.z.string().optional().describe("Optional Chrome profile directory to import, for example Default or Profile 1."),
|
|
1721
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional source Chrome user data directory. Defaults to the current OS Chrome profile root."),
|
|
1722
|
-
output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
|
|
1723
|
-
browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path for local browser launches. Defaults to the installed Google Chrome path when found."),
|
|
1724
|
-
overwrite: import_zod.z.boolean().default(false).describe("Overwrite an existing managed profile clone with the same name.")
|
|
1725
|
-
};
|
|
1726
|
-
var BrowserProfileSyncInputSchema = {
|
|
1727
|
-
name: import_zod.z.string().describe("Managed MCP Scraper browser profile name to refresh from its recorded source Chrome profile."),
|
|
1728
|
-
output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
|
|
1729
|
-
browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path to store in the refreshed profile manifest.")
|
|
1730
|
-
};
|
|
1731
303
|
var BrowserSessionInputSchema = {
|
|
1732
304
|
session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. Use only a returned session_id; do not construct one yourself.")
|
|
1733
305
|
};
|
|
@@ -1896,14 +468,6 @@ var BrowserOpenOutputSchema = {
|
|
|
1896
468
|
hint: import_zod.z.string(),
|
|
1897
469
|
raw: BrowserRawObject.optional()
|
|
1898
470
|
};
|
|
1899
|
-
var BrowserProfileListOutputSchema = {
|
|
1900
|
-
ok: import_zod.z.boolean(),
|
|
1901
|
-
tool: import_zod.z.literal("browser_profile_list"),
|
|
1902
|
-
session_id: import_zod.z.null(),
|
|
1903
|
-
profiles: import_zod.z.array(BrowserRawObject),
|
|
1904
|
-
count: import_zod.z.number().int().min(0),
|
|
1905
|
-
hint: import_zod.z.string()
|
|
1906
|
-
};
|
|
1907
471
|
var BrowserProfileOnboardOutputSchema = {
|
|
1908
472
|
ok: import_zod.z.boolean(),
|
|
1909
473
|
tool: import_zod.z.literal("browser_profile_onboard"),
|
|
@@ -1912,27 +476,10 @@ var BrowserProfileOnboardOutputSchema = {
|
|
|
1912
476
|
live_view_url: NullableString,
|
|
1913
477
|
profile: import_zod.z.string(),
|
|
1914
478
|
setup_url: import_zod.z.string(),
|
|
1915
|
-
|
|
479
|
+
account_email: NullableString,
|
|
1916
480
|
next_steps: import_zod.z.array(import_zod.z.string()),
|
|
1917
481
|
raw: BrowserRawObject.optional()
|
|
1918
482
|
};
|
|
1919
|
-
var BrowserProfileImportOutputSchema = {
|
|
1920
|
-
ok: import_zod.z.boolean(),
|
|
1921
|
-
tool: import_zod.z.literal("browser_profile_import"),
|
|
1922
|
-
session_id: import_zod.z.null(),
|
|
1923
|
-
profile: BrowserRawObject,
|
|
1924
|
-
source_profile: BrowserRawObject,
|
|
1925
|
-
next_steps: import_zod.z.array(import_zod.z.string()),
|
|
1926
|
-
hint: import_zod.z.string()
|
|
1927
|
-
};
|
|
1928
|
-
var BrowserProfileSyncOutputSchema = {
|
|
1929
|
-
ok: import_zod.z.boolean(),
|
|
1930
|
-
tool: import_zod.z.literal("browser_profile_sync"),
|
|
1931
|
-
session_id: import_zod.z.null(),
|
|
1932
|
-
profile: BrowserRawObject,
|
|
1933
|
-
source_profile: BrowserRawObject,
|
|
1934
|
-
hint: import_zod.z.string()
|
|
1935
|
-
};
|
|
1936
483
|
var BrowserScreenshotOutputSchema = {
|
|
1937
484
|
...BrowserBaseOutput,
|
|
1938
485
|
tool: import_zod.z.literal("browser_screenshot"),
|
|
@@ -2033,9 +580,9 @@ var BrowserListSessionsOutputSchema = {
|
|
|
2033
580
|
|
|
2034
581
|
// src/mcp/replay-annotator.ts
|
|
2035
582
|
var import_node_child_process = require("child_process");
|
|
2036
|
-
var
|
|
2037
|
-
var
|
|
2038
|
-
var
|
|
583
|
+
var import_promises = require("fs/promises");
|
|
584
|
+
var import_node_os = require("os");
|
|
585
|
+
var import_node_path = require("path");
|
|
2039
586
|
var import_node_util = require("util");
|
|
2040
587
|
var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
|
|
2041
588
|
function finiteNumber(value) {
|
|
@@ -2045,11 +592,11 @@ function clamp(value, min, max) {
|
|
|
2045
592
|
return Math.min(max, Math.max(min, value));
|
|
2046
593
|
}
|
|
2047
594
|
function formatAssTime(seconds) {
|
|
2048
|
-
const
|
|
2049
|
-
const hours = Math.floor(
|
|
2050
|
-
const minutes = Math.floor(
|
|
2051
|
-
const wholeSeconds = Math.floor(
|
|
2052
|
-
const centiseconds = Math.floor((
|
|
595
|
+
const safe = Math.max(0, seconds);
|
|
596
|
+
const hours = Math.floor(safe / 3600);
|
|
597
|
+
const minutes = Math.floor(safe % 3600 / 60);
|
|
598
|
+
const wholeSeconds = Math.floor(safe % 60);
|
|
599
|
+
const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
|
|
2053
600
|
return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
|
|
2054
601
|
}
|
|
2055
602
|
function cssHexToAssColor(color) {
|
|
@@ -2258,10 +805,10 @@ function ffmpegFilterPath(path) {
|
|
|
2258
805
|
async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
2259
806
|
if (!options.annotations.length) throw new Error("annotations must include at least one item");
|
|
2260
807
|
const size = await videoSize(inputFilePath);
|
|
2261
|
-
const tmp = await (0,
|
|
2262
|
-
const assPath = (0,
|
|
808
|
+
const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
|
|
809
|
+
const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
|
|
2263
810
|
try {
|
|
2264
|
-
await (0,
|
|
811
|
+
await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
|
|
2265
812
|
await execFileAsync("ffmpeg", [
|
|
2266
813
|
"-y",
|
|
2267
814
|
"-i",
|
|
@@ -2278,7 +825,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
2278
825
|
"copy",
|
|
2279
826
|
outputFilePath
|
|
2280
827
|
], { maxBuffer: 1024 * 1024 * 20 });
|
|
2281
|
-
const out = await (0,
|
|
828
|
+
const out = await (0, import_promises.stat)(outputFilePath);
|
|
2282
829
|
return {
|
|
2283
830
|
filePath: outputFilePath,
|
|
2284
831
|
bytes: out.size,
|
|
@@ -2287,7 +834,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
2287
834
|
annotationCount: options.annotations.length
|
|
2288
835
|
};
|
|
2289
836
|
} finally {
|
|
2290
|
-
await (0,
|
|
837
|
+
await (0, import_promises.rm)(tmp, { recursive: true, force: true });
|
|
2291
838
|
}
|
|
2292
839
|
}
|
|
2293
840
|
|
|
@@ -2327,8 +874,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
|
|
|
2327
874
|
nextRecommendedTool
|
|
2328
875
|
});
|
|
2329
876
|
}
|
|
2330
|
-
function
|
|
2331
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
877
|
+
function outputBaseDir() {
|
|
878
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
|
|
2332
879
|
}
|
|
2333
880
|
function safeFilePart(value) {
|
|
2334
881
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
|
|
@@ -2337,7 +884,12 @@ function replayFilePath(sessionId, replayId, filename) {
|
|
|
2337
884
|
const requested = filename?.trim();
|
|
2338
885
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2339
886
|
const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
|
|
2340
|
-
return (0,
|
|
887
|
+
return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
|
|
888
|
+
}
|
|
889
|
+
function savedProfileNameFromEmail(email) {
|
|
890
|
+
const value = email?.trim().toLowerCase();
|
|
891
|
+
if (!value) return null;
|
|
892
|
+
return value.replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || null;
|
|
2341
893
|
}
|
|
2342
894
|
function annotatedReplayFilePath(sessionId, replayId, filename) {
|
|
2343
895
|
const requested = filename?.trim();
|
|
@@ -2368,7 +920,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2368
920
|
const baseUrl2 = opts.baseUrl.replace(/\/$/, "");
|
|
2369
921
|
const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
|
|
2370
922
|
const timeoutMs = opts.timeoutMs ?? 9e4;
|
|
2371
|
-
const localBrowser = new LocalBrowserAgentService();
|
|
2372
923
|
async function req(method, path, body) {
|
|
2373
924
|
try {
|
|
2374
925
|
const res = await fetch(`${baseUrl2}${path}`, {
|
|
@@ -2397,8 +948,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2397
948
|
}
|
|
2398
949
|
const bytes = Buffer.from(await res.arrayBuffer());
|
|
2399
950
|
const filePath = replayFilePath(sessionId, replayId, filename);
|
|
2400
|
-
(0,
|
|
2401
|
-
(0,
|
|
951
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
|
|
952
|
+
(0, import_node_fs.writeFileSync)(filePath, bytes);
|
|
2402
953
|
return {
|
|
2403
954
|
ok: true,
|
|
2404
955
|
data: {
|
|
@@ -2420,131 +971,20 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2420
971
|
idempotentHint: false,
|
|
2421
972
|
openWorldHint: true
|
|
2422
973
|
});
|
|
2423
|
-
function selectChromeProfile(matches, email) {
|
|
2424
|
-
const wanted = email?.trim().toLowerCase();
|
|
2425
|
-
if (!wanted) return matches[0] ?? null;
|
|
2426
|
-
return matches.find((profile) => profile.email.toLowerCase() === wanted) ?? matches[0] ?? null;
|
|
2427
|
-
}
|
|
2428
|
-
function localReplayUnsupported(tool, sessionId = null, replayId = null) {
|
|
2429
|
-
return errorResult(tool, {
|
|
2430
|
-
error: "Local browser mode does not support replay recording yet. Use hosted mode for browser_replay_* tools.",
|
|
2431
|
-
mode: "local"
|
|
2432
|
-
}, sessionId, replayId);
|
|
2433
|
-
}
|
|
2434
|
-
server2.registerTool(
|
|
2435
|
-
"browser_profile_list",
|
|
2436
|
-
{
|
|
2437
|
-
title: "Developer: List Local Chrome Profiles",
|
|
2438
|
-
description: "Developer/local-machine helper only. List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2439
|
-
inputSchema: BrowserProfileListInputSchema,
|
|
2440
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
2441
|
-
annotations: annotations("Developer: List Local Chrome Profiles", true)
|
|
2442
|
-
},
|
|
2443
|
-
async (input) => {
|
|
2444
|
-
try {
|
|
2445
|
-
const profiles = await listLocalChromeProfiles(input.user_data_dir, input.email);
|
|
2446
|
-
return structuredResult({
|
|
2447
|
-
ok: true,
|
|
2448
|
-
tool: "browser_profile_list",
|
|
2449
|
-
session_id: null,
|
|
2450
|
-
profiles,
|
|
2451
|
-
count: profiles.length,
|
|
2452
|
-
hint: "For customer auth setup, use browser_profile_onboard so the user signs in through the hosted watch_url. Local import is only for developer/local-machine mode."
|
|
2453
|
-
});
|
|
2454
|
-
} catch (err) {
|
|
2455
|
-
return errorResult("browser_profile_list", { error: err instanceof Error ? err.message : String(err) });
|
|
2456
|
-
}
|
|
2457
|
-
}
|
|
2458
|
-
);
|
|
2459
|
-
server2.registerTool(
|
|
2460
|
-
"browser_profile_import",
|
|
2461
|
-
{
|
|
2462
|
-
title: "Developer: Import Local Chrome Profile",
|
|
2463
|
-
description: "Developer/local-machine helper only. Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2464
|
-
inputSchema: BrowserProfileImportInputSchema,
|
|
2465
|
-
outputSchema: BrowserProfileImportOutputSchema,
|
|
2466
|
-
annotations: { title: "Developer: Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
|
|
2467
|
-
},
|
|
2468
|
-
async (input) => {
|
|
2469
|
-
try {
|
|
2470
|
-
const result = await importChromeProfile({
|
|
2471
|
-
email: input.email,
|
|
2472
|
-
name: input.name,
|
|
2473
|
-
chromeProfileDirectory: input.chrome_profile_directory,
|
|
2474
|
-
sourceUserDataDir: input.user_data_dir,
|
|
2475
|
-
outputDir: input.output_dir,
|
|
2476
|
-
overwrite: input.overwrite,
|
|
2477
|
-
browserExecutablePath: input.browser_executable_path
|
|
2478
|
-
});
|
|
2479
|
-
return structuredResult({
|
|
2480
|
-
ok: true,
|
|
2481
|
-
tool: "browser_profile_import",
|
|
2482
|
-
session_id: null,
|
|
2483
|
-
profile: result.profile,
|
|
2484
|
-
source_profile: result.sourceProfile,
|
|
2485
|
-
next_steps: [
|
|
2486
|
-
`Set MCP_SCRAPER_BROWSER_MODE=local in the MCP server config.`,
|
|
2487
|
-
`Set MCP_SCRAPER_BROWSER_PROFILE=${result.profile.name}.`,
|
|
2488
|
-
"Restart the MCP client so it starts the local browser mode process.",
|
|
2489
|
-
"Call browser_open with the same MCP server; it will open a local Chrome window using the cloned profile."
|
|
2490
|
-
],
|
|
2491
|
-
hint: "Developer-only local clone. For customer auth setup, use browser_profile_onboard at Google, provide the hosted watch_url, then browser_close to save the hosted profile."
|
|
2492
|
-
});
|
|
2493
|
-
} catch (err) {
|
|
2494
|
-
return errorResult("browser_profile_import", { error: err instanceof Error ? err.message : String(err) });
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
);
|
|
2498
|
-
server2.registerTool(
|
|
2499
|
-
"browser_profile_sync",
|
|
2500
|
-
{
|
|
2501
|
-
title: "Developer: Sync Local Chrome Profile",
|
|
2502
|
-
description: "Developer/local-machine helper only. Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2503
|
-
inputSchema: BrowserProfileSyncInputSchema,
|
|
2504
|
-
outputSchema: BrowserProfileSyncOutputSchema,
|
|
2505
|
-
annotations: { title: "Developer: Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
|
|
2506
|
-
},
|
|
2507
|
-
async (input) => {
|
|
2508
|
-
try {
|
|
2509
|
-
const result = await syncImportedChromeProfile(input.name, {
|
|
2510
|
-
outputDir: input.output_dir,
|
|
2511
|
-
browserExecutablePath: input.browser_executable_path
|
|
2512
|
-
});
|
|
2513
|
-
return structuredResult({
|
|
2514
|
-
ok: true,
|
|
2515
|
-
tool: "browser_profile_sync",
|
|
2516
|
-
session_id: null,
|
|
2517
|
-
profile: result.profile,
|
|
2518
|
-
source_profile: result.sourceProfile,
|
|
2519
|
-
hint: `Developer-only local clone refreshed for ${result.profile.name}. Customer reconnects should use browser_profile_onboard with the same hosted profile name.`
|
|
2520
|
-
});
|
|
2521
|
-
} catch (err) {
|
|
2522
|
-
return errorResult("browser_profile_sync", { error: err instanceof Error ? err.message : String(err) });
|
|
2523
|
-
}
|
|
2524
|
-
}
|
|
2525
|
-
);
|
|
2526
974
|
server2.registerTool(
|
|
2527
975
|
"browser_profile_onboard",
|
|
2528
976
|
{
|
|
2529
977
|
title: "Connect Hosted Browser Profile",
|
|
2530
|
-
description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and
|
|
978
|
+
description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and storage. If a saved profile later shows login controls, run this same setup flow again.",
|
|
2531
979
|
inputSchema: BrowserProfileOnboardInputSchema,
|
|
2532
980
|
outputSchema: BrowserProfileOnboardOutputSchema,
|
|
2533
981
|
annotations: annotations("Connect Hosted Browser Profile")
|
|
2534
982
|
},
|
|
2535
983
|
async (input) => {
|
|
2536
|
-
|
|
2537
|
-
if (input.email) {
|
|
2538
|
-
try {
|
|
2539
|
-
chromeProfile = selectChromeProfile(await listLocalChromeProfiles(input.user_data_dir, input.email), input.email);
|
|
2540
|
-
} catch (err) {
|
|
2541
|
-
return errorResult("browser_profile_onboard", { error: err instanceof Error ? err.message : String(err) });
|
|
2542
|
-
}
|
|
2543
|
-
}
|
|
2544
|
-
const profile = input.profile?.trim() || chromeProfile?.recommendedKernelProfileName || browserServiceProfileName();
|
|
984
|
+
const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email) || browserServiceProfileName();
|
|
2545
985
|
if (!profile) {
|
|
2546
986
|
return errorResult("browser_profile_onboard", {
|
|
2547
|
-
error: "profile is required when
|
|
987
|
+
error: "profile or email is required when BROWSER_AGENT_PROFILE_NAME is not available"
|
|
2548
988
|
});
|
|
2549
989
|
}
|
|
2550
990
|
const setupUrl = input.url ?? "https://google.com/";
|
|
@@ -2567,10 +1007,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2567
1007
|
live_view_url: session.live_view_url ?? null,
|
|
2568
1008
|
profile,
|
|
2569
1009
|
setup_url: setupUrl,
|
|
2570
|
-
|
|
1010
|
+
account_email: input.email?.trim() || null,
|
|
2571
1011
|
next_steps: [
|
|
2572
1012
|
"Open the watch_url and let the user click Sign in on Google.",
|
|
2573
|
-
"After the user confirms login is complete, call browser_close with this session_id to persist cookies and
|
|
1013
|
+
"After the user confirms login is complete, call browser_close with this session_id to persist cookies and storage to the saved profile.",
|
|
2574
1014
|
"If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
|
|
2575
1015
|
"Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
|
|
2576
1016
|
],
|
|
@@ -2588,23 +1028,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2588
1028
|
annotations: annotations("Open Browser Session")
|
|
2589
1029
|
},
|
|
2590
1030
|
async (input) => {
|
|
2591
|
-
if (localBrowserModeEnabled()) {
|
|
2592
|
-
try {
|
|
2593
|
-
const session2 = await localBrowser.open(input);
|
|
2594
|
-
return structuredResult({
|
|
2595
|
-
ok: true,
|
|
2596
|
-
tool: "browser_open",
|
|
2597
|
-
session_id: session2.session_id,
|
|
2598
|
-
watch_url: session2.watch_url,
|
|
2599
|
-
live_view_url: null,
|
|
2600
|
-
url: input.url ?? null,
|
|
2601
|
-
hint: "Local Chrome is open on this machine. Call browser_screenshot to see the page and click by x,y coordinates.",
|
|
2602
|
-
raw: session2
|
|
2603
|
-
});
|
|
2604
|
-
} catch (err) {
|
|
2605
|
-
return errorResult("browser_open", { error: err instanceof Error ? err.message : String(err), mode: "local" });
|
|
2606
|
-
}
|
|
2607
|
-
}
|
|
2608
1031
|
const profile = input.profile ?? browserServiceProfileName();
|
|
2609
1032
|
const saveProfileChanges = input.save_profile_changes ?? browserServiceProfileSaveChanges();
|
|
2610
1033
|
const open = await req("POST", "/agent/sessions", {
|
|
@@ -2641,27 +1064,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2641
1064
|
annotations: annotations("See Page", true)
|
|
2642
1065
|
},
|
|
2643
1066
|
async (input) => {
|
|
2644
|
-
if (localBrowserModeEnabled()) {
|
|
2645
|
-
try {
|
|
2646
|
-
const res2 = await localBrowser.screenshot(input.session_id);
|
|
2647
|
-
const content2 = [];
|
|
2648
|
-
if (res2.image_base64) content2.push({ type: "image", data: res2.image_base64, mimeType: res2.mime_type });
|
|
2649
|
-
const structured2 = {
|
|
2650
|
-
ok: true,
|
|
2651
|
-
tool: "browser_screenshot",
|
|
2652
|
-
session_id: input.session_id,
|
|
2653
|
-
url: res2.url ?? null,
|
|
2654
|
-
title: res2.title ?? null,
|
|
2655
|
-
text: res2.text,
|
|
2656
|
-
elements: res2.elements,
|
|
2657
|
-
screenshot: { mime_type: res2.mime_type, inline: true }
|
|
2658
|
-
};
|
|
2659
|
-
content2.push({ type: "text", text: JSON.stringify(structured2) });
|
|
2660
|
-
return { content: content2, structuredContent: structured2 };
|
|
2661
|
-
} catch (err) {
|
|
2662
|
-
return errorResult("browser_screenshot", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2663
|
-
}
|
|
2664
|
-
}
|
|
2665
1067
|
const res = await req("POST", `/agent/sessions/${input.session_id}/screenshot`);
|
|
2666
1068
|
if (!res.ok) return errorResult("browser_screenshot", res.data, input.session_id);
|
|
2667
1069
|
const { image_base64, mime_type, url, title, elements, text } = res.data;
|
|
@@ -2694,23 +1096,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2694
1096
|
annotations: annotations("Read Page", true)
|
|
2695
1097
|
},
|
|
2696
1098
|
async (input) => {
|
|
2697
|
-
if (localBrowserModeEnabled()) {
|
|
2698
|
-
try {
|
|
2699
|
-
const res2 = await localBrowser.read(input.session_id);
|
|
2700
|
-
return structuredResult({
|
|
2701
|
-
ok: true,
|
|
2702
|
-
tool: "browser_read",
|
|
2703
|
-
session_id: input.session_id,
|
|
2704
|
-
url: res2.url ?? null,
|
|
2705
|
-
title: res2.title ?? null,
|
|
2706
|
-
text: res2.text,
|
|
2707
|
-
elements: res2.elements,
|
|
2708
|
-
raw: res2
|
|
2709
|
-
});
|
|
2710
|
-
} catch (err) {
|
|
2711
|
-
return errorResult("browser_read", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2712
|
-
}
|
|
2713
|
-
}
|
|
2714
1099
|
const res = await req("POST", `/agent/sessions/${input.session_id}/read`);
|
|
2715
1100
|
if (!res.ok) return errorResult("browser_read", res.data, input.session_id);
|
|
2716
1101
|
return structuredResult({
|
|
@@ -2735,24 +1120,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2735
1120
|
annotations: annotations("Locate DOM Targets", true)
|
|
2736
1121
|
},
|
|
2737
1122
|
async (input) => {
|
|
2738
|
-
if (localBrowserModeEnabled()) {
|
|
2739
|
-
try {
|
|
2740
|
-
const res2 = await localBrowser.locate(input.session_id, input.targets);
|
|
2741
|
-
return structuredResult({
|
|
2742
|
-
ok: true,
|
|
2743
|
-
tool: "browser_locate",
|
|
2744
|
-
session_id: input.session_id,
|
|
2745
|
-
url: res2.url ?? null,
|
|
2746
|
-
title: res2.title ?? null,
|
|
2747
|
-
viewport: res2.viewport ?? null,
|
|
2748
|
-
replay: null,
|
|
2749
|
-
targets: Array.isArray(res2.targets) ? res2.targets : [],
|
|
2750
|
-
raw: res2
|
|
2751
|
-
});
|
|
2752
|
-
} catch (err) {
|
|
2753
|
-
return errorResult("browser_locate", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2754
|
-
}
|
|
2755
|
-
}
|
|
2756
1123
|
const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: input.targets });
|
|
2757
1124
|
if (!res.ok) return errorResult("browser_locate", res.data, input.session_id);
|
|
2758
1125
|
return structuredResult({
|
|
@@ -2778,14 +1145,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2778
1145
|
annotations: annotations("Navigate To URL")
|
|
2779
1146
|
},
|
|
2780
1147
|
async (input) => {
|
|
2781
|
-
if (localBrowserModeEnabled()) {
|
|
2782
|
-
try {
|
|
2783
|
-
const res2 = await localBrowser.goto(input.session_id, input.url);
|
|
2784
|
-
return actionResult("browser_goto", input.session_id, true, res2, "browser_screenshot");
|
|
2785
|
-
} catch (err) {
|
|
2786
|
-
return errorResult("browser_goto", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2787
|
-
}
|
|
2788
|
-
}
|
|
2789
1148
|
const res = await req("POST", `/agent/sessions/${input.session_id}/goto`, { url: input.url });
|
|
2790
1149
|
return actionResult("browser_goto", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2791
1150
|
}
|
|
@@ -2800,14 +1159,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2800
1159
|
annotations: annotations("Click")
|
|
2801
1160
|
},
|
|
2802
1161
|
async (input) => {
|
|
2803
|
-
if (localBrowserModeEnabled()) {
|
|
2804
|
-
try {
|
|
2805
|
-
const res2 = await localBrowser.click(input.session_id, input);
|
|
2806
|
-
return actionResult("browser_click", input.session_id, true, res2, "browser_screenshot");
|
|
2807
|
-
} catch (err) {
|
|
2808
|
-
return errorResult("browser_click", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2809
|
-
}
|
|
2810
|
-
}
|
|
2811
1162
|
const res = await req("POST", `/agent/sessions/${input.session_id}/click`, {
|
|
2812
1163
|
x: input.x,
|
|
2813
1164
|
y: input.y,
|
|
@@ -2827,14 +1178,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2827
1178
|
annotations: annotations("Type Text")
|
|
2828
1179
|
},
|
|
2829
1180
|
async (input) => {
|
|
2830
|
-
if (localBrowserModeEnabled()) {
|
|
2831
|
-
try {
|
|
2832
|
-
const res2 = await localBrowser.type(input.session_id, input);
|
|
2833
|
-
return actionResult("browser_type", input.session_id, true, res2, "browser_screenshot");
|
|
2834
|
-
} catch (err) {
|
|
2835
|
-
return errorResult("browser_type", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2836
|
-
}
|
|
2837
|
-
}
|
|
2838
1181
|
const res = await req("POST", `/agent/sessions/${input.session_id}/type`, { text: input.text, delay: input.delay });
|
|
2839
1182
|
return actionResult("browser_type", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2840
1183
|
}
|
|
@@ -2849,14 +1192,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2849
1192
|
annotations: annotations("Scroll")
|
|
2850
1193
|
},
|
|
2851
1194
|
async (input) => {
|
|
2852
|
-
if (localBrowserModeEnabled()) {
|
|
2853
|
-
try {
|
|
2854
|
-
const res2 = await localBrowser.scroll(input.session_id, input);
|
|
2855
|
-
return actionResult("browser_scroll", input.session_id, true, res2, "browser_screenshot");
|
|
2856
|
-
} catch (err) {
|
|
2857
|
-
return errorResult("browser_scroll", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2858
|
-
}
|
|
2859
|
-
}
|
|
2860
1195
|
const res = await req("POST", `/agent/sessions/${input.session_id}/scroll`, {
|
|
2861
1196
|
delta_y: input.delta_y,
|
|
2862
1197
|
delta_x: input.delta_x,
|
|
@@ -2876,14 +1211,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2876
1211
|
annotations: annotations("Press Keys")
|
|
2877
1212
|
},
|
|
2878
1213
|
async (input) => {
|
|
2879
|
-
if (localBrowserModeEnabled()) {
|
|
2880
|
-
try {
|
|
2881
|
-
const res2 = await localBrowser.press(input.session_id, input);
|
|
2882
|
-
return actionResult("browser_press", input.session_id, true, res2, "browser_screenshot");
|
|
2883
|
-
} catch (err) {
|
|
2884
|
-
return errorResult("browser_press", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2885
|
-
}
|
|
2886
|
-
}
|
|
2887
1214
|
const res = await req("POST", `/agent/sessions/${input.session_id}/press`, { keys: input.keys });
|
|
2888
1215
|
return actionResult("browser_press", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2889
1216
|
}
|
|
@@ -2898,7 +1225,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2898
1225
|
annotations: annotations("Start Recording")
|
|
2899
1226
|
},
|
|
2900
1227
|
async (input) => {
|
|
2901
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_start", input.session_id);
|
|
2902
1228
|
const res = await req("POST", `/agent/sessions/${input.session_id}/replay/start`);
|
|
2903
1229
|
if (!res.ok) return errorResult("browser_replay_start", res.data, input.session_id);
|
|
2904
1230
|
return structuredResult({
|
|
@@ -2916,13 +1242,12 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2916
1242
|
"browser_replay_stop",
|
|
2917
1243
|
{
|
|
2918
1244
|
title: "Stop Recording",
|
|
2919
|
-
description: "Stop a replay recording and expose the final view_url and download_url. Use browser_replay_download to save the MP4
|
|
1245
|
+
description: "Stop a replay recording and expose the final view_url and download_url. Use browser_replay_download to save the MP4 to the configured output directory.",
|
|
2920
1246
|
inputSchema: BrowserReplayStopInputSchema,
|
|
2921
1247
|
outputSchema: BrowserReplayStopOutputSchema,
|
|
2922
1248
|
annotations: annotations("Stop Recording")
|
|
2923
1249
|
},
|
|
2924
1250
|
async (input) => {
|
|
2925
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_stop", input.session_id, input.replay_id);
|
|
2926
1251
|
const res = await req("POST", `/agent/sessions/${input.session_id}/replay/stop`, { replay_id: input.replay_id });
|
|
2927
1252
|
if (!res.ok) return errorResult("browser_replay_stop", res.data, input.session_id, input.replay_id);
|
|
2928
1253
|
return structuredResult({
|
|
@@ -2946,7 +1271,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2946
1271
|
annotations: annotations("List Replay Videos", true)
|
|
2947
1272
|
},
|
|
2948
1273
|
async (input) => {
|
|
2949
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_list_replays", input.session_id);
|
|
2950
1274
|
const res = await req("GET", `/agent/sessions/${input.session_id}/replays`);
|
|
2951
1275
|
if (!res.ok) return errorResult("browser_list_replays", res.data, input.session_id);
|
|
2952
1276
|
const replays = Array.isArray(res.data?.replays) ? res.data.replays : [];
|
|
@@ -2963,13 +1287,12 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2963
1287
|
"browser_replay_download",
|
|
2964
1288
|
{
|
|
2965
1289
|
title: "Download Replay MP4",
|
|
2966
|
-
description: "Download a replay recording through MCP Scraper and save the MP4
|
|
1290
|
+
description: "Download a replay recording through MCP Scraper and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
2967
1291
|
inputSchema: BrowserReplayDownloadInputSchema,
|
|
2968
1292
|
outputSchema: BrowserReplayDownloadOutputSchema,
|
|
2969
1293
|
annotations: annotations("Download Replay MP4")
|
|
2970
1294
|
},
|
|
2971
1295
|
async (input) => {
|
|
2972
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_download", input.session_id, input.replay_id);
|
|
2973
1296
|
const res = await downloadReplay(input.session_id, input.replay_id, input.filename);
|
|
2974
1297
|
if (!res.ok) return errorResult("browser_replay_download", res.data, input.session_id, input.replay_id);
|
|
2975
1298
|
return structuredResult({
|
|
@@ -2994,7 +1317,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2994
1317
|
annotations: annotations("Mark Replay Annotation", true)
|
|
2995
1318
|
},
|
|
2996
1319
|
async (input) => {
|
|
2997
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_mark", input.session_id);
|
|
2998
1320
|
const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: [input.target] });
|
|
2999
1321
|
if (!res.ok) return errorResult("browser_replay_mark", res.data, input.session_id);
|
|
3000
1322
|
const target = res.data?.targets?.[0];
|
|
@@ -3039,20 +1361,19 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
3039
1361
|
"browser_replay_annotate",
|
|
3040
1362
|
{
|
|
3041
1363
|
title: "Annotate Replay MP4",
|
|
3042
|
-
description: "Download a browser replay MP4, render visual annotations over it, and save a new annotated MP4
|
|
1364
|
+
description: "Download a browser replay MP4, render visual annotations over it, and save a new annotated MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use this after browser_replay_stop when the user wants proof videos with circles, boxes, arrows, underlines, or labels. For accurate timing and placement, prefer annotations returned by browser_replay_mark while the replay is recording; otherwise use exact left/top/width/height bounds from browser_locate. If the replay video size differs from the screenshot coordinate space, pass source_width and source_height.",
|
|
3043
1365
|
inputSchema: BrowserReplayAnnotateInputSchema,
|
|
3044
1366
|
outputSchema: BrowserReplayAnnotateOutputSchema,
|
|
3045
1367
|
annotations: annotations("Annotate Replay MP4")
|
|
3046
1368
|
},
|
|
3047
1369
|
async (input) => {
|
|
3048
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_annotate", input.session_id, input.replay_id);
|
|
3049
1370
|
const sourceName = input.filename ? `${input.filename}-source` : void 0;
|
|
3050
1371
|
const downloaded = await downloadReplay(input.session_id, input.replay_id, sourceName);
|
|
3051
1372
|
if (!downloaded.ok) return errorResult("browser_replay_annotate", downloaded.data, input.session_id, input.replay_id);
|
|
3052
1373
|
try {
|
|
3053
1374
|
const sourcePath = String(downloaded.data.file_path);
|
|
3054
1375
|
const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
|
|
3055
|
-
(0,
|
|
1376
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
|
|
3056
1377
|
const result = await annotateReplayVideo(sourcePath, outputPath, {
|
|
3057
1378
|
annotations: input.annotations,
|
|
3058
1379
|
sourceWidth: input.source_width,
|
|
@@ -3088,20 +1409,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
3088
1409
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
3089
1410
|
},
|
|
3090
1411
|
async (input) => {
|
|
3091
|
-
if (localBrowserModeEnabled()) {
|
|
3092
|
-
try {
|
|
3093
|
-
const res2 = await localBrowser.close(input.session_id);
|
|
3094
|
-
return structuredResult({
|
|
3095
|
-
ok: true,
|
|
3096
|
-
tool: "browser_close",
|
|
3097
|
-
session_id: input.session_id,
|
|
3098
|
-
closed: true,
|
|
3099
|
-
raw: res2
|
|
3100
|
-
});
|
|
3101
|
-
} catch (err) {
|
|
3102
|
-
return errorResult("browser_close", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
3103
|
-
}
|
|
3104
|
-
}
|
|
3105
1412
|
const res = await req("DELETE", `/agent/sessions/${input.session_id}`);
|
|
3106
1413
|
if (!res.ok) return errorResult("browser_close", res.data, input.session_id);
|
|
3107
1414
|
return structuredResult({
|
|
@@ -3123,16 +1430,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
3123
1430
|
annotations: annotations("List Browser Sessions", true)
|
|
3124
1431
|
},
|
|
3125
1432
|
async (input) => {
|
|
3126
|
-
if (localBrowserModeEnabled()) {
|
|
3127
|
-
const sessions2 = localBrowser.list(input.include_closed);
|
|
3128
|
-
return structuredResult({
|
|
3129
|
-
ok: true,
|
|
3130
|
-
tool: "browser_list_sessions",
|
|
3131
|
-
session_id: null,
|
|
3132
|
-
sessions: sessions2,
|
|
3133
|
-
count: sessions2.length
|
|
3134
|
-
});
|
|
3135
|
-
}
|
|
3136
1433
|
const res = await req("GET", `/agent/sessions${input.include_closed ? "?all=1" : ""}`);
|
|
3137
1434
|
if (!res.ok) return errorResult("browser_list_sessions", res.data);
|
|
3138
1435
|
const sessions = (res.data.sessions ?? []).map((s) => ({ ...s, watch_url: `${consoleBase}/console/${s.session_id}` }));
|
|
@@ -3174,14 +1471,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
3174
1471
|
exports: exports2 ?? null,
|
|
3175
1472
|
...result.debug ? { debug: result.debug } : {}
|
|
3176
1473
|
});
|
|
3177
|
-
if (localBrowserModeEnabled()) {
|
|
3178
|
-
try {
|
|
3179
|
-
const { result, exports: exports2 } = await localBrowser.captureFanout(input.session_id, input);
|
|
3180
|
-
return emit(result, exports2);
|
|
3181
|
-
} catch (err) {
|
|
3182
|
-
return errorResult("browser_capture_fanout", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
3183
|
-
}
|
|
3184
|
-
}
|
|
3185
1474
|
const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
|
|
3186
1475
|
prompt: input.prompt,
|
|
3187
1476
|
wait_ms: input.wait_ms,
|
|
@@ -3197,13 +1486,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
3197
1486
|
|
|
3198
1487
|
// src/mcp/paa-mcp-server.ts
|
|
3199
1488
|
var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
3200
|
-
var
|
|
3201
|
-
var
|
|
1489
|
+
var import_node_fs3 = require("fs");
|
|
1490
|
+
var import_node_path4 = require("path");
|
|
3202
1491
|
|
|
3203
1492
|
// src/mcp/mcp-response-formatter.ts
|
|
3204
|
-
var
|
|
3205
|
-
var
|
|
3206
|
-
var
|
|
1493
|
+
var import_node_fs2 = require("fs");
|
|
1494
|
+
var import_node_os3 = require("os");
|
|
1495
|
+
var import_node_path3 = require("path");
|
|
3207
1496
|
|
|
3208
1497
|
// src/errors.ts
|
|
3209
1498
|
function sanitizeVendorName(message) {
|
|
@@ -3339,17 +1628,17 @@ function reportTitle(full) {
|
|
|
3339
1628
|
const title = full.split("\n").find((line) => line.startsWith("# "));
|
|
3340
1629
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
3341
1630
|
}
|
|
3342
|
-
function
|
|
3343
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
1631
|
+
function outputBaseDir2() {
|
|
1632
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
|
|
3344
1633
|
}
|
|
3345
1634
|
function saveFullReport(full) {
|
|
3346
1635
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
3347
|
-
const outDir =
|
|
1636
|
+
const outDir = outputBaseDir2();
|
|
3348
1637
|
try {
|
|
3349
|
-
(0,
|
|
1638
|
+
(0, import_node_fs2.mkdirSync)(outDir, { recursive: true });
|
|
3350
1639
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
3351
|
-
const file = (0,
|
|
3352
|
-
(0,
|
|
1640
|
+
const file = (0, import_node_path3.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
1641
|
+
(0, import_node_fs2.writeFileSync)(file, full, "utf8");
|
|
3353
1642
|
return file;
|
|
3354
1643
|
} catch {
|
|
3355
1644
|
return null;
|
|
@@ -3358,12 +1647,12 @@ function saveFullReport(full) {
|
|
|
3358
1647
|
function persistScreenshotLocally(base64, url) {
|
|
3359
1648
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
3360
1649
|
try {
|
|
3361
|
-
const dir = (0,
|
|
3362
|
-
(0,
|
|
1650
|
+
const dir = (0, import_node_path3.join)(outputBaseDir2(), "screenshots");
|
|
1651
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
|
3363
1652
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
3364
1653
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
3365
|
-
const filePath = (0,
|
|
3366
|
-
(0,
|
|
1654
|
+
const filePath = (0, import_node_path3.join)(dir, `${stamp}-${slug}.png`);
|
|
1655
|
+
(0, import_node_fs2.writeFileSync)(filePath, Buffer.from(base64, "base64"));
|
|
3367
1656
|
return filePath;
|
|
3368
1657
|
} catch {
|
|
3369
1658
|
return null;
|
|
@@ -4656,16 +2945,13 @@ ${chunkRows}` : "",
|
|
|
4656
2945
|
}
|
|
4657
2946
|
function structuredInstagramBrowser(raw) {
|
|
4658
2947
|
const browser = raw && typeof raw === "object" ? raw : {};
|
|
4659
|
-
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
4660
|
-
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
4661
|
-
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
4662
2948
|
return {
|
|
4663
|
-
mode,
|
|
4664
|
-
requestedMode,
|
|
2949
|
+
mode: "hosted",
|
|
2950
|
+
requestedMode: "hosted",
|
|
4665
2951
|
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
4666
|
-
profileSource,
|
|
4667
|
-
profileDirConfigured:
|
|
4668
|
-
executablePathConfigured:
|
|
2952
|
+
profileSource: "hosted",
|
|
2953
|
+
profileDirConfigured: false,
|
|
2954
|
+
executablePathConfigured: false
|
|
4669
2955
|
};
|
|
4670
2956
|
}
|
|
4671
2957
|
function structuredInstagramPagination(raw, input) {
|
|
@@ -4707,7 +2993,7 @@ function formatInstagramProfileContent(raw, input) {
|
|
|
4707
2993
|
const itemRows = items.slice(0, 100).map(
|
|
4708
2994
|
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
4709
2995
|
).join("\n");
|
|
4710
|
-
const browserLabel =
|
|
2996
|
+
const browserLabel = "hosted browser";
|
|
4711
2997
|
const full = [
|
|
4712
2998
|
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
4713
2999
|
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
@@ -4782,7 +3068,7 @@ function formatInstagramMediaDownload(raw, input) {
|
|
|
4782
3068
|
const transcriptText = transcript?.text ?? "";
|
|
4783
3069
|
const chunks = transcript?.chunks ?? [];
|
|
4784
3070
|
const browser = structuredInstagramBrowser(d.browser);
|
|
4785
|
-
const browserLabel =
|
|
3071
|
+
const browserLabel = "hosted browser";
|
|
4786
3072
|
const downloadRows = downloads.map((download, i) => {
|
|
4787
3073
|
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
4788
3074
|
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
@@ -5025,14 +3311,12 @@ var FacebookVideoTranscribeInputSchema = {
|
|
|
5025
3311
|
quality: import_zod3.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
5026
3312
|
};
|
|
5027
3313
|
var InstagramProfileContentInputSchema = {
|
|
5028
|
-
handle: import_zod3.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url.
|
|
3314
|
+
handle: import_zod3.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
|
|
5029
3315
|
url: import_zod3.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
5030
|
-
maxItems: import_zod3.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values
|
|
5031
|
-
maxScrolls: import_zod3.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase
|
|
3316
|
+
maxItems: import_zod3.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
|
|
3317
|
+
maxScrolls: import_zod3.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
|
|
5032
3318
|
scrollDelayMs: import_zod3.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
5033
|
-
stableScrollLimit: import_zod3.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
5034
|
-
browserMode: import_zod3.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
5035
|
-
browserProfile: import_zod3.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
3319
|
+
stableScrollLimit: import_zod3.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
5036
3320
|
};
|
|
5037
3321
|
var InstagramMediaDownloadInputSchema = {
|
|
5038
3322
|
url: import_zod3.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
@@ -5040,9 +3324,7 @@ var InstagramMediaDownloadInputSchema = {
|
|
|
5040
3324
|
downloadMedia: import_zod3.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
5041
3325
|
downloadAllTracks: import_zod3.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
5042
3326
|
includeTranscript: import_zod3.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
5043
|
-
mux: import_zod3.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
5044
|
-
browserMode: import_zod3.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
5045
|
-
browserProfile: import_zod3.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
3327
|
+
mux: import_zod3.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
5046
3328
|
};
|
|
5047
3329
|
var MapsPlaceIntelInputSchema = {
|
|
5048
3330
|
businessName: import_zod3.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
@@ -5441,10 +3723,10 @@ var TranscriptChunkOutput = import_zod3.z.object({
|
|
|
5441
3723
|
text: import_zod3.z.string()
|
|
5442
3724
|
});
|
|
5443
3725
|
var InstagramBrowserOutput = import_zod3.z.object({
|
|
5444
|
-
mode: import_zod3.z.
|
|
5445
|
-
requestedMode: import_zod3.z.
|
|
3726
|
+
mode: import_zod3.z.literal("hosted"),
|
|
3727
|
+
requestedMode: import_zod3.z.literal("hosted"),
|
|
5446
3728
|
profileName: NullableString2,
|
|
5447
|
-
profileSource: import_zod3.z.
|
|
3729
|
+
profileSource: import_zod3.z.literal("hosted"),
|
|
5448
3730
|
profileDirConfigured: import_zod3.z.boolean(),
|
|
5449
3731
|
executablePathConfigured: import_zod3.z.boolean()
|
|
5450
3732
|
});
|
|
@@ -6065,8 +4347,8 @@ function localPlanningToolAnnotations(title) {
|
|
|
6065
4347
|
}
|
|
6066
4348
|
function listSavedReports() {
|
|
6067
4349
|
try {
|
|
6068
|
-
const dir =
|
|
6069
|
-
return (0,
|
|
4350
|
+
const dir = outputBaseDir2();
|
|
4351
|
+
return (0, import_node_fs3.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs3.statSync)((0, import_node_path4.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
6070
4352
|
} catch {
|
|
6071
4353
|
return [];
|
|
6072
4354
|
}
|
|
@@ -6090,16 +4372,16 @@ function registerSavedReportResources(server2) {
|
|
|
6090
4372
|
},
|
|
6091
4373
|
async (uri, variables) => {
|
|
6092
4374
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
6093
|
-
const filename = (0,
|
|
4375
|
+
const filename = (0, import_node_path4.basename)(decodeURIComponent(String(requested ?? "")));
|
|
6094
4376
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
6095
|
-
const text = (0,
|
|
4377
|
+
const text = (0, import_node_fs3.readFileSync)((0, import_node_path4.join)(outputBaseDir2(), filename), "utf8");
|
|
6096
4378
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
6097
4379
|
}
|
|
6098
4380
|
);
|
|
6099
4381
|
}
|
|
6100
4382
|
function registerPaaExtractorMcpTools(server2, executor, options = {}) {
|
|
6101
4383
|
const savesReports = options.savesReportsLocally !== false;
|
|
6102
|
-
const reportNote = savesReports ? " Saves a full Markdown report
|
|
4384
|
+
const reportNote = savesReports ? " Saves a full Markdown report to disk." : " Reports are returned inline; no files are saved on this hosted endpoint.";
|
|
6103
4385
|
const withReportNote = (description) => `${description}${reportNote}`;
|
|
6104
4386
|
if (savesReports) registerSavedReportResources(server2);
|
|
6105
4387
|
server2.registerTool("harvest_paa", {
|
|
@@ -6181,14 +4463,14 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
|
|
|
6181
4463
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
6182
4464
|
server2.registerTool("instagram_profile_content", {
|
|
6183
4465
|
title: "Instagram Profile Content Discovery",
|
|
6184
|
-
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser
|
|
4466
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser details, pagination attempts, stop reason, and limitations."),
|
|
6185
4467
|
inputSchema: InstagramProfileContentInputSchema,
|
|
6186
4468
|
outputSchema: InstagramProfileContentOutputSchema,
|
|
6187
4469
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
6188
4470
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
6189
4471
|
server2.registerTool("instagram_media_download", {
|
|
6190
4472
|
title: "Instagram Post/Reel Media Download",
|
|
6191
|
-
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available.
|
|
4473
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available."),
|
|
6192
4474
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
6193
4475
|
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
6194
4476
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
@@ -6330,16 +4612,16 @@ function renderInstallTerminal(options) {
|
|
|
6330
4612
|
"1/1 install surfaces ready",
|
|
6331
4613
|
colorize("Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.", "lime", color),
|
|
6332
4614
|
"",
|
|
6333
|
-
`${colorize("Tools", "cyan", color)} ${colorize("(
|
|
4615
|
+
`${colorize("Tools", "cyan", color)} ${colorize("(43 MCP tools)", "muted", color)}`,
|
|
6334
4616
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
6335
4617
|
toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
|
|
6336
4618
|
toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
|
|
6337
4619
|
toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
|
|
6338
4620
|
toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
|
|
6339
|
-
toolRow("account", ["credits_info", "
|
|
4621
|
+
toolRow("account", ["credits_info", "reports", "MCP resources"], color),
|
|
6340
4622
|
`${colorize("Workflows", "cyan", color)} ${colorize("(MCP + CLI + API)", "muted", color)}`,
|
|
6341
4623
|
toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
|
|
6342
|
-
toolRow("seo", ["directory", "agent-packet", "
|
|
4624
|
+
toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
|
|
6343
4625
|
"",
|
|
6344
4626
|
colorize("Usage tips:", "amber", color),
|
|
6345
4627
|
"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
|
|
@@ -6392,10 +4674,10 @@ if (!forceStdio && (interactiveTerminal || wantsHelp)) {
|
|
|
6392
4674
|
}
|
|
6393
4675
|
function readApiKeyFile() {
|
|
6394
4676
|
const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
|
|
6395
|
-
const paths = [explicitPath, (0,
|
|
4677
|
+
const paths = [explicitPath, (0, import_node_path5.join)((0, import_node_os4.homedir)(), ".mcp-scraper-key")].filter(Boolean);
|
|
6396
4678
|
for (const path of paths) {
|
|
6397
4679
|
try {
|
|
6398
|
-
const value = (0,
|
|
4680
|
+
const value = (0, import_node_fs4.readFileSync)(path, "utf8").trim();
|
|
6399
4681
|
if (value) return value;
|
|
6400
4682
|
} catch {
|
|
6401
4683
|
}
|