mcp-scraper 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -26
- package/dist/bin/api-server.cjs +164 -329
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -4
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +47 -328
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +7 -91
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +18 -25
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2NEVRPK3.js +7 -0
- package/dist/chunk-2NEVRPK3.js.map +1 -0
- package/dist/{chunk-WZT3XKES.js → chunk-2O7RAK6E.js} +19 -26
- package/dist/chunk-2O7RAK6E.js.map +1 -0
- package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
- package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
- package/dist/{chunk-CMK5MMEE.js → chunk-IWNLHCKA.js} +13 -694
- package/dist/chunk-IWNLHCKA.js.map +1 -0
- package/dist/{server-BC4Z2L4P.js → server-HRVD52CX.js} +842 -122
- package/dist/server-HRVD52CX.js.map +1 -0
- package/docs/mcp-tool-craft-lint.generated.md +2 -5
- package/docs/mcp-tool-manifest.generated.json +9 -101
- package/package.json +1 -1
- package/dist/chunk-CMK5MMEE.js.map +0 -1
- package/dist/chunk-DUEW4EOO.js +0 -214
- package/dist/chunk-DUEW4EOO.js.map +0 -1
- package/dist/chunk-GHBU5SGQ.js +0 -7
- package/dist/chunk-GHBU5SGQ.js.map +0 -1
- package/dist/chunk-NXRWFOEZ.js +0 -816
- package/dist/chunk-NXRWFOEZ.js.map +0 -1
- package/dist/chunk-WZT3XKES.js.map +0 -1
- package/dist/server-BC4Z2L4P.js.map +0 -1
|
@@ -1,71 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
-
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
-
for (let key of __getOwnPropNames(from))
|
|
12
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
-
}
|
|
15
|
-
return to;
|
|
16
|
-
};
|
|
17
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
18
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
19
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
20
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
21
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
22
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
23
|
-
mod
|
|
24
|
-
));
|
|
25
3
|
|
|
26
4
|
// bin/browser-agent-stdio-server.ts
|
|
27
|
-
var
|
|
28
|
-
var
|
|
29
|
-
var
|
|
5
|
+
var import_node_fs2 = require("fs");
|
|
6
|
+
var import_node_os3 = require("os");
|
|
7
|
+
var import_node_path3 = require("path");
|
|
30
8
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
31
9
|
|
|
32
10
|
// src/mcp/browser-agent-mcp-server.ts
|
|
33
11
|
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
34
|
-
var
|
|
35
|
-
var
|
|
36
|
-
var
|
|
37
|
-
|
|
38
|
-
// src/lib/chrome-profiles.ts
|
|
39
|
-
var import_promises = require("fs/promises");
|
|
40
|
-
var import_node_os = require("os");
|
|
41
|
-
var import_node_path = require("path");
|
|
42
|
-
function defaultChromeUserDataDir() {
|
|
43
|
-
if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
|
|
44
|
-
if (process.platform === "win32") {
|
|
45
|
-
return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
|
|
46
|
-
}
|
|
47
|
-
return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
|
|
48
|
-
}
|
|
49
|
-
function recommendedKernelProfileName(value) {
|
|
50
|
-
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
51
|
-
}
|
|
52
|
-
async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
|
|
53
|
-
const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
|
|
54
|
-
const localState = JSON.parse(raw);
|
|
55
|
-
const wanted = emailFilter?.trim().toLowerCase();
|
|
56
|
-
return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
|
|
57
|
-
const email = (info.user_name ?? "").trim();
|
|
58
|
-
const displayName = (info.name ?? info.shortcut_name ?? "").trim();
|
|
59
|
-
const labelSource = email || displayName || directory;
|
|
60
|
-
return {
|
|
61
|
-
email,
|
|
62
|
-
displayName,
|
|
63
|
-
chromeProfileDirectory: directory,
|
|
64
|
-
chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
|
|
65
|
-
recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
|
|
66
|
-
};
|
|
67
|
-
}).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
|
|
68
|
-
}
|
|
12
|
+
var import_node_fs = require("fs");
|
|
13
|
+
var import_node_os2 = require("os");
|
|
14
|
+
var import_node_path2 = require("path");
|
|
69
15
|
|
|
70
16
|
// src/lib/browser-service-env.ts
|
|
71
17
|
function browserServiceProfileName() {
|
|
@@ -80,1363 +26,8 @@ function browserServiceProfileSaveChanges() {
|
|
|
80
26
|
return void 0;
|
|
81
27
|
}
|
|
82
28
|
|
|
83
|
-
// src/lib/local-browser-profiles.ts
|
|
84
|
-
var import_node_fs = require("fs");
|
|
85
|
-
var import_promises2 = require("fs/promises");
|
|
86
|
-
var import_node_os2 = require("os");
|
|
87
|
-
var import_node_path2 = require("path");
|
|
88
|
-
var EXCLUDED_PROFILE_NAMES = [
|
|
89
|
-
"Cache",
|
|
90
|
-
"Code Cache",
|
|
91
|
-
"Crashpad",
|
|
92
|
-
"DawnCache",
|
|
93
|
-
"DawnGraphiteCache",
|
|
94
|
-
"DawnWebGPUCache",
|
|
95
|
-
"GPUCache",
|
|
96
|
-
"GrShaderCache",
|
|
97
|
-
"ShaderCache",
|
|
98
|
-
"Shared Dictionary"
|
|
99
|
-
];
|
|
100
|
-
function defaultImportedBrowserProfilesDir() {
|
|
101
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
102
|
-
}
|
|
103
|
-
function defaultChromeExecutablePath() {
|
|
104
|
-
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
105
|
-
if (envPath) return envPath;
|
|
106
|
-
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
107
|
-
(0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
108
|
-
(0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
109
|
-
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
110
|
-
return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
|
|
111
|
-
}
|
|
112
|
-
function localBrowserModeEnabled() {
|
|
113
|
-
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
114
|
-
}
|
|
115
|
-
function importedProfileName(value) {
|
|
116
|
-
return recommendedKernelProfileName(value);
|
|
117
|
-
}
|
|
118
|
-
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
119
|
-
return (0, import_node_path2.join)(outputDir, importedProfileName(name));
|
|
120
|
-
}
|
|
121
|
-
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
122
|
-
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
123
|
-
}
|
|
124
|
-
function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
125
|
-
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
|
|
126
|
-
}
|
|
127
|
-
async function readExistingManifest(path) {
|
|
128
|
-
try {
|
|
129
|
-
return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
|
|
130
|
-
} catch {
|
|
131
|
-
return null;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
function isExcludedProfilePath(sourceRoot, sourcePath) {
|
|
135
|
-
const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
|
|
136
|
-
if (!rel) return false;
|
|
137
|
-
const parts = rel.split(import_node_path2.sep).filter(Boolean);
|
|
138
|
-
const last = parts[parts.length - 1] ?? "";
|
|
139
|
-
if (last.startsWith("Singleton")) return true;
|
|
140
|
-
if (last === "LOCK" || last.endsWith(".lock")) return true;
|
|
141
|
-
if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
|
|
142
|
-
if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
|
|
143
|
-
return false;
|
|
144
|
-
}
|
|
145
|
-
async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
|
|
146
|
-
await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
|
|
147
|
-
recursive: true,
|
|
148
|
-
force: true,
|
|
149
|
-
errorOnExist: false,
|
|
150
|
-
dereference: false,
|
|
151
|
-
filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
function selectProfile(profiles, options) {
|
|
155
|
-
const requestedDirectory = options.chromeProfileDirectory?.trim();
|
|
156
|
-
const requestedEmail = options.email?.trim().toLowerCase();
|
|
157
|
-
const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
|
|
158
|
-
if (exactDirectory) return exactDirectory;
|
|
159
|
-
const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
|
|
160
|
-
if (exactEmail) return exactEmail;
|
|
161
|
-
const first = profiles[0];
|
|
162
|
-
if (first) return first;
|
|
163
|
-
const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
|
|
164
|
-
throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
|
|
165
|
-
}
|
|
166
|
-
async function ensureProfilePathExists(profile) {
|
|
167
|
-
const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
|
|
168
|
-
if (!details?.isDirectory()) {
|
|
169
|
-
throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
|
|
173
|
-
const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
|
|
174
|
-
const localState = JSON.parse(raw);
|
|
175
|
-
const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
|
|
176
|
-
localState.profile = {
|
|
177
|
-
...localState.profile ?? {},
|
|
178
|
-
info_cache: { Default: selectedInfo },
|
|
179
|
-
last_used: "Default",
|
|
180
|
-
last_active_profiles: ["Default"]
|
|
181
|
-
};
|
|
182
|
-
await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
|
|
183
|
-
`);
|
|
184
|
-
}
|
|
185
|
-
async function importChromeProfile(options = {}) {
|
|
186
|
-
const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
|
|
187
|
-
const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
|
|
188
|
-
const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
|
|
189
|
-
await ensureProfilePathExists(sourceProfile);
|
|
190
|
-
const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
|
|
191
|
-
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
192
|
-
const profileRoot = importedProfileRoot(name, outputDir);
|
|
193
|
-
const manifestPath = importedProfileManifestPath(name, outputDir);
|
|
194
|
-
const userDataDir = importedProfileUserDataDir(name, outputDir);
|
|
195
|
-
const existingManifest = await readExistingManifest(manifestPath);
|
|
196
|
-
const existing = (0, import_node_fs.existsSync)(profileRoot);
|
|
197
|
-
if (existing && !options.overwrite) {
|
|
198
|
-
throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
|
|
199
|
-
}
|
|
200
|
-
if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
|
|
201
|
-
await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
|
|
202
|
-
await writeNormalizedLocalState(
|
|
203
|
-
(0, import_node_path2.join)(sourceUserDataDir, "Local State"),
|
|
204
|
-
(0, import_node_path2.join)(userDataDir, "Local State"),
|
|
205
|
-
sourceProfile.chromeProfileDirectory
|
|
206
|
-
);
|
|
207
|
-
await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
|
|
208
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
209
|
-
const manifest = {
|
|
210
|
-
layoutVersion: 1,
|
|
211
|
-
name,
|
|
212
|
-
profileRoot,
|
|
213
|
-
userDataDir,
|
|
214
|
-
browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
|
|
215
|
-
sourceUserDataDir,
|
|
216
|
-
sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
|
|
217
|
-
sourceProfilePath: sourceProfile.chromeProfilePath,
|
|
218
|
-
email: sourceProfile.email,
|
|
219
|
-
displayName: sourceProfile.displayName,
|
|
220
|
-
createdAt: existingManifest?.createdAt ?? now,
|
|
221
|
-
updatedAt: now,
|
|
222
|
-
copyMode: "chrome-profile-clone"
|
|
223
|
-
};
|
|
224
|
-
await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
225
|
-
`);
|
|
226
|
-
return {
|
|
227
|
-
profile: manifest,
|
|
228
|
-
sourceProfile,
|
|
229
|
-
excludedNames: EXCLUDED_PROFILE_NAMES
|
|
230
|
-
};
|
|
231
|
-
}
|
|
232
|
-
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
233
|
-
const profileName = importedProfileName(name);
|
|
234
|
-
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
235
|
-
const manifest = await readExistingManifest(manifestPath);
|
|
236
|
-
if (!manifest) {
|
|
237
|
-
throw new Error(
|
|
238
|
-
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
239
|
-
);
|
|
240
|
-
}
|
|
241
|
-
return manifest;
|
|
242
|
-
}
|
|
243
|
-
async function syncImportedChromeProfile(name, options = {}) {
|
|
244
|
-
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
245
|
-
const manifest = await loadImportedBrowserProfile(name, outputDir);
|
|
246
|
-
return importChromeProfile({
|
|
247
|
-
name: manifest.name,
|
|
248
|
-
sourceUserDataDir: manifest.sourceUserDataDir,
|
|
249
|
-
chromeProfileDirectory: manifest.sourceProfileDirectory,
|
|
250
|
-
outputDir,
|
|
251
|
-
overwrite: true,
|
|
252
|
-
browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
// src/services/browser-agent/local-browser-agent-service.ts
|
|
257
|
-
var import_node_crypto = require("crypto");
|
|
258
|
-
var import_node_fs3 = require("fs");
|
|
259
|
-
var import_node_path4 = require("path");
|
|
260
|
-
var import_playwright_extra = require("playwright-extra");
|
|
261
|
-
var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
|
|
262
|
-
|
|
263
|
-
// src/services/fanout/cdp-capture.ts
|
|
264
|
-
var ANSWER_RE = /\/backend-api\/(f\/)?conversation$|\/chat_conversations\/[^/]+\/(completion|retry_completion)$|\/completion$/i;
|
|
265
|
-
var FanoutCdpCapture = class {
|
|
266
|
-
client = null;
|
|
267
|
-
reqUrl = /* @__PURE__ */ new Map();
|
|
268
|
-
ctById = /* @__PURE__ */ new Map();
|
|
269
|
-
answerIds = /* @__PURE__ */ new Set();
|
|
270
|
-
bodies = [];
|
|
271
|
-
pending = [];
|
|
272
|
-
answerStreams = 0;
|
|
273
|
-
async attach(page) {
|
|
274
|
-
if (this.client) return;
|
|
275
|
-
const client = await page.context().newCDPSession(page);
|
|
276
|
-
this.client = client;
|
|
277
|
-
await client.send("Network.enable", { maxTotalBufferSize: 2e8, maxResourceBufferSize: 2e8 });
|
|
278
|
-
client.on("Network.requestWillBeSent", (e) => {
|
|
279
|
-
const id = e.requestId;
|
|
280
|
-
const req = e.request;
|
|
281
|
-
if (id) this.reqUrl.set(id, req?.url || "");
|
|
282
|
-
});
|
|
283
|
-
client.on("Network.responseReceived", (e) => {
|
|
284
|
-
const id = e.requestId;
|
|
285
|
-
const res = e.response;
|
|
286
|
-
const url = res?.url || this.reqUrl.get(id) || "";
|
|
287
|
-
const headers = res?.headers || {};
|
|
288
|
-
const ct = String(headers["content-type"] || headers["Content-Type"] || "");
|
|
289
|
-
if (id) this.ctById.set(id, ct);
|
|
290
|
-
if (id && (this.isAnswerUrl(url) || /event-stream/i.test(ct))) this.answerIds.add(id);
|
|
291
|
-
});
|
|
292
|
-
client.on("Network.loadingFinished", (e) => this.grab(e.requestId));
|
|
293
|
-
}
|
|
294
|
-
reset() {
|
|
295
|
-
this.bodies = [];
|
|
296
|
-
this.answerStreams = 0;
|
|
297
|
-
this.answerIds.clear();
|
|
298
|
-
this.pending = [];
|
|
299
|
-
}
|
|
300
|
-
answerStreamCount() {
|
|
301
|
-
return this.answerStreams;
|
|
302
|
-
}
|
|
303
|
-
async drain() {
|
|
304
|
-
await Promise.all(this.pending);
|
|
305
|
-
}
|
|
306
|
-
bestAnswerText() {
|
|
307
|
-
const eventStreams = this.bodies.filter((b) => /event-stream/i.test(b.ct));
|
|
308
|
-
const answerStreams = eventStreams.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
|
|
309
|
-
if (answerStreams.length) return answerStreams.map((b) => b.text).join("\n");
|
|
310
|
-
const answerJson = this.bodies.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
|
|
311
|
-
if (answerJson.length) return answerJson[0].text;
|
|
312
|
-
if (eventStreams.length) return eventStreams.sort((a, b) => b.text.length - a.text.length)[0].text;
|
|
313
|
-
return "";
|
|
314
|
-
}
|
|
315
|
-
async detach() {
|
|
316
|
-
try {
|
|
317
|
-
await this.client?.detach();
|
|
318
|
-
} catch {
|
|
319
|
-
this.client = null;
|
|
320
|
-
}
|
|
321
|
-
this.client = null;
|
|
322
|
-
}
|
|
323
|
-
isAnswerUrl(url) {
|
|
324
|
-
let path = "";
|
|
325
|
-
try {
|
|
326
|
-
path = new URL(url).pathname;
|
|
327
|
-
} catch {
|
|
328
|
-
path = String(url || "").split("?")[0];
|
|
329
|
-
}
|
|
330
|
-
return ANSWER_RE.test(path);
|
|
331
|
-
}
|
|
332
|
-
grab(requestId) {
|
|
333
|
-
if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
|
|
334
|
-
const url = this.reqUrl.get(requestId) || "";
|
|
335
|
-
const ct = this.ctById.get(requestId) || "";
|
|
336
|
-
const client = this.client;
|
|
337
|
-
const p = client.send("Network.getResponseBody", { requestId }).then((r) => {
|
|
338
|
-
const text = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
|
|
339
|
-
if (text) {
|
|
340
|
-
this.bodies.push({ url, ct, text });
|
|
341
|
-
if (this.isAnswerUrl(url)) this.answerStreams++;
|
|
342
|
-
}
|
|
343
|
-
}).catch(() => void 0);
|
|
344
|
-
this.pending.push(p);
|
|
345
|
-
}
|
|
346
|
-
};
|
|
347
|
-
|
|
348
|
-
// src/services/fanout/parse-helpers.ts
|
|
349
|
-
var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
|
|
350
|
-
var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
|
|
351
|
-
function domainOf(u) {
|
|
352
|
-
try {
|
|
353
|
-
return new URL(u).hostname.replace(/^www\./, "");
|
|
354
|
-
} catch {
|
|
355
|
-
return "";
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
function normUrl(u) {
|
|
359
|
-
try {
|
|
360
|
-
const x = new URL(u);
|
|
361
|
-
x.hash = "";
|
|
362
|
-
for (const k of [...x.searchParams.keys()]) {
|
|
363
|
-
if (TRACK.test(k)) x.searchParams.delete(k);
|
|
364
|
-
}
|
|
365
|
-
return x.toString().replace(/\/$/, "").toLowerCase();
|
|
366
|
-
} catch {
|
|
367
|
-
return String(u || "").toLowerCase();
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
function isExternal(u) {
|
|
371
|
-
const h = domainOf(u);
|
|
372
|
-
return Boolean(h) && !INTERNAL.test(h);
|
|
373
|
-
}
|
|
374
|
-
function unesc(s) {
|
|
375
|
-
try {
|
|
376
|
-
return JSON.parse('"' + s + '"');
|
|
377
|
-
} catch {
|
|
378
|
-
return String(s).replace(/\\(.)/g, "$1");
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
function stripCitationMarkers(s) {
|
|
382
|
-
return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
|
|
383
|
-
}
|
|
384
|
-
function sliceBalanced(text, openIdx) {
|
|
385
|
-
const open = text[openIdx];
|
|
386
|
-
const close = open === "[" ? "]" : "}";
|
|
387
|
-
let depth = 0;
|
|
388
|
-
let inStr = false;
|
|
389
|
-
let esc2 = false;
|
|
390
|
-
for (let i = openIdx; i < text.length; i++) {
|
|
391
|
-
const ch = text[i];
|
|
392
|
-
if (inStr) {
|
|
393
|
-
if (esc2) esc2 = false;
|
|
394
|
-
else if (ch === "\\") esc2 = true;
|
|
395
|
-
else if (ch === '"') inStr = false;
|
|
396
|
-
continue;
|
|
397
|
-
}
|
|
398
|
-
if (ch === '"') {
|
|
399
|
-
inStr = true;
|
|
400
|
-
continue;
|
|
401
|
-
}
|
|
402
|
-
if (ch === open) depth++;
|
|
403
|
-
else if (ch === close) {
|
|
404
|
-
depth--;
|
|
405
|
-
if (depth === 0) return text.slice(openIdx, i + 1);
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
return null;
|
|
409
|
-
}
|
|
410
|
-
function findJsonValues(text, field) {
|
|
411
|
-
const out = [];
|
|
412
|
-
const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
|
|
413
|
-
let m;
|
|
414
|
-
while (m = re.exec(text)) {
|
|
415
|
-
const openIdx = m.index + m[0].length - 1;
|
|
416
|
-
const slice = sliceBalanced(text, openIdx);
|
|
417
|
-
if (slice) {
|
|
418
|
-
try {
|
|
419
|
-
out.push(JSON.parse(slice));
|
|
420
|
-
} catch {
|
|
421
|
-
continue;
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
return out;
|
|
426
|
-
}
|
|
427
|
-
function stringsInArrayBody(arrBody) {
|
|
428
|
-
const out = [];
|
|
429
|
-
const re = /"((?:[^"\\]|\\.){2,300}?)"/g;
|
|
430
|
-
let m;
|
|
431
|
-
while (m = re.exec(arrBody)) out.push(unesc(m[1]));
|
|
432
|
-
return out;
|
|
433
|
-
}
|
|
434
|
-
function cleanQuery(q) {
|
|
435
|
-
const t = (q || "").trim();
|
|
436
|
-
if (t.length < 2 || t.length > 300) return null;
|
|
437
|
-
return t;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
// src/services/fanout/adapters/chatgpt.ts
|
|
441
|
-
function roundOf(refId) {
|
|
442
|
-
const m = /^turn(\d+)/.exec(refId || "");
|
|
443
|
-
return m ? Number(m[1]) : null;
|
|
444
|
-
}
|
|
445
|
-
function normRef(r) {
|
|
446
|
-
if (typeof r === "string") {
|
|
447
|
-
const m = /turn(\d+)(search|news|ref|video|image|product)(\d+)/.exec(r);
|
|
448
|
-
return m ? `turn${m[1]}${m[2]}${m[3]}` : null;
|
|
449
|
-
}
|
|
450
|
-
if (r && typeof r === "object") {
|
|
451
|
-
const o = r;
|
|
452
|
-
if (o.turn_index != null) {
|
|
453
|
-
return `turn${o.turn_index}${o.ref_type || "search"}${o.ref_index != null ? o.ref_index : ""}`;
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
return null;
|
|
457
|
-
}
|
|
458
|
-
function refsInText(s) {
|
|
459
|
-
return (String(s || "").match(/turn\d+(?:search|news|ref|video|image|product)\d+/g) || []).map(normRef).filter((x) => Boolean(x));
|
|
460
|
-
}
|
|
461
|
-
function collectInto(arr, v) {
|
|
462
|
-
if (Array.isArray(v)) arr.push(...v);
|
|
463
|
-
else if (v && typeof v === "object") arr.push(v);
|
|
464
|
-
}
|
|
465
|
-
function reconstructFromSsePatches(raw) {
|
|
466
|
-
const contentReferences = [];
|
|
467
|
-
const searchResultGroups = [];
|
|
468
|
-
const citations = [];
|
|
469
|
-
const citedUrls = /* @__PURE__ */ new Set();
|
|
470
|
-
const state = { found: false };
|
|
471
|
-
const handleOp = (op) => {
|
|
472
|
-
if (!op || typeof op !== "object") return;
|
|
473
|
-
const o = op;
|
|
474
|
-
const v = o.v;
|
|
475
|
-
if (o.o === "patch" && Array.isArray(v)) {
|
|
476
|
-
state.found = true;
|
|
477
|
-
v.forEach(handleOp);
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
const path = typeof o.p === "string" ? o.p : "";
|
|
481
|
-
if ((path === "" || path === "/message") && v && typeof v === "object") {
|
|
482
|
-
const root = v;
|
|
483
|
-
const msg = root.message || root;
|
|
484
|
-
const md = msg && msg.metadata;
|
|
485
|
-
if (md) {
|
|
486
|
-
collectInto(contentReferences, md.content_references);
|
|
487
|
-
collectInto(searchResultGroups, md.search_result_groups);
|
|
488
|
-
collectInto(citations, md.citations);
|
|
489
|
-
state.found = true;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
if (!path) return;
|
|
493
|
-
if (/\/safe_urls$/.test(path) && Array.isArray(v)) {
|
|
494
|
-
for (const u of v) if (typeof u === "string") citedUrls.add(u);
|
|
495
|
-
state.found = true;
|
|
496
|
-
} else if (path.includes("content_references")) {
|
|
497
|
-
collectInto(contentReferences, v);
|
|
498
|
-
state.found = true;
|
|
499
|
-
} else if (path.includes("search_result_groups")) {
|
|
500
|
-
collectInto(searchResultGroups, v);
|
|
501
|
-
state.found = true;
|
|
502
|
-
} else if (path.includes("citations")) {
|
|
503
|
-
collectInto(citations, v);
|
|
504
|
-
state.found = true;
|
|
505
|
-
}
|
|
506
|
-
};
|
|
507
|
-
for (const line of raw.split("\n")) {
|
|
508
|
-
const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
|
|
509
|
-
if (!s || s === "[DONE]" || s[0] !== "{" && s[0] !== "[") continue;
|
|
510
|
-
let parsed;
|
|
511
|
-
try {
|
|
512
|
-
parsed = JSON.parse(s);
|
|
513
|
-
} catch {
|
|
514
|
-
continue;
|
|
515
|
-
}
|
|
516
|
-
const ops = Array.isArray(parsed) ? parsed : [parsed];
|
|
517
|
-
ops.forEach(handleOp);
|
|
518
|
-
}
|
|
519
|
-
if (!state.found) return raw;
|
|
520
|
-
const citedRefs = [...citedUrls].map((url) => ({ type: "grouped_webpages", items: [{ url }] }));
|
|
521
|
-
const synthetic = JSON.stringify({
|
|
522
|
-
content_references: [...contentReferences, ...citedRefs],
|
|
523
|
-
search_result_groups: searchResultGroups,
|
|
524
|
-
citations
|
|
525
|
-
});
|
|
526
|
-
return raw + "\n" + synthetic;
|
|
527
|
-
}
|
|
528
|
-
var chatgptAdapter = {
|
|
529
|
-
platform: "ChatGPT",
|
|
530
|
-
matchesHost(hostname) {
|
|
531
|
-
return /(^|\.)chatgpt\.com$|(^|\.)openai\.com$/i.test(hostname);
|
|
532
|
-
},
|
|
533
|
-
parse(rawInput, prompt) {
|
|
534
|
-
const raw = reconstructFromSsePatches(rawInput);
|
|
535
|
-
const sources = /* @__PURE__ */ new Map();
|
|
536
|
-
const snippets = /* @__PURE__ */ new Map();
|
|
537
|
-
const queries = /* @__PURE__ */ new Set();
|
|
538
|
-
const unmapped = /* @__PURE__ */ new Set();
|
|
539
|
-
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
540
|
-
const ensure = (url) => {
|
|
541
|
-
if (!isExternal(url)) return null;
|
|
542
|
-
const k = normUrl(url);
|
|
543
|
-
let s = sources.get(k);
|
|
544
|
-
if (!s) {
|
|
545
|
-
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
|
|
546
|
-
sources.set(k, s);
|
|
547
|
-
}
|
|
548
|
-
return s;
|
|
549
|
-
};
|
|
550
|
-
const addSnippet = (url, title, text) => {
|
|
551
|
-
const clean = stripCitationMarkers(text);
|
|
552
|
-
if (!isExternal(url) || clean.length < 2) return;
|
|
553
|
-
snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean.slice(0, 400) });
|
|
554
|
-
};
|
|
555
|
-
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
556
|
-
for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
|
|
557
|
-
for (const q of stringsInArrayBody(m[1])) {
|
|
558
|
-
const c = cleanQuery(q);
|
|
559
|
-
if (c) queries.add(c);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
for (const m of raw.matchAll(/"(?:search_queries|queries|search_query)"\s*:\s*\[([^\]]*)\]/g)) {
|
|
563
|
-
for (const q of stringsInArrayBody(m[1])) {
|
|
564
|
-
const c = cleanQuery(q);
|
|
565
|
-
if (c) queries.add(c);
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
for (const v of findJsonValues(raw, "search_result_groups")) {
|
|
569
|
-
if (!Array.isArray(v)) continue;
|
|
570
|
-
for (const g of v) {
|
|
571
|
-
const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
|
|
572
|
-
for (const e of entries) {
|
|
573
|
-
const url = typeof e.url === "string" ? e.url : "";
|
|
574
|
-
const s = ensure(url);
|
|
575
|
-
if (!s) continue;
|
|
576
|
-
if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
|
|
577
|
-
const ref = normRef(e.ref_id);
|
|
578
|
-
if (ref && s.round == null) s.round = roundOf(ref);
|
|
579
|
-
if (typeof e.snippet === "string" && e.snippet) addSnippet(url, s.title, e.snippet);
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"(?:[\s\S]{0,120}?"(?:title|name)"\s*:\s*"((?:[^"\\]|\\.){0,200})")?/g)) {
|
|
584
|
-
const s = ensure(m[1]);
|
|
585
|
-
if (s && !s.title && m[2]) s.title = unesc(m[2]).slice(0, 200);
|
|
586
|
-
}
|
|
587
|
-
let spanTotal = 0;
|
|
588
|
-
for (const cr of findJsonValues(raw, "content_references")) {
|
|
589
|
-
if (!Array.isArray(cr)) continue;
|
|
590
|
-
for (const ref of cr) {
|
|
591
|
-
const type = typeof ref.type === "string" ? ref.type : "";
|
|
592
|
-
const isBundle = /sources_footnote|nav_list|alt_text/.test(type);
|
|
593
|
-
const refs = refsInText(typeof ref.matched_text === "string" ? ref.matched_text : "");
|
|
594
|
-
if (!isBundle) spanTotal++;
|
|
595
|
-
const ord = typeof ref.start_idx === "number" ? ref.start_idx : isBundle ? Infinity : spanTotal;
|
|
596
|
-
const items = Array.isArray(ref.items) ? ref.items : [];
|
|
597
|
-
items.forEach((it, idx) => {
|
|
598
|
-
const refForItem = refs.length === items.length ? refs[idx] : refs[0];
|
|
599
|
-
const urls = [it.url].concat((Array.isArray(it.supporting_websites) ? it.supporting_websites : []).map((s) => s?.url)).filter((u) => typeof u === "string");
|
|
600
|
-
for (const u of urls) {
|
|
601
|
-
const s = ensure(u);
|
|
602
|
-
if (!s) continue;
|
|
603
|
-
if (!isBundle) {
|
|
604
|
-
s.cited = true;
|
|
605
|
-
s.timesCited++;
|
|
606
|
-
if (ord < s.firstOrder) s.firstOrder = ord;
|
|
607
|
-
} else if (!s.cited) {
|
|
608
|
-
s.cited = true;
|
|
609
|
-
}
|
|
610
|
-
if (refForItem && s.round == null) s.round = roundOf(refForItem);
|
|
611
|
-
}
|
|
612
|
-
if (typeof it.url === "string" && typeof it.snippet === "string") addSnippet(it.url, typeof it.title === "string" ? it.title : "", it.snippet);
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
for (const v of findJsonValues(raw, "citations")) {
|
|
617
|
-
if (!Array.isArray(v)) continue;
|
|
618
|
-
v.forEach((c, i) => {
|
|
619
|
-
const md = c && typeof c === "object" ? c.metadata : null;
|
|
620
|
-
if (!md || typeof md.url !== "string") return;
|
|
621
|
-
const s = ensure(md.url);
|
|
622
|
-
if (!s) return;
|
|
623
|
-
s.cited = true;
|
|
624
|
-
s.timesCited++;
|
|
625
|
-
const ord = typeof c.start_ix === "number" ? c.start_ix : i;
|
|
626
|
-
if (ord < s.firstOrder) s.firstOrder = ord;
|
|
627
|
-
if (typeof md.text === "string") addSnippet(md.url, typeof md.title === "string" ? md.title : "", md.text);
|
|
628
|
-
});
|
|
629
|
-
}
|
|
630
|
-
for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
|
|
631
|
-
const url = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
|
|
632
|
-
const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
|
|
633
|
-
if (url) {
|
|
634
|
-
ensure(url);
|
|
635
|
-
addSnippet(url, title ? unesc(title) : "", unesc(m[2]));
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
|
|
639
|
-
const slugMatches = modelMatches.length ? modelMatches : [...raw.matchAll(/"model_slug"\s*:\s*"([^"]+)"/g)];
|
|
640
|
-
if (slugMatches.length) meta.model = slugMatches[slugMatches.length - 1][1];
|
|
641
|
-
const finish = raw.match(/"finish_details"\s*:\s*\{[^}]*"type"\s*:\s*"([^"]+)"/);
|
|
642
|
-
if (finish) meta.finishType = finish[1];
|
|
643
|
-
const searchRounds = (raw.match(/"recipient"\s*:\s*"(?:web|browser)"[\s\S]{0,400}?"content_type"\s*:\s*"code"[\s\S]{0,200}?search\(/g) || []).length;
|
|
644
|
-
const sourceRounds = /* @__PURE__ */ new Set();
|
|
645
|
-
for (const s of sources.values()) if (s.round != null) sourceRounds.add(s.round);
|
|
646
|
-
meta.rounds = Math.max(searchRounds, sourceRounds.size);
|
|
647
|
-
const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited || a.firstOrder - b.firstOrder : a.cited ? -1 : 1).map((s) => ({
|
|
648
|
-
url: s.url,
|
|
649
|
-
domain: s.domain,
|
|
650
|
-
title: s.title || s.domain,
|
|
651
|
-
cited: s.cited,
|
|
652
|
-
timesCited: s.timesCited,
|
|
653
|
-
snippet: snippets.get(normUrl(s.url))?.text || "",
|
|
654
|
-
round: s.round
|
|
655
|
-
}));
|
|
656
|
-
return {
|
|
657
|
-
subQueries: [...queries],
|
|
658
|
-
sources: ordered,
|
|
659
|
-
snippets: [...snippets.values()],
|
|
660
|
-
meta,
|
|
661
|
-
unmappedKeys: [...unmapped].slice(0, 50)
|
|
662
|
-
};
|
|
663
|
-
}
|
|
664
|
-
};
|
|
665
|
-
|
|
666
|
-
// src/services/fanout/adapters/claude.ts
|
|
667
|
-
function isFavicon(u) {
|
|
668
|
-
return /\/s2\/favicons/i.test(u) || /favicon/i.test(u);
|
|
669
|
-
}
|
|
670
|
-
var claudeAdapter = {
|
|
671
|
-
platform: "Claude",
|
|
672
|
-
matchesHost(hostname) {
|
|
673
|
-
return /(^|\.)claude\.ai$/i.test(hostname);
|
|
674
|
-
},
|
|
675
|
-
parse(raw, prompt) {
|
|
676
|
-
const sources = /* @__PURE__ */ new Map();
|
|
677
|
-
const snippets = /* @__PURE__ */ new Map();
|
|
678
|
-
const queries = /* @__PURE__ */ new Set();
|
|
679
|
-
const unmapped = /* @__PURE__ */ new Set();
|
|
680
|
-
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
681
|
-
const ensure = (url) => {
|
|
682
|
-
if (!url || !isExternal(url) || isFavicon(url)) return null;
|
|
683
|
-
const k = normUrl(url);
|
|
684
|
-
let s = sources.get(k);
|
|
685
|
-
if (!s) {
|
|
686
|
-
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
|
|
687
|
-
sources.set(k, s);
|
|
688
|
-
}
|
|
689
|
-
return s;
|
|
690
|
-
};
|
|
691
|
-
const toolInputs = /* @__PURE__ */ new Map();
|
|
692
|
-
const searchBlocks = /* @__PURE__ */ new Set();
|
|
693
|
-
const finalizeTool = (index) => {
|
|
694
|
-
const t = toolInputs.get(index);
|
|
695
|
-
if (!t || !/web_search|web_fetch|search|browse/i.test(t.name)) return;
|
|
696
|
-
try {
|
|
697
|
-
const input = JSON.parse(t.buf);
|
|
698
|
-
if (typeof input.query === "string") {
|
|
699
|
-
const q = cleanQuery(input.query);
|
|
700
|
-
if (q) queries.add(q);
|
|
701
|
-
}
|
|
702
|
-
} catch {
|
|
703
|
-
const m = t.buf.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)"/);
|
|
704
|
-
if (m) {
|
|
705
|
-
const q = cleanQuery(unesc(m[1]));
|
|
706
|
-
if (q) queries.add(q);
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
};
|
|
710
|
-
for (const line of raw.split("\n")) {
|
|
711
|
-
const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
|
|
712
|
-
if (!s || s[0] !== "{" && s[0] !== "[") continue;
|
|
713
|
-
let ev;
|
|
714
|
-
try {
|
|
715
|
-
ev = JSON.parse(s);
|
|
716
|
-
} catch {
|
|
717
|
-
continue;
|
|
718
|
-
}
|
|
719
|
-
const type = ev.type;
|
|
720
|
-
if (type === "content_block_start") {
|
|
721
|
-
const cb = ev.content_block || {};
|
|
722
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
723
|
-
if (cb.type === "tool_use" && index >= 0) {
|
|
724
|
-
const name = typeof cb.name === "string" ? cb.name : "";
|
|
725
|
-
toolInputs.set(index, { name, buf: "" });
|
|
726
|
-
if (/web_search|web_fetch|search|browse/i.test(name)) searchBlocks.add(index);
|
|
727
|
-
}
|
|
728
|
-
} else if (type === "content_block_delta") {
|
|
729
|
-
const d = ev.delta || {};
|
|
730
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
731
|
-
if (d.type === "input_json_delta" && typeof d.partial_json === "string") {
|
|
732
|
-
const t = toolInputs.get(index);
|
|
733
|
-
if (t) t.buf += d.partial_json;
|
|
734
|
-
} else if (d.type === "citation_start_delta") {
|
|
735
|
-
const c = d.citation || {};
|
|
736
|
-
if (typeof c.url === "string") {
|
|
737
|
-
const src = ensure(c.url);
|
|
738
|
-
if (src) {
|
|
739
|
-
src.cited = true;
|
|
740
|
-
src.timesCited++;
|
|
741
|
-
if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
|
|
742
|
-
if (typeof c.cited_text === "string" && c.cited_text) {
|
|
743
|
-
snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
} else if (type === "content_block_stop") {
|
|
749
|
-
const index = typeof ev.index === "number" ? ev.index : -1;
|
|
750
|
-
if (index >= 0) finalizeTool(index);
|
|
751
|
-
} else if (type === "message_delta" || type === "message_stop" || type === "message_start") {
|
|
752
|
-
const d = ev.delta || ev.message || {};
|
|
753
|
-
if (typeof d.stop_reason === "string" && !meta.finishType) meta.finishType = d.stop_reason;
|
|
754
|
-
if (typeof d.model === "string" && !meta.model) meta.model = d.model;
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
for (const index of searchBlocks) finalizeTool(index);
|
|
758
|
-
for (const m of raw.matchAll(/"title"\s*:\s*"((?:[^"\\]|\\.){1,200})"\s*,\s*"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) {
|
|
759
|
-
const src = ensure(m[2]);
|
|
760
|
-
if (src && !src.title) src.title = unesc(m[1]).slice(0, 200);
|
|
761
|
-
}
|
|
762
|
-
for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) ensure(m[1]);
|
|
763
|
-
for (const m of raw.matchAll(/\\"title\\"\s*:\s*\\"((?:[^"\\]|\\.){1,160}?)\\"[\s\S]{0,160}?\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) {
|
|
764
|
-
const src = ensure(m[2]);
|
|
765
|
-
if (src && !src.title) src.title = unesc(unesc(m[1])).slice(0, 200);
|
|
766
|
-
}
|
|
767
|
-
for (const m of raw.matchAll(/\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) ensure(m[1]);
|
|
768
|
-
if (!meta.model) {
|
|
769
|
-
const mm = raw.match(/"model"\s*:\s*"(claude[^"]+)"/);
|
|
770
|
-
if (mm) meta.model = mm[1];
|
|
771
|
-
}
|
|
772
|
-
meta.rounds = queries.size > 0 ? searchBlocks.size : 0;
|
|
773
|
-
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
774
|
-
const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited : a.cited ? -1 : 1).map((s) => ({
|
|
775
|
-
url: s.url,
|
|
776
|
-
domain: s.domain,
|
|
777
|
-
title: s.title || s.domain,
|
|
778
|
-
cited: s.cited,
|
|
779
|
-
timesCited: s.timesCited,
|
|
780
|
-
snippet: snippets.get(normUrl(s.url))?.text || "",
|
|
781
|
-
round: s.round
|
|
782
|
-
}));
|
|
783
|
-
return {
|
|
784
|
-
subQueries: [...queries],
|
|
785
|
-
sources: ordered,
|
|
786
|
-
snippets: [...snippets.values()],
|
|
787
|
-
meta,
|
|
788
|
-
unmappedKeys: [...unmapped].slice(0, 50)
|
|
789
|
-
};
|
|
790
|
-
}
|
|
791
|
-
};
|
|
792
|
-
|
|
793
|
-
// src/services/fanout/adapters/index.ts
|
|
794
|
-
var FANOUT_ADAPTERS = [chatgptAdapter, claudeAdapter];
|
|
795
|
-
function adapterForHost(hostname) {
|
|
796
|
-
return FANOUT_ADAPTERS.find((a) => a.matchesHost(hostname)) ?? null;
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
// src/services/fanout/build.ts
|
|
800
|
-
function buildFanoutResult(ctx) {
|
|
801
|
-
const { platform, parsed, ready, rawBytes } = ctx;
|
|
802
|
-
const browsedUrls = parsed.sources;
|
|
803
|
-
const citedUrls = browsedUrls.filter((s) => s.cited);
|
|
804
|
-
const browsedOnly = browsedUrls.filter((s) => !s.cited);
|
|
805
|
-
const prompt = ctx.prompt || parsed.meta.title || "";
|
|
806
|
-
const result = {
|
|
807
|
-
platform,
|
|
808
|
-
capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
809
|
-
prompt,
|
|
810
|
-
meta: parsed.meta,
|
|
811
|
-
subQueries: parsed.subQueries,
|
|
812
|
-
browsedUrls,
|
|
813
|
-
citedUrls,
|
|
814
|
-
browsedOnly,
|
|
815
|
-
snippets: parsed.snippets,
|
|
816
|
-
counts: {
|
|
817
|
-
subQueries: parsed.subQueries.length,
|
|
818
|
-
browsed: browsedUrls.length,
|
|
819
|
-
cited: citedUrls.length,
|
|
820
|
-
browsedOnly: browsedOnly.length
|
|
821
|
-
}
|
|
822
|
-
};
|
|
823
|
-
if (parsed.subQueries.length === 0 && browsedUrls.length === 0) {
|
|
824
|
-
result.debug = {
|
|
825
|
-
interceptorReady: ready,
|
|
826
|
-
rawBytes,
|
|
827
|
-
unmappedKeys: parsed.unmappedKeys,
|
|
828
|
-
note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. The session must be opened with MCP_SCRAPER_BROWSER_MODE=local and navigated to chatgpt.com or claude.ai after opening, so the hook installs before the site scripts run."
|
|
829
|
-
};
|
|
830
|
-
}
|
|
831
|
-
return result;
|
|
832
|
-
}
|
|
833
|
-
|
|
834
|
-
// src/services/fanout/classify.ts
|
|
835
|
-
var SOCIAL = /(^|\.)(youtube\.com|youtu\.be|instagram\.com|tiktok\.com|facebook\.com|fb\.com|x\.com|twitter\.com|linkedin\.com|pinterest\.com|threads\.net|vimeo\.com)$/i;
|
|
836
|
-
var ENCYCLOPEDIA = /(^|\.)(wikipedia\.org|wikimedia\.org|britannica\.com|merriam-webster\.com|dictionary\.com|fandom\.com|investopedia\.com)$/i;
|
|
837
|
-
var REVIEW = /(^|\.)(yelp\.com|trustpilot\.com|g2\.com|capterra\.com|angi\.com|angieslist\.com|bbb\.org|consumeraffairs\.com|bestcompany\.com|sitejabber\.com|clutch\.co|bestpickreports\.com|homeguide\.com|porch\.com|ontoplist\.com|thumbtack\.com)$/i;
|
|
838
|
-
var NEWS = /(^|\.)(reuters\.com|apnews\.com|nytimes\.com|wsj\.com|washingtonpost\.com|bloomberg\.com|cnbc\.com|forbes\.com|bbc\.com|theguardian\.com|cnn\.com|npr\.org|usatoday\.com|expressnews\.com|mysanantonio\.com|beaumontenterprise\.com)$/i;
|
|
839
|
-
function classifyDomain(domain, firstPartyDomain) {
|
|
840
|
-
const d = domain.toLowerCase().replace(/^www\./, "");
|
|
841
|
-
if (firstPartyDomain) {
|
|
842
|
-
const fp = firstPartyDomain.toLowerCase().replace(/^www\./, "");
|
|
843
|
-
if (fp && (d === fp || d.endsWith("." + fp))) return "First-party/vendor";
|
|
844
|
-
}
|
|
845
|
-
if (/(^|\.)reddit\.com$/i.test(d)) return "Reddit";
|
|
846
|
-
if (SOCIAL.test(d)) return "Social/video";
|
|
847
|
-
if (ENCYCLOPEDIA.test(d)) return "Encyclopedia";
|
|
848
|
-
if (REVIEW.test(d)) return "Review site";
|
|
849
|
-
if (NEWS.test(d) || /(^|\.)news\./i.test(d)) return "News/media";
|
|
850
|
-
if (/^docs?\.|^developer\.|^developers\.|readthedocs|\.dev$/i.test(d)) return "Docs";
|
|
851
|
-
return "Blog";
|
|
852
|
-
}
|
|
853
|
-
function emptyCategoryCounts() {
|
|
854
|
-
return { "First-party/vendor": 0, "News/media": 0, Reddit: 0, "Social/video": 0, Encyclopedia: 0, "Review site": 0, Docs: 0, Blog: 0 };
|
|
855
|
-
}
|
|
856
|
-
function enrichFanout(result, opts = {}) {
|
|
857
|
-
const addSiteType = (s) => ({ ...s, siteType: classifyDomain(s.domain, opts.firstPartyDomain) });
|
|
858
|
-
const browsedUrls = result.browsedUrls.map(addSiteType);
|
|
859
|
-
const citedUrls = result.citedUrls.map(addSiteType);
|
|
860
|
-
const browsedOnly = result.browsedOnly.map(addSiteType);
|
|
861
|
-
const byCategory = emptyCategoryCounts();
|
|
862
|
-
const domainAgg = /* @__PURE__ */ new Map();
|
|
863
|
-
for (const s of browsedUrls) {
|
|
864
|
-
byCategory[s.siteType]++;
|
|
865
|
-
const cur = domainAgg.get(s.domain);
|
|
866
|
-
if (!cur) domainAgg.set(s.domain, { domain: s.domain, count: 1, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType });
|
|
867
|
-
else {
|
|
868
|
-
cur.count++;
|
|
869
|
-
cur.cited = cur.cited || s.cited;
|
|
870
|
-
cur.timesCited += s.timesCited;
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
const topSites = [...domainAgg.values()].sort((a, b) => b.count - a.count || b.timesCited - a.timesCited);
|
|
874
|
-
const citationOrder = citedUrls.slice().sort((a, b) => b.timesCited - a.timesCited).map((s, i) => ({ rank: i + 1, domain: s.domain, url: s.url, timesCited: s.timesCited }));
|
|
875
|
-
return {
|
|
876
|
-
platform: result.platform,
|
|
877
|
-
capturedAt: result.capturedAt,
|
|
878
|
-
prompt: result.prompt,
|
|
879
|
-
meta: result.meta,
|
|
880
|
-
queries: result.subQueries,
|
|
881
|
-
browsedUrls,
|
|
882
|
-
citedUrls,
|
|
883
|
-
browsedOnly,
|
|
884
|
-
snippets: result.snippets,
|
|
885
|
-
counts: result.counts,
|
|
886
|
-
aggregates: { topSites, citationOrder, byCategory },
|
|
887
|
-
firstPartyDomain: opts.firstPartyDomain ? opts.firstPartyDomain.toLowerCase().replace(/^www\./, "") : null,
|
|
888
|
-
...result.debug ? { debug: result.debug } : {}
|
|
889
|
-
};
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
// src/services/fanout/export.ts
|
|
893
|
-
var import_node_fs2 = require("fs");
|
|
894
|
-
var import_node_os3 = require("os");
|
|
895
|
-
var import_node_path3 = require("path");
|
|
896
|
-
var import_papaparse = __toESM(require("papaparse"), 1);
|
|
897
|
-
function outputBaseDir() {
|
|
898
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
|
|
899
|
-
}
|
|
900
|
-
function safe(value) {
|
|
901
|
-
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
902
|
-
}
|
|
903
|
-
function writeTable(path, rows, delimiter) {
|
|
904
|
-
(0, import_node_fs2.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
905
|
-
}
|
|
906
|
-
function exportFanout(enriched) {
|
|
907
|
-
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
908
|
-
const dir = (0, import_node_path3.join)(outputBaseDir(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
909
|
-
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
|
910
|
-
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
911
|
-
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
912
|
-
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
913
|
-
return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
|
|
914
|
-
});
|
|
915
|
-
const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
916
|
-
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
917
|
-
const paths = {
|
|
918
|
-
dir,
|
|
919
|
-
json: (0, import_node_path3.join)(dir, "fanout.json"),
|
|
920
|
-
queriesCsv: (0, import_node_path3.join)(dir, "queries.csv"),
|
|
921
|
-
queriesTsv: (0, import_node_path3.join)(dir, "queries.tsv"),
|
|
922
|
-
citationsCsv: (0, import_node_path3.join)(dir, "citations.csv"),
|
|
923
|
-
sourcesCsv: (0, import_node_path3.join)(dir, "sources.csv"),
|
|
924
|
-
domainsCsv: (0, import_node_path3.join)(dir, "domains.csv"),
|
|
925
|
-
report: (0, import_node_path3.join)(dir, "report.html")
|
|
926
|
-
};
|
|
927
|
-
(0, import_node_fs2.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
|
|
928
|
-
writeTable(paths.queriesCsv, queryRows, ",");
|
|
929
|
-
writeTable(paths.queriesTsv, queryRows, " ");
|
|
930
|
-
writeTable(paths.citationsCsv, citationRows, ",");
|
|
931
|
-
writeTable(paths.sourcesCsv, sourceRows, ",");
|
|
932
|
-
writeTable(paths.domainsCsv, domainRows, ",");
|
|
933
|
-
(0, import_node_fs2.writeFileSync)(paths.report, renderReportHtml(enriched));
|
|
934
|
-
return paths;
|
|
935
|
-
}
|
|
936
|
-
function esc(s) {
|
|
937
|
-
return String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
938
|
-
}
|
|
939
|
-
function renderReportHtml(enriched) {
|
|
940
|
-
const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
|
|
941
|
-
return `<!doctype html>
|
|
942
|
-
<html lang="en">
|
|
943
|
-
<head>
|
|
944
|
-
<meta charset="utf-8" />
|
|
945
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
946
|
-
<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
|
|
947
|
-
<style>
|
|
948
|
-
:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
|
|
949
|
-
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
|
|
950
|
-
.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
|
|
951
|
-
h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
|
|
952
|
-
.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
|
|
953
|
-
.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
|
|
954
|
-
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
955
|
-
.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
|
|
956
|
-
.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
|
|
957
|
-
.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
|
|
958
|
-
.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
|
|
959
|
-
.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
|
|
960
|
-
.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
|
|
961
|
-
.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
|
|
962
|
-
.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
|
|
963
|
-
.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
|
|
964
|
-
.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
|
|
965
|
-
.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
|
|
966
|
-
.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
|
|
967
|
-
.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
|
|
968
|
-
.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
|
|
969
|
-
.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
|
|
970
|
-
.hide{display:none}
|
|
971
|
-
</style>
|
|
972
|
-
</head>
|
|
973
|
-
<body><div class="wrap">
|
|
974
|
-
<h1>AI Search Fan-Out</h1>
|
|
975
|
-
<p class="sub" id="sub"></p>
|
|
976
|
-
<div class="stat" id="stat"></div>
|
|
977
|
-
<div class="cards" id="cards"></div>
|
|
978
|
-
<div id="charts"></div>
|
|
979
|
-
<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
|
|
980
|
-
<div class="chips" id="chips"></div>
|
|
981
|
-
<div id="list"></div>
|
|
982
|
-
</div>
|
|
983
|
-
<script>
|
|
984
|
-
const D=${data};
|
|
985
|
-
const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
|
|
986
|
-
document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
|
|
987
|
-
document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
|
|
988
|
-
const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
|
|
989
|
-
const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
|
|
990
|
-
function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
|
|
991
|
-
const charts=document.getElementById('charts');
|
|
992
|
-
charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
|
|
993
|
-
let tab='q',filter=null;
|
|
994
|
-
const chips=document.getElementById('chips'),list=document.getElementById('list');
|
|
995
|
-
function render(){
|
|
996
|
-
chips.innerHTML='';list.innerHTML='';
|
|
997
|
-
if(tab==='q'){
|
|
998
|
-
D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
|
|
999
|
-
} else {
|
|
1000
|
-
const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
|
|
1001
|
-
Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
|
|
1002
|
-
D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}
|
|
1006
|
-
document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
|
|
1007
|
-
render();
|
|
1008
|
-
</script>
|
|
1009
|
-
</body></html>`;
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
// src/services/fanout/run-capture.ts
|
|
1013
|
-
var COMPOSER_SELECTORS = [
|
|
1014
|
-
"#prompt-textarea",
|
|
1015
|
-
"div#prompt-textarea",
|
|
1016
|
-
'textarea[data-testid="prompt-textarea"]',
|
|
1017
|
-
'div.ProseMirror[contenteditable="true"]',
|
|
1018
|
-
'div[contenteditable="true"]',
|
|
1019
|
-
"textarea"
|
|
1020
|
-
];
|
|
1021
|
-
async function sendPrompt(page, text) {
|
|
1022
|
-
for (const selector of COMPOSER_SELECTORS) {
|
|
1023
|
-
const locator = page.locator(selector).first();
|
|
1024
|
-
try {
|
|
1025
|
-
await locator.waitFor({ state: "visible", timeout: 12e3 });
|
|
1026
|
-
} catch {
|
|
1027
|
-
continue;
|
|
1028
|
-
}
|
|
1029
|
-
await locator.click({ timeout: 4e3 }).catch(() => void 0);
|
|
1030
|
-
await locator.focus().catch(() => void 0);
|
|
1031
|
-
await page.keyboard.type(text, { delay: 12 });
|
|
1032
|
-
await page.waitForTimeout(300).catch(() => void 0);
|
|
1033
|
-
await page.keyboard.press("Enter");
|
|
1034
|
-
return true;
|
|
1035
|
-
}
|
|
1036
|
-
return false;
|
|
1037
|
-
}
|
|
1038
|
-
async function waitForAnswer(page, cap, baseline, waitMs) {
|
|
1039
|
-
const start = Date.now();
|
|
1040
|
-
while (Date.now() - start < waitMs && cap.answerStreamCount() <= baseline) {
|
|
1041
|
-
await page.waitForTimeout(400).catch(() => void 0);
|
|
1042
|
-
}
|
|
1043
|
-
await page.waitForTimeout(1e3).catch(() => void 0);
|
|
1044
|
-
}
|
|
1045
|
-
async function runFanoutCapture(page, input) {
|
|
1046
|
-
const cap = new FanoutCdpCapture();
|
|
1047
|
-
await cap.attach(page);
|
|
1048
|
-
try {
|
|
1049
|
-
const baseline = cap.answerStreamCount();
|
|
1050
|
-
if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
|
|
1051
|
-
const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
|
|
1052
|
-
await waitForAnswer(page, cap, baseline, waitMs);
|
|
1053
|
-
await cap.drain();
|
|
1054
|
-
let hostname = "";
|
|
1055
|
-
try {
|
|
1056
|
-
hostname = new URL(page.url()).hostname;
|
|
1057
|
-
} catch {
|
|
1058
|
-
hostname = "";
|
|
1059
|
-
}
|
|
1060
|
-
const adapter = adapterForHost(hostname);
|
|
1061
|
-
if (!adapter) {
|
|
1062
|
-
throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
|
|
1063
|
-
}
|
|
1064
|
-
const text = cap.bestAnswerText();
|
|
1065
|
-
const parsed = adapter.parse(text, input.prompt || "");
|
|
1066
|
-
const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text.length });
|
|
1067
|
-
const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
|
|
1068
|
-
const exports2 = input.export ? exportFanout(enriched) : null;
|
|
1069
|
-
return { result: enriched, exports: exports2 };
|
|
1070
|
-
} finally {
|
|
1071
|
-
await cap.detach().catch(() => void 0);
|
|
1072
|
-
}
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
// src/services/browser-agent/local-browser-agent-service.ts
|
|
1076
|
-
import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
|
|
1077
|
-
function directProfileDir() {
|
|
1078
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
|
|
1079
|
-
}
|
|
1080
|
-
function configuredProfileName(profile) {
|
|
1081
|
-
return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directProfileDir() ? (0, import_node_path4.basename)(directProfileDir()) : "default");
|
|
1082
|
-
}
|
|
1083
|
-
function launchExecutablePath(profile) {
|
|
1084
|
-
const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || profile.browserExecutablePath || defaultChromeExecutablePath();
|
|
1085
|
-
return configured && (0, import_node_fs3.existsSync)(configured) ? configured : void 0;
|
|
1086
|
-
}
|
|
1087
|
-
function ignoreDefaultArgsFor(executablePath) {
|
|
1088
|
-
if (!executablePath) return void 0;
|
|
1089
|
-
if (process.platform !== "darwin") return void 0;
|
|
1090
|
-
if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
|
|
1091
|
-
return ["--password-store=basic", "--use-mock-keychain"];
|
|
1092
|
-
}
|
|
1093
|
-
function normalizeKeyCombo(value) {
|
|
1094
|
-
return value.split("+").map((part) => {
|
|
1095
|
-
const trimmed = part.trim();
|
|
1096
|
-
const lower = trimmed.toLowerCase();
|
|
1097
|
-
if (lower === "ctrl" || lower === "control") return "Control";
|
|
1098
|
-
if (lower === "cmd" || lower === "command" || lower === "meta") return "Meta";
|
|
1099
|
-
if (lower === "option" || lower === "alt") return "Alt";
|
|
1100
|
-
if (lower === "return") return "Enter";
|
|
1101
|
-
if (lower === "esc") return "Escape";
|
|
1102
|
-
if (trimmed.length === 1) return trimmed.toUpperCase();
|
|
1103
|
-
return trimmed;
|
|
1104
|
-
}).join("+");
|
|
1105
|
-
}
|
|
1106
|
-
function sessionSummary(session) {
|
|
1107
|
-
return {
|
|
1108
|
-
session_id: session.sessionId,
|
|
1109
|
-
status: session.status,
|
|
1110
|
-
label: session.label,
|
|
1111
|
-
profile: session.profileName,
|
|
1112
|
-
user_data_dir: session.userDataDir,
|
|
1113
|
-
created_at: session.createdAt,
|
|
1114
|
-
last_action_at: session.lastActionAt,
|
|
1115
|
-
closed_at: session.closedAt,
|
|
1116
|
-
watch_url: `local://browser-session/${session.sessionId}`,
|
|
1117
|
-
url: session.status === "open" ? session.page.url() : null
|
|
1118
|
-
};
|
|
1119
|
-
}
|
|
1120
|
-
var LocalBrowserAgentService = class {
|
|
1121
|
-
contexts = /* @__PURE__ */ new Map();
|
|
1122
|
-
sessions = /* @__PURE__ */ new Map();
|
|
1123
|
-
async open(input) {
|
|
1124
|
-
const profile = await this.resolveProfile(input.profile);
|
|
1125
|
-
const contextState = await this.ensureContext(profile);
|
|
1126
|
-
const pages = contextState.context.pages();
|
|
1127
|
-
const page = pages.length === 1 && !this.hasOpenSessionForContext(contextState.key) ? pages[0] : await contextState.context.newPage();
|
|
1128
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1129
|
-
const sessionId = `lbs_${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
1130
|
-
const session = {
|
|
1131
|
-
sessionId,
|
|
1132
|
-
label: input.label ?? null,
|
|
1133
|
-
page,
|
|
1134
|
-
contextKey: contextState.key,
|
|
1135
|
-
profileName: profile.name,
|
|
1136
|
-
userDataDir: profile.userDataDir,
|
|
1137
|
-
createdAt: now,
|
|
1138
|
-
lastActionAt: now,
|
|
1139
|
-
closedAt: null,
|
|
1140
|
-
status: "open"
|
|
1141
|
-
};
|
|
1142
|
-
const timeoutSeconds = input.timeout_seconds ?? 600;
|
|
1143
|
-
if (timeoutSeconds > 0) {
|
|
1144
|
-
session.timeout = setTimeout(() => {
|
|
1145
|
-
void this.close(sessionId);
|
|
1146
|
-
}, timeoutSeconds * 1e3);
|
|
1147
|
-
session.timeout.unref();
|
|
1148
|
-
}
|
|
1149
|
-
this.sessions.set(sessionId, session);
|
|
1150
|
-
if (input.url) await this.goto(sessionId, input.url);
|
|
1151
|
-
return {
|
|
1152
|
-
session_id: sessionId,
|
|
1153
|
-
watch_url: `local://browser-session/${sessionId}`,
|
|
1154
|
-
live_view_url: null,
|
|
1155
|
-
profile: profile.name,
|
|
1156
|
-
user_data_dir: profile.userDataDir,
|
|
1157
|
-
url: input.url ?? null,
|
|
1158
|
-
mode: "local"
|
|
1159
|
-
};
|
|
1160
|
-
}
|
|
1161
|
-
async screenshot(sessionId) {
|
|
1162
|
-
const session = this.requireOpenSession(sessionId);
|
|
1163
|
-
await this.waitForPage(session.page);
|
|
1164
|
-
const image = await session.page.screenshot({ type: "png", fullPage: false });
|
|
1165
|
-
const page = await this.read(sessionId);
|
|
1166
|
-
return {
|
|
1167
|
-
...page,
|
|
1168
|
-
image_base64: image.toString("base64"),
|
|
1169
|
-
mime_type: "image/png"
|
|
1170
|
-
};
|
|
1171
|
-
}
|
|
1172
|
-
async read(sessionId) {
|
|
1173
|
-
const session = this.requireOpenSession(sessionId);
|
|
1174
|
-
await this.waitForPage(session.page);
|
|
1175
|
-
this.touch(session);
|
|
1176
|
-
const title = await session.page.title().catch(() => "");
|
|
1177
|
-
const snapshot = await session.page.evaluate(() => {
|
|
1178
|
-
const cssEscape = (value) => {
|
|
1179
|
-
const escapeFn = globalThis.CSS?.escape;
|
|
1180
|
-
return escapeFn ? escapeFn(value) : value.replace(/["\\#.:>+~[\]()]/g, "\\$&");
|
|
1181
|
-
};
|
|
1182
|
-
const textFor = (element) => {
|
|
1183
|
-
const html = element;
|
|
1184
|
-
return (element.getAttribute("aria-label") || element.getAttribute("title") || html.placeholder || html.value || html.innerText || element.textContent || "").replace(/\s+/g, " ").trim();
|
|
1185
|
-
};
|
|
1186
|
-
const selectorFor = (element) => {
|
|
1187
|
-
const id = element.id;
|
|
1188
|
-
if (id) return `#${cssEscape(id)}`;
|
|
1189
|
-
const name = element.getAttribute("name");
|
|
1190
|
-
const tag = element.tagName.toLowerCase();
|
|
1191
|
-
if (name) return `${tag}[name="${name.replace(/"/g, '\\"')}"]`;
|
|
1192
|
-
const parts = [];
|
|
1193
|
-
let current = element;
|
|
1194
|
-
while (current && current !== document.body && parts.length < 4) {
|
|
1195
|
-
const currentTag = current.tagName.toLowerCase();
|
|
1196
|
-
const parent = current.parentElement;
|
|
1197
|
-
if (!parent) {
|
|
1198
|
-
parts.unshift(currentTag);
|
|
1199
|
-
break;
|
|
1200
|
-
}
|
|
1201
|
-
const children = Array.from(parent.children);
|
|
1202
|
-
const siblings = children.filter((child) => child.tagName === current.tagName);
|
|
1203
|
-
const index = siblings.indexOf(current) + 1;
|
|
1204
|
-
parts.unshift(siblings.length > 1 ? `${currentTag}:nth-of-type(${index})` : currentTag);
|
|
1205
|
-
current = parent;
|
|
1206
|
-
}
|
|
1207
|
-
return parts.join(" > ") || tag;
|
|
1208
|
-
};
|
|
1209
|
-
const candidates = Array.from(document.querySelectorAll(
|
|
1210
|
-
'a, button, input, textarea, select, summary, [role], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'
|
|
1211
|
-
));
|
|
1212
|
-
const elements = candidates.map((element, index) => {
|
|
1213
|
-
const rect = element.getBoundingClientRect();
|
|
1214
|
-
const style = window.getComputedStyle(element);
|
|
1215
|
-
const visible = rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && Number(style.opacity || "1") > 0;
|
|
1216
|
-
if (!visible) return null;
|
|
1217
|
-
const label = textFor(element);
|
|
1218
|
-
return {
|
|
1219
|
-
index,
|
|
1220
|
-
tag: element.tagName.toLowerCase(),
|
|
1221
|
-
role: element.getAttribute("role") || null,
|
|
1222
|
-
text: label.slice(0, 180),
|
|
1223
|
-
selector: selectorFor(element),
|
|
1224
|
-
x: Math.round(rect.left + rect.width / 2),
|
|
1225
|
-
y: Math.round(rect.top + rect.height / 2),
|
|
1226
|
-
left: Math.round(rect.left),
|
|
1227
|
-
top: Math.round(rect.top),
|
|
1228
|
-
width: Math.round(rect.width),
|
|
1229
|
-
height: Math.round(rect.height)
|
|
1230
|
-
};
|
|
1231
|
-
}).filter(Boolean).slice(0, 120);
|
|
1232
|
-
return {
|
|
1233
|
-
text: (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").slice(0, 3e4),
|
|
1234
|
-
elements,
|
|
1235
|
-
viewport: {
|
|
1236
|
-
width: window.innerWidth,
|
|
1237
|
-
height: window.innerHeight,
|
|
1238
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
1239
|
-
scrollX: window.scrollX,
|
|
1240
|
-
scrollY: window.scrollY
|
|
1241
|
-
}
|
|
1242
|
-
};
|
|
1243
|
-
});
|
|
1244
|
-
return {
|
|
1245
|
-
url: session.page.url(),
|
|
1246
|
-
title,
|
|
1247
|
-
text: snapshot.text,
|
|
1248
|
-
elements: snapshot.elements,
|
|
1249
|
-
viewport: snapshot.viewport
|
|
1250
|
-
};
|
|
1251
|
-
}
|
|
1252
|
-
async locate(sessionId, targets) {
|
|
1253
|
-
const session = this.requireOpenSession(sessionId);
|
|
1254
|
-
this.touch(session);
|
|
1255
|
-
const located = [];
|
|
1256
|
-
for (const target of targets) {
|
|
1257
|
-
try {
|
|
1258
|
-
const locator = target.selector ? session.page.locator(target.selector) : session.page.getByText(target.text ?? "", { exact: target.match === "exact" });
|
|
1259
|
-
const selected = locator.nth(target.index ?? 0);
|
|
1260
|
-
await selected.waitFor({ state: "visible", timeout: 3e3 }).catch(() => void 0);
|
|
1261
|
-
const box = await selected.boundingBox();
|
|
1262
|
-
const text = await selected.textContent({ timeout: 1e3 }).catch(() => null);
|
|
1263
|
-
located.push({
|
|
1264
|
-
input: target,
|
|
1265
|
-
found: Boolean(box),
|
|
1266
|
-
element: box ? {
|
|
1267
|
-
left: Math.round(box.x),
|
|
1268
|
-
top: Math.round(box.y),
|
|
1269
|
-
width: Math.round(box.width),
|
|
1270
|
-
height: Math.round(box.height),
|
|
1271
|
-
x: Math.round(box.x + box.width / 2),
|
|
1272
|
-
y: Math.round(box.y + box.height / 2),
|
|
1273
|
-
text: (text ?? "").replace(/\s+/g, " ").trim().slice(0, 220)
|
|
1274
|
-
} : null
|
|
1275
|
-
});
|
|
1276
|
-
} catch (err) {
|
|
1277
|
-
located.push({
|
|
1278
|
-
input: target,
|
|
1279
|
-
found: false,
|
|
1280
|
-
element: null,
|
|
1281
|
-
error: err instanceof Error ? err.message : String(err)
|
|
1282
|
-
});
|
|
1283
|
-
}
|
|
1284
|
-
}
|
|
1285
|
-
const read = await this.read(sessionId);
|
|
1286
|
-
return {
|
|
1287
|
-
url: read.url,
|
|
1288
|
-
title: read.title,
|
|
1289
|
-
viewport: read.viewport,
|
|
1290
|
-
replay: null,
|
|
1291
|
-
targets: located
|
|
1292
|
-
};
|
|
1293
|
-
}
|
|
1294
|
-
async goto(sessionId, url) {
|
|
1295
|
-
const session = this.requireOpenSession(sessionId);
|
|
1296
|
-
this.touch(session);
|
|
1297
|
-
const response = await session.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 }).catch((err) => {
|
|
1298
|
-
throw new Error(err instanceof Error ? err.message : String(err));
|
|
1299
|
-
});
|
|
1300
|
-
await this.waitForPage(session.page);
|
|
1301
|
-
return {
|
|
1302
|
-
ok: true,
|
|
1303
|
-
url: session.page.url(),
|
|
1304
|
-
status: response?.status() ?? null,
|
|
1305
|
-
mode: "local"
|
|
1306
|
-
};
|
|
1307
|
-
}
|
|
1308
|
-
async click(sessionId, input) {
|
|
1309
|
-
const session = this.requireOpenSession(sessionId);
|
|
1310
|
-
this.touch(session);
|
|
1311
|
-
await session.page.mouse.click(input.x, input.y, {
|
|
1312
|
-
button: input.button ?? "left",
|
|
1313
|
-
clickCount: input.num_clicks ?? 1
|
|
1314
|
-
});
|
|
1315
|
-
await this.waitForPage(session.page);
|
|
1316
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1317
|
-
}
|
|
1318
|
-
async type(sessionId, input) {
|
|
1319
|
-
const session = this.requireOpenSession(sessionId);
|
|
1320
|
-
this.touch(session);
|
|
1321
|
-
await session.page.keyboard.type(input.text, { delay: input.delay });
|
|
1322
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1323
|
-
}
|
|
1324
|
-
async scroll(sessionId, input) {
|
|
1325
|
-
const session = this.requireOpenSession(sessionId);
|
|
1326
|
-
this.touch(session);
|
|
1327
|
-
if (typeof input.x === "number" && typeof input.y === "number") {
|
|
1328
|
-
await session.page.mouse.move(input.x, input.y);
|
|
1329
|
-
}
|
|
1330
|
-
await session.page.mouse.wheel((input.delta_x ?? 0) * 120, (input.delta_y ?? 5) * 120);
|
|
1331
|
-
await session.page.waitForTimeout(150).catch(() => void 0);
|
|
1332
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1333
|
-
}
|
|
1334
|
-
async press(sessionId, input) {
|
|
1335
|
-
const session = this.requireOpenSession(sessionId);
|
|
1336
|
-
this.touch(session);
|
|
1337
|
-
for (const key of input.keys) {
|
|
1338
|
-
await session.page.keyboard.press(normalizeKeyCombo(key));
|
|
1339
|
-
}
|
|
1340
|
-
await this.waitForPage(session.page);
|
|
1341
|
-
return { ok: true, mode: "local", url: session.page.url() };
|
|
1342
|
-
}
|
|
1343
|
-
async close(sessionId) {
|
|
1344
|
-
const session = this.sessions.get(sessionId);
|
|
1345
|
-
if (!session || session.status === "closed") {
|
|
1346
|
-
return { ok: true, closed: true, already_closed: true, mode: "local" };
|
|
1347
|
-
}
|
|
1348
|
-
if (session.timeout) clearTimeout(session.timeout);
|
|
1349
|
-
session.status = "closed";
|
|
1350
|
-
session.closedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1351
|
-
await session.page.close().catch(() => void 0);
|
|
1352
|
-
await this.closeContextIfUnused(session.contextKey);
|
|
1353
|
-
return { ok: true, closed: true, mode: "local" };
|
|
1354
|
-
}
|
|
1355
|
-
list(includeClosed = false) {
|
|
1356
|
-
return Array.from(this.sessions.values()).filter((session) => includeClosed || session.status === "open").map(sessionSummary);
|
|
1357
|
-
}
|
|
1358
|
-
async captureFanout(sessionId, input) {
|
|
1359
|
-
const session = this.requireOpenSession(sessionId);
|
|
1360
|
-
this.touch(session);
|
|
1361
|
-
return runFanoutCapture(session.page, input);
|
|
1362
|
-
}
|
|
1363
|
-
async screenshotOnly(sessionId) {
|
|
1364
|
-
const session = this.requireOpenSession(sessionId);
|
|
1365
|
-
await this.waitForPage(session.page);
|
|
1366
|
-
const img = await session.page.screenshot({ type: "png", fullPage: false });
|
|
1367
|
-
return img.toString("base64");
|
|
1368
|
-
}
|
|
1369
|
-
async resolveProfile(profile) {
|
|
1370
|
-
const directDir = directProfileDir();
|
|
1371
|
-
const requestedName = configuredProfileName(profile);
|
|
1372
|
-
if (directDir && (!profile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
|
|
1373
|
-
return {
|
|
1374
|
-
name: requestedName,
|
|
1375
|
-
userDataDir: directDir,
|
|
1376
|
-
browserExecutablePath: process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || defaultChromeExecutablePath() || null,
|
|
1377
|
-
manifest: null
|
|
1378
|
-
};
|
|
1379
|
-
}
|
|
1380
|
-
const manifest = await loadImportedBrowserProfile(requestedName);
|
|
1381
|
-
return {
|
|
1382
|
-
name: manifest.name,
|
|
1383
|
-
userDataDir: manifest.userDataDir,
|
|
1384
|
-
browserExecutablePath: manifest.browserExecutablePath,
|
|
1385
|
-
manifest
|
|
1386
|
-
};
|
|
1387
|
-
}
|
|
1388
|
-
async ensureContext(profile) {
|
|
1389
|
-
const existing = this.contexts.get(profile.userDataDir);
|
|
1390
|
-
if (existing) return existing;
|
|
1391
|
-
const executablePath = launchExecutablePath(profile);
|
|
1392
|
-
const context = await import_playwright_extra.chromium.launchPersistentContext(profile.userDataDir, {
|
|
1393
|
-
headless: false,
|
|
1394
|
-
viewport: { width: 1440, height: 900 },
|
|
1395
|
-
acceptDownloads: true,
|
|
1396
|
-
...executablePath ? { executablePath } : {},
|
|
1397
|
-
...ignoreDefaultArgsFor(executablePath) ? { ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath) } : {}
|
|
1398
|
-
});
|
|
1399
|
-
const state = { key: profile.userDataDir, context, profile };
|
|
1400
|
-
context.on("close", () => {
|
|
1401
|
-
this.contexts.delete(profile.userDataDir);
|
|
1402
|
-
const closedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1403
|
-
for (const session of this.sessions.values()) {
|
|
1404
|
-
if (session.contextKey === profile.userDataDir && session.status === "open") {
|
|
1405
|
-
session.status = "closed";
|
|
1406
|
-
session.closedAt = closedAt;
|
|
1407
|
-
if (session.timeout) clearTimeout(session.timeout);
|
|
1408
|
-
}
|
|
1409
|
-
}
|
|
1410
|
-
});
|
|
1411
|
-
this.contexts.set(profile.userDataDir, state);
|
|
1412
|
-
return state;
|
|
1413
|
-
}
|
|
1414
|
-
requireOpenSession(sessionId) {
|
|
1415
|
-
const session = this.sessions.get(sessionId);
|
|
1416
|
-
if (!session || session.status !== "open") throw new Error(`Local browser session not found or closed: ${sessionId}`);
|
|
1417
|
-
return session;
|
|
1418
|
-
}
|
|
1419
|
-
touch(session) {
|
|
1420
|
-
session.lastActionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1421
|
-
}
|
|
1422
|
-
hasOpenSessionForContext(contextKey) {
|
|
1423
|
-
return Array.from(this.sessions.values()).some((session) => session.contextKey === contextKey && session.status === "open");
|
|
1424
|
-
}
|
|
1425
|
-
async closeContextIfUnused(contextKey) {
|
|
1426
|
-
if (this.hasOpenSessionForContext(contextKey)) return;
|
|
1427
|
-
const state = this.contexts.get(contextKey);
|
|
1428
|
-
if (!state) return;
|
|
1429
|
-
this.contexts.delete(contextKey);
|
|
1430
|
-
await state.context.close().catch(() => void 0);
|
|
1431
|
-
}
|
|
1432
|
-
async waitForPage(page) {
|
|
1433
|
-
await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
|
|
1434
|
-
await page.waitForTimeout(200).catch(() => void 0);
|
|
1435
|
-
}
|
|
1436
|
-
};
|
|
1437
|
-
|
|
1438
29
|
// src/version.ts
|
|
1439
|
-
var PACKAGE_VERSION = "0.3.
|
|
30
|
+
var PACKAGE_VERSION = "0.3.7";
|
|
1440
31
|
|
|
1441
32
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
1442
33
|
var import_zod = require("zod");
|
|
@@ -1459,32 +50,13 @@ var BrowserOpenInputSchema = {
|
|
|
1459
50
|
save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and local storage back to the named profile when the session is closed. Use this for profile setup or intentional auth refreshes; avoid parallel sessions writing to the same profile."),
|
|
1460
51
|
timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy browser session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
|
|
1461
52
|
};
|
|
1462
|
-
var BrowserProfileListInputSchema = {
|
|
1463
|
-
email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to filter for, for example seo@example.com."),
|
|
1464
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory. Defaults to the current OS Chrome profile root.")
|
|
1465
|
-
};
|
|
1466
53
|
var BrowserProfileOnboardInputSchema = {
|
|
1467
54
|
email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
|
|
1468
55
|
profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
|
|
1469
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional Chrome metadata directory for deriving a profile name only. This is not a local browser setup path."),
|
|
1470
56
|
url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
|
|
1471
57
|
label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
|
|
1472
58
|
timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
|
|
1473
59
|
};
|
|
1474
|
-
var BrowserProfileImportInputSchema = {
|
|
1475
|
-
email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to select from Chrome Local State."),
|
|
1476
|
-
name: import_zod.z.string().optional().describe("Name for the managed MCP Scraper browser profile. Defaults to a slug derived from the selected Chrome account."),
|
|
1477
|
-
chrome_profile_directory: import_zod.z.string().optional().describe("Optional Chrome profile directory to import, for example Default or Profile 1."),
|
|
1478
|
-
user_data_dir: import_zod.z.string().optional().describe("Optional source Chrome user data directory. Defaults to the current OS Chrome profile root."),
|
|
1479
|
-
output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
|
|
1480
|
-
browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path for local browser launches. Defaults to the installed Google Chrome path when found."),
|
|
1481
|
-
overwrite: import_zod.z.boolean().default(false).describe("Overwrite an existing managed profile clone with the same name.")
|
|
1482
|
-
};
|
|
1483
|
-
var BrowserProfileSyncInputSchema = {
|
|
1484
|
-
name: import_zod.z.string().describe("Managed MCP Scraper browser profile name to refresh from its recorded source Chrome profile."),
|
|
1485
|
-
output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
|
|
1486
|
-
browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path to store in the refreshed profile manifest.")
|
|
1487
|
-
};
|
|
1488
60
|
var BrowserSessionInputSchema = {
|
|
1489
61
|
session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. Use only a returned session_id; do not construct one yourself.")
|
|
1490
62
|
};
|
|
@@ -1653,14 +225,6 @@ var BrowserOpenOutputSchema = {
|
|
|
1653
225
|
hint: import_zod.z.string(),
|
|
1654
226
|
raw: BrowserRawObject.optional()
|
|
1655
227
|
};
|
|
1656
|
-
var BrowserProfileListOutputSchema = {
|
|
1657
|
-
ok: import_zod.z.boolean(),
|
|
1658
|
-
tool: import_zod.z.literal("browser_profile_list"),
|
|
1659
|
-
session_id: import_zod.z.null(),
|
|
1660
|
-
profiles: import_zod.z.array(BrowserRawObject),
|
|
1661
|
-
count: import_zod.z.number().int().min(0),
|
|
1662
|
-
hint: import_zod.z.string()
|
|
1663
|
-
};
|
|
1664
228
|
var BrowserProfileOnboardOutputSchema = {
|
|
1665
229
|
ok: import_zod.z.boolean(),
|
|
1666
230
|
tool: import_zod.z.literal("browser_profile_onboard"),
|
|
@@ -1669,27 +233,10 @@ var BrowserProfileOnboardOutputSchema = {
|
|
|
1669
233
|
live_view_url: NullableString,
|
|
1670
234
|
profile: import_zod.z.string(),
|
|
1671
235
|
setup_url: import_zod.z.string(),
|
|
1672
|
-
|
|
236
|
+
account_email: NullableString,
|
|
1673
237
|
next_steps: import_zod.z.array(import_zod.z.string()),
|
|
1674
238
|
raw: BrowserRawObject.optional()
|
|
1675
239
|
};
|
|
1676
|
-
var BrowserProfileImportOutputSchema = {
|
|
1677
|
-
ok: import_zod.z.boolean(),
|
|
1678
|
-
tool: import_zod.z.literal("browser_profile_import"),
|
|
1679
|
-
session_id: import_zod.z.null(),
|
|
1680
|
-
profile: BrowserRawObject,
|
|
1681
|
-
source_profile: BrowserRawObject,
|
|
1682
|
-
next_steps: import_zod.z.array(import_zod.z.string()),
|
|
1683
|
-
hint: import_zod.z.string()
|
|
1684
|
-
};
|
|
1685
|
-
var BrowserProfileSyncOutputSchema = {
|
|
1686
|
-
ok: import_zod.z.boolean(),
|
|
1687
|
-
tool: import_zod.z.literal("browser_profile_sync"),
|
|
1688
|
-
session_id: import_zod.z.null(),
|
|
1689
|
-
profile: BrowserRawObject,
|
|
1690
|
-
source_profile: BrowserRawObject,
|
|
1691
|
-
hint: import_zod.z.string()
|
|
1692
|
-
};
|
|
1693
240
|
var BrowserScreenshotOutputSchema = {
|
|
1694
241
|
...BrowserBaseOutput,
|
|
1695
242
|
tool: import_zod.z.literal("browser_screenshot"),
|
|
@@ -1790,9 +337,9 @@ var BrowserListSessionsOutputSchema = {
|
|
|
1790
337
|
|
|
1791
338
|
// src/mcp/replay-annotator.ts
|
|
1792
339
|
var import_node_child_process = require("child_process");
|
|
1793
|
-
var
|
|
1794
|
-
var
|
|
1795
|
-
var
|
|
340
|
+
var import_promises = require("fs/promises");
|
|
341
|
+
var import_node_os = require("os");
|
|
342
|
+
var import_node_path = require("path");
|
|
1796
343
|
var import_node_util = require("util");
|
|
1797
344
|
var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
|
|
1798
345
|
function finiteNumber(value) {
|
|
@@ -1802,11 +349,11 @@ function clamp(value, min, max) {
|
|
|
1802
349
|
return Math.min(max, Math.max(min, value));
|
|
1803
350
|
}
|
|
1804
351
|
function formatAssTime(seconds) {
|
|
1805
|
-
const
|
|
1806
|
-
const hours = Math.floor(
|
|
1807
|
-
const minutes = Math.floor(
|
|
1808
|
-
const wholeSeconds = Math.floor(
|
|
1809
|
-
const centiseconds = Math.floor((
|
|
352
|
+
const safe = Math.max(0, seconds);
|
|
353
|
+
const hours = Math.floor(safe / 3600);
|
|
354
|
+
const minutes = Math.floor(safe % 3600 / 60);
|
|
355
|
+
const wholeSeconds = Math.floor(safe % 60);
|
|
356
|
+
const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
|
|
1810
357
|
return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
|
|
1811
358
|
}
|
|
1812
359
|
function cssHexToAssColor(color) {
|
|
@@ -2015,10 +562,10 @@ function ffmpegFilterPath(path) {
|
|
|
2015
562
|
async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
2016
563
|
if (!options.annotations.length) throw new Error("annotations must include at least one item");
|
|
2017
564
|
const size = await videoSize(inputFilePath);
|
|
2018
|
-
const tmp = await (0,
|
|
2019
|
-
const assPath = (0,
|
|
565
|
+
const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
|
|
566
|
+
const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
|
|
2020
567
|
try {
|
|
2021
|
-
await (0,
|
|
568
|
+
await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
|
|
2022
569
|
await execFileAsync("ffmpeg", [
|
|
2023
570
|
"-y",
|
|
2024
571
|
"-i",
|
|
@@ -2035,7 +582,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
2035
582
|
"copy",
|
|
2036
583
|
outputFilePath
|
|
2037
584
|
], { maxBuffer: 1024 * 1024 * 20 });
|
|
2038
|
-
const out = await (0,
|
|
585
|
+
const out = await (0, import_promises.stat)(outputFilePath);
|
|
2039
586
|
return {
|
|
2040
587
|
filePath: outputFilePath,
|
|
2041
588
|
bytes: out.size,
|
|
@@ -2044,7 +591,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
2044
591
|
annotationCount: options.annotations.length
|
|
2045
592
|
};
|
|
2046
593
|
} finally {
|
|
2047
|
-
await (0,
|
|
594
|
+
await (0, import_promises.rm)(tmp, { recursive: true, force: true });
|
|
2048
595
|
}
|
|
2049
596
|
}
|
|
2050
597
|
|
|
@@ -2084,8 +631,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
|
|
|
2084
631
|
nextRecommendedTool
|
|
2085
632
|
});
|
|
2086
633
|
}
|
|
2087
|
-
function
|
|
2088
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
634
|
+
function outputBaseDir() {
|
|
635
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
|
|
2089
636
|
}
|
|
2090
637
|
function safeFilePart(value) {
|
|
2091
638
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
|
|
@@ -2094,7 +641,12 @@ function replayFilePath(sessionId, replayId, filename) {
|
|
|
2094
641
|
const requested = filename?.trim();
|
|
2095
642
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2096
643
|
const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
|
|
2097
|
-
return (0,
|
|
644
|
+
return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
|
|
645
|
+
}
|
|
646
|
+
function savedProfileNameFromEmail(email) {
|
|
647
|
+
const value = email?.trim().toLowerCase();
|
|
648
|
+
if (!value) return null;
|
|
649
|
+
return value.replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || null;
|
|
2098
650
|
}
|
|
2099
651
|
function annotatedReplayFilePath(sessionId, replayId, filename) {
|
|
2100
652
|
const requested = filename?.trim();
|
|
@@ -2130,7 +682,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2130
682
|
const baseUrl2 = opts.baseUrl.replace(/\/$/, "");
|
|
2131
683
|
const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
|
|
2132
684
|
const timeoutMs = opts.timeoutMs ?? 9e4;
|
|
2133
|
-
const localBrowser = new LocalBrowserAgentService();
|
|
2134
685
|
async function req(method, path, body) {
|
|
2135
686
|
try {
|
|
2136
687
|
const res = await fetch(`${baseUrl2}${path}`, {
|
|
@@ -2159,8 +710,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2159
710
|
}
|
|
2160
711
|
const bytes = Buffer.from(await res.arrayBuffer());
|
|
2161
712
|
const filePath = replayFilePath(sessionId, replayId, filename);
|
|
2162
|
-
(0,
|
|
2163
|
-
(0,
|
|
713
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
|
|
714
|
+
(0, import_node_fs.writeFileSync)(filePath, bytes);
|
|
2164
715
|
return {
|
|
2165
716
|
ok: true,
|
|
2166
717
|
data: {
|
|
@@ -2182,131 +733,20 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2182
733
|
idempotentHint: false,
|
|
2183
734
|
openWorldHint: true
|
|
2184
735
|
});
|
|
2185
|
-
function selectChromeProfile(matches, email) {
|
|
2186
|
-
const wanted = email?.trim().toLowerCase();
|
|
2187
|
-
if (!wanted) return matches[0] ?? null;
|
|
2188
|
-
return matches.find((profile) => profile.email.toLowerCase() === wanted) ?? matches[0] ?? null;
|
|
2189
|
-
}
|
|
2190
|
-
function localReplayUnsupported(tool, sessionId = null, replayId = null) {
|
|
2191
|
-
return errorResult(tool, {
|
|
2192
|
-
error: "Local browser mode does not support replay recording yet. Use hosted mode for browser_replay_* tools.",
|
|
2193
|
-
mode: "local"
|
|
2194
|
-
}, sessionId, replayId);
|
|
2195
|
-
}
|
|
2196
|
-
server2.registerTool(
|
|
2197
|
-
"browser_profile_list",
|
|
2198
|
-
{
|
|
2199
|
-
title: "Developer: List Local Chrome Profiles",
|
|
2200
|
-
description: "Developer/local-machine helper only. List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2201
|
-
inputSchema: BrowserProfileListInputSchema,
|
|
2202
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
2203
|
-
annotations: annotations("Developer: List Local Chrome Profiles", true)
|
|
2204
|
-
},
|
|
2205
|
-
async (input) => {
|
|
2206
|
-
try {
|
|
2207
|
-
const profiles = await listLocalChromeProfiles(input.user_data_dir, input.email);
|
|
2208
|
-
return structuredResult({
|
|
2209
|
-
ok: true,
|
|
2210
|
-
tool: "browser_profile_list",
|
|
2211
|
-
session_id: null,
|
|
2212
|
-
profiles,
|
|
2213
|
-
count: profiles.length,
|
|
2214
|
-
hint: "For customer auth setup, use browser_profile_onboard so the user signs in through the hosted watch_url. Local import is only for developer/local-machine mode."
|
|
2215
|
-
});
|
|
2216
|
-
} catch (err) {
|
|
2217
|
-
return errorResult("browser_profile_list", { error: err instanceof Error ? err.message : String(err) });
|
|
2218
|
-
}
|
|
2219
|
-
}
|
|
2220
|
-
);
|
|
2221
|
-
server2.registerTool(
|
|
2222
|
-
"browser_profile_import",
|
|
2223
|
-
{
|
|
2224
|
-
title: "Developer: Import Local Chrome Profile",
|
|
2225
|
-
description: "Developer/local-machine helper only. Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2226
|
-
inputSchema: BrowserProfileImportInputSchema,
|
|
2227
|
-
outputSchema: BrowserProfileImportOutputSchema,
|
|
2228
|
-
annotations: { title: "Developer: Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
|
|
2229
|
-
},
|
|
2230
|
-
async (input) => {
|
|
2231
|
-
try {
|
|
2232
|
-
const result = await importChromeProfile({
|
|
2233
|
-
email: input.email,
|
|
2234
|
-
name: input.name,
|
|
2235
|
-
chromeProfileDirectory: input.chrome_profile_directory,
|
|
2236
|
-
sourceUserDataDir: input.user_data_dir,
|
|
2237
|
-
outputDir: input.output_dir,
|
|
2238
|
-
overwrite: input.overwrite,
|
|
2239
|
-
browserExecutablePath: input.browser_executable_path
|
|
2240
|
-
});
|
|
2241
|
-
return structuredResult({
|
|
2242
|
-
ok: true,
|
|
2243
|
-
tool: "browser_profile_import",
|
|
2244
|
-
session_id: null,
|
|
2245
|
-
profile: result.profile,
|
|
2246
|
-
source_profile: result.sourceProfile,
|
|
2247
|
-
next_steps: [
|
|
2248
|
-
`Set MCP_SCRAPER_BROWSER_MODE=local in the MCP server config.`,
|
|
2249
|
-
`Set MCP_SCRAPER_BROWSER_PROFILE=${result.profile.name}.`,
|
|
2250
|
-
"Restart the MCP client so it starts the local browser mode process.",
|
|
2251
|
-
"Call browser_open with the same MCP server; it will open a local Chrome window using the cloned profile."
|
|
2252
|
-
],
|
|
2253
|
-
hint: "Developer-only local clone. For customer auth setup, use browser_profile_onboard at Google, provide the hosted watch_url, then browser_close to save the hosted profile."
|
|
2254
|
-
});
|
|
2255
|
-
} catch (err) {
|
|
2256
|
-
return errorResult("browser_profile_import", { error: err instanceof Error ? err.message : String(err) });
|
|
2257
|
-
}
|
|
2258
|
-
}
|
|
2259
|
-
);
|
|
2260
|
-
server2.registerTool(
|
|
2261
|
-
"browser_profile_sync",
|
|
2262
|
-
{
|
|
2263
|
-
title: "Developer: Sync Local Chrome Profile",
|
|
2264
|
-
description: "Developer/local-machine helper only. Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
|
|
2265
|
-
inputSchema: BrowserProfileSyncInputSchema,
|
|
2266
|
-
outputSchema: BrowserProfileSyncOutputSchema,
|
|
2267
|
-
annotations: { title: "Developer: Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
|
|
2268
|
-
},
|
|
2269
|
-
async (input) => {
|
|
2270
|
-
try {
|
|
2271
|
-
const result = await syncImportedChromeProfile(input.name, {
|
|
2272
|
-
outputDir: input.output_dir,
|
|
2273
|
-
browserExecutablePath: input.browser_executable_path
|
|
2274
|
-
});
|
|
2275
|
-
return structuredResult({
|
|
2276
|
-
ok: true,
|
|
2277
|
-
tool: "browser_profile_sync",
|
|
2278
|
-
session_id: null,
|
|
2279
|
-
profile: result.profile,
|
|
2280
|
-
source_profile: result.sourceProfile,
|
|
2281
|
-
hint: `Developer-only local clone refreshed for ${result.profile.name}. Customer reconnects should use browser_profile_onboard with the same hosted profile name.`
|
|
2282
|
-
});
|
|
2283
|
-
} catch (err) {
|
|
2284
|
-
return errorResult("browser_profile_sync", { error: err instanceof Error ? err.message : String(err) });
|
|
2285
|
-
}
|
|
2286
|
-
}
|
|
2287
|
-
);
|
|
2288
736
|
server2.registerTool(
|
|
2289
737
|
"browser_profile_onboard",
|
|
2290
738
|
{
|
|
2291
739
|
title: "Connect Hosted Browser Profile",
|
|
2292
|
-
description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and
|
|
740
|
+
description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and storage. If a saved profile later shows login controls, run this same setup flow again.",
|
|
2293
741
|
inputSchema: BrowserProfileOnboardInputSchema,
|
|
2294
742
|
outputSchema: BrowserProfileOnboardOutputSchema,
|
|
2295
743
|
annotations: annotations("Connect Hosted Browser Profile")
|
|
2296
744
|
},
|
|
2297
745
|
async (input) => {
|
|
2298
|
-
|
|
2299
|
-
if (input.email) {
|
|
2300
|
-
try {
|
|
2301
|
-
chromeProfile = selectChromeProfile(await listLocalChromeProfiles(input.user_data_dir, input.email), input.email);
|
|
2302
|
-
} catch (err) {
|
|
2303
|
-
return errorResult("browser_profile_onboard", { error: err instanceof Error ? err.message : String(err) });
|
|
2304
|
-
}
|
|
2305
|
-
}
|
|
2306
|
-
const profile = input.profile?.trim() || chromeProfile?.recommendedKernelProfileName || browserServiceProfileName();
|
|
746
|
+
const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email) || browserServiceProfileName();
|
|
2307
747
|
if (!profile) {
|
|
2308
748
|
return errorResult("browser_profile_onboard", {
|
|
2309
|
-
error: "profile is required when
|
|
749
|
+
error: "profile or email is required when BROWSER_AGENT_PROFILE_NAME is not available"
|
|
2310
750
|
});
|
|
2311
751
|
}
|
|
2312
752
|
const setupUrl = input.url ?? "https://google.com/";
|
|
@@ -2329,10 +769,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2329
769
|
live_view_url: session.live_view_url ?? null,
|
|
2330
770
|
profile,
|
|
2331
771
|
setup_url: setupUrl,
|
|
2332
|
-
|
|
772
|
+
account_email: input.email?.trim() || null,
|
|
2333
773
|
next_steps: [
|
|
2334
774
|
"Open the watch_url and let the user click Sign in on Google.",
|
|
2335
|
-
"After the user confirms login is complete, call browser_close with this session_id to persist cookies and
|
|
775
|
+
"After the user confirms login is complete, call browser_close with this session_id to persist cookies and storage to the saved profile.",
|
|
2336
776
|
"If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
|
|
2337
777
|
"Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
|
|
2338
778
|
],
|
|
@@ -2350,23 +790,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2350
790
|
annotations: annotations("Open Browser Session")
|
|
2351
791
|
},
|
|
2352
792
|
async (input) => {
|
|
2353
|
-
if (localBrowserModeEnabled()) {
|
|
2354
|
-
try {
|
|
2355
|
-
const session2 = await localBrowser.open(input);
|
|
2356
|
-
return structuredResult({
|
|
2357
|
-
ok: true,
|
|
2358
|
-
tool: "browser_open",
|
|
2359
|
-
session_id: session2.session_id,
|
|
2360
|
-
watch_url: session2.watch_url,
|
|
2361
|
-
live_view_url: null,
|
|
2362
|
-
url: input.url ?? null,
|
|
2363
|
-
hint: "Local Chrome is open on this machine. Call browser_screenshot to see the page and click by x,y coordinates.",
|
|
2364
|
-
raw: session2
|
|
2365
|
-
});
|
|
2366
|
-
} catch (err) {
|
|
2367
|
-
return errorResult("browser_open", { error: err instanceof Error ? err.message : String(err), mode: "local" });
|
|
2368
|
-
}
|
|
2369
|
-
}
|
|
2370
793
|
const profile = input.profile ?? browserServiceProfileName();
|
|
2371
794
|
const saveProfileChanges = input.save_profile_changes ?? browserServiceProfileSaveChanges();
|
|
2372
795
|
const open = await req("POST", "/agent/sessions", {
|
|
@@ -2403,27 +826,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2403
826
|
annotations: annotations("See Page", true)
|
|
2404
827
|
},
|
|
2405
828
|
async (input) => {
|
|
2406
|
-
if (localBrowserModeEnabled()) {
|
|
2407
|
-
try {
|
|
2408
|
-
const res2 = await localBrowser.screenshot(input.session_id);
|
|
2409
|
-
const content2 = [];
|
|
2410
|
-
if (res2.image_base64) content2.push({ type: "image", data: res2.image_base64, mimeType: res2.mime_type });
|
|
2411
|
-
const structured2 = {
|
|
2412
|
-
ok: true,
|
|
2413
|
-
tool: "browser_screenshot",
|
|
2414
|
-
session_id: input.session_id,
|
|
2415
|
-
url: res2.url ?? null,
|
|
2416
|
-
title: res2.title ?? null,
|
|
2417
|
-
text: res2.text,
|
|
2418
|
-
elements: res2.elements,
|
|
2419
|
-
screenshot: { mime_type: res2.mime_type, inline: true }
|
|
2420
|
-
};
|
|
2421
|
-
content2.push({ type: "text", text: JSON.stringify(structured2) });
|
|
2422
|
-
return { content: content2, structuredContent: structured2 };
|
|
2423
|
-
} catch (err) {
|
|
2424
|
-
return errorResult("browser_screenshot", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2425
|
-
}
|
|
2426
|
-
}
|
|
2427
829
|
const res = await req("POST", `/agent/sessions/${input.session_id}/screenshot`);
|
|
2428
830
|
if (!res.ok) return errorResult("browser_screenshot", res.data, input.session_id);
|
|
2429
831
|
const { image_base64, mime_type, url, title, elements, text } = res.data;
|
|
@@ -2456,23 +858,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2456
858
|
annotations: annotations("Read Page", true)
|
|
2457
859
|
},
|
|
2458
860
|
async (input) => {
|
|
2459
|
-
if (localBrowserModeEnabled()) {
|
|
2460
|
-
try {
|
|
2461
|
-
const res2 = await localBrowser.read(input.session_id);
|
|
2462
|
-
return structuredResult({
|
|
2463
|
-
ok: true,
|
|
2464
|
-
tool: "browser_read",
|
|
2465
|
-
session_id: input.session_id,
|
|
2466
|
-
url: res2.url ?? null,
|
|
2467
|
-
title: res2.title ?? null,
|
|
2468
|
-
text: res2.text,
|
|
2469
|
-
elements: res2.elements,
|
|
2470
|
-
raw: res2
|
|
2471
|
-
});
|
|
2472
|
-
} catch (err) {
|
|
2473
|
-
return errorResult("browser_read", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2474
|
-
}
|
|
2475
|
-
}
|
|
2476
861
|
const res = await req("POST", `/agent/sessions/${input.session_id}/read`);
|
|
2477
862
|
if (!res.ok) return errorResult("browser_read", res.data, input.session_id);
|
|
2478
863
|
return structuredResult({
|
|
@@ -2497,24 +882,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2497
882
|
annotations: annotations("Locate DOM Targets", true)
|
|
2498
883
|
},
|
|
2499
884
|
async (input) => {
|
|
2500
|
-
if (localBrowserModeEnabled()) {
|
|
2501
|
-
try {
|
|
2502
|
-
const res2 = await localBrowser.locate(input.session_id, input.targets);
|
|
2503
|
-
return structuredResult({
|
|
2504
|
-
ok: true,
|
|
2505
|
-
tool: "browser_locate",
|
|
2506
|
-
session_id: input.session_id,
|
|
2507
|
-
url: res2.url ?? null,
|
|
2508
|
-
title: res2.title ?? null,
|
|
2509
|
-
viewport: res2.viewport ?? null,
|
|
2510
|
-
replay: null,
|
|
2511
|
-
targets: Array.isArray(res2.targets) ? res2.targets : [],
|
|
2512
|
-
raw: res2
|
|
2513
|
-
});
|
|
2514
|
-
} catch (err) {
|
|
2515
|
-
return errorResult("browser_locate", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2516
|
-
}
|
|
2517
|
-
}
|
|
2518
885
|
const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: input.targets });
|
|
2519
886
|
if (!res.ok) return errorResult("browser_locate", res.data, input.session_id);
|
|
2520
887
|
return structuredResult({
|
|
@@ -2540,14 +907,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2540
907
|
annotations: annotations("Navigate To URL")
|
|
2541
908
|
},
|
|
2542
909
|
async (input) => {
|
|
2543
|
-
if (localBrowserModeEnabled()) {
|
|
2544
|
-
try {
|
|
2545
|
-
const res2 = await localBrowser.goto(input.session_id, input.url);
|
|
2546
|
-
return actionResult("browser_goto", input.session_id, true, res2, "browser_screenshot");
|
|
2547
|
-
} catch (err) {
|
|
2548
|
-
return errorResult("browser_goto", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2551
910
|
const res = await req("POST", `/agent/sessions/${input.session_id}/goto`, { url: input.url });
|
|
2552
911
|
return actionResult("browser_goto", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2553
912
|
}
|
|
@@ -2562,14 +921,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2562
921
|
annotations: annotations("Click")
|
|
2563
922
|
},
|
|
2564
923
|
async (input) => {
|
|
2565
|
-
if (localBrowserModeEnabled()) {
|
|
2566
|
-
try {
|
|
2567
|
-
const res2 = await localBrowser.click(input.session_id, input);
|
|
2568
|
-
return actionResult("browser_click", input.session_id, true, res2, "browser_screenshot");
|
|
2569
|
-
} catch (err) {
|
|
2570
|
-
return errorResult("browser_click", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2571
|
-
}
|
|
2572
|
-
}
|
|
2573
924
|
const res = await req("POST", `/agent/sessions/${input.session_id}/click`, {
|
|
2574
925
|
x: input.x,
|
|
2575
926
|
y: input.y,
|
|
@@ -2589,14 +940,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2589
940
|
annotations: annotations("Type Text")
|
|
2590
941
|
},
|
|
2591
942
|
async (input) => {
|
|
2592
|
-
if (localBrowserModeEnabled()) {
|
|
2593
|
-
try {
|
|
2594
|
-
const res2 = await localBrowser.type(input.session_id, input);
|
|
2595
|
-
return actionResult("browser_type", input.session_id, true, res2, "browser_screenshot");
|
|
2596
|
-
} catch (err) {
|
|
2597
|
-
return errorResult("browser_type", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2598
|
-
}
|
|
2599
|
-
}
|
|
2600
943
|
const res = await req("POST", `/agent/sessions/${input.session_id}/type`, { text: input.text, delay: input.delay });
|
|
2601
944
|
return actionResult("browser_type", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2602
945
|
}
|
|
@@ -2611,14 +954,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2611
954
|
annotations: annotations("Scroll")
|
|
2612
955
|
},
|
|
2613
956
|
async (input) => {
|
|
2614
|
-
if (localBrowserModeEnabled()) {
|
|
2615
|
-
try {
|
|
2616
|
-
const res2 = await localBrowser.scroll(input.session_id, input);
|
|
2617
|
-
return actionResult("browser_scroll", input.session_id, true, res2, "browser_screenshot");
|
|
2618
|
-
} catch (err) {
|
|
2619
|
-
return errorResult("browser_scroll", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2620
|
-
}
|
|
2621
|
-
}
|
|
2622
957
|
const res = await req("POST", `/agent/sessions/${input.session_id}/scroll`, {
|
|
2623
958
|
delta_y: input.delta_y,
|
|
2624
959
|
delta_x: input.delta_x,
|
|
@@ -2638,14 +973,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2638
973
|
annotations: annotations("Press Keys")
|
|
2639
974
|
},
|
|
2640
975
|
async (input) => {
|
|
2641
|
-
if (localBrowserModeEnabled()) {
|
|
2642
|
-
try {
|
|
2643
|
-
const res2 = await localBrowser.press(input.session_id, input);
|
|
2644
|
-
return actionResult("browser_press", input.session_id, true, res2, "browser_screenshot");
|
|
2645
|
-
} catch (err) {
|
|
2646
|
-
return errorResult("browser_press", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2647
|
-
}
|
|
2648
|
-
}
|
|
2649
976
|
const res = await req("POST", `/agent/sessions/${input.session_id}/press`, { keys: input.keys });
|
|
2650
977
|
return actionResult("browser_press", input.session_id, res.ok, res.data, "browser_screenshot");
|
|
2651
978
|
}
|
|
@@ -2660,7 +987,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2660
987
|
annotations: annotations("Start Recording")
|
|
2661
988
|
},
|
|
2662
989
|
async (input) => {
|
|
2663
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_start", input.session_id);
|
|
2664
990
|
const res = await req("POST", `/agent/sessions/${input.session_id}/replay/start`);
|
|
2665
991
|
if (!res.ok) return errorResult("browser_replay_start", res.data, input.session_id);
|
|
2666
992
|
return structuredResult({
|
|
@@ -2684,7 +1010,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2684
1010
|
annotations: annotations("Stop Recording")
|
|
2685
1011
|
},
|
|
2686
1012
|
async (input) => {
|
|
2687
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_stop", input.session_id, input.replay_id);
|
|
2688
1013
|
const res = await req("POST", `/agent/sessions/${input.session_id}/replay/stop`, { replay_id: input.replay_id });
|
|
2689
1014
|
if (!res.ok) return errorResult("browser_replay_stop", res.data, input.session_id, input.replay_id);
|
|
2690
1015
|
return structuredResult({
|
|
@@ -2708,7 +1033,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2708
1033
|
annotations: annotations("List Replay Videos", true)
|
|
2709
1034
|
},
|
|
2710
1035
|
async (input) => {
|
|
2711
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_list_replays", input.session_id);
|
|
2712
1036
|
const res = await req("GET", `/agent/sessions/${input.session_id}/replays`);
|
|
2713
1037
|
if (!res.ok) return errorResult("browser_list_replays", res.data, input.session_id);
|
|
2714
1038
|
const replays = Array.isArray(res.data?.replays) ? res.data.replays : [];
|
|
@@ -2731,7 +1055,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2731
1055
|
annotations: annotations("Download Replay MP4")
|
|
2732
1056
|
},
|
|
2733
1057
|
async (input) => {
|
|
2734
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_download", input.session_id, input.replay_id);
|
|
2735
1058
|
const res = await downloadReplay(input.session_id, input.replay_id, input.filename);
|
|
2736
1059
|
if (!res.ok) return errorResult("browser_replay_download", res.data, input.session_id, input.replay_id);
|
|
2737
1060
|
return structuredResult({
|
|
@@ -2756,7 +1079,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2756
1079
|
annotations: annotations("Mark Replay Annotation", true)
|
|
2757
1080
|
},
|
|
2758
1081
|
async (input) => {
|
|
2759
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_mark", input.session_id);
|
|
2760
1082
|
const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: [input.target] });
|
|
2761
1083
|
if (!res.ok) return errorResult("browser_replay_mark", res.data, input.session_id);
|
|
2762
1084
|
const target = res.data?.targets?.[0];
|
|
@@ -2807,14 +1129,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2807
1129
|
annotations: annotations("Annotate Replay MP4")
|
|
2808
1130
|
},
|
|
2809
1131
|
async (input) => {
|
|
2810
|
-
if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_annotate", input.session_id, input.replay_id);
|
|
2811
1132
|
const sourceName = input.filename ? `${input.filename}-source` : void 0;
|
|
2812
1133
|
const downloaded = await downloadReplay(input.session_id, input.replay_id, sourceName);
|
|
2813
1134
|
if (!downloaded.ok) return errorResult("browser_replay_annotate", downloaded.data, input.session_id, input.replay_id);
|
|
2814
1135
|
try {
|
|
2815
1136
|
const sourcePath = String(downloaded.data.file_path);
|
|
2816
1137
|
const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
|
|
2817
|
-
(0,
|
|
1138
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
|
|
2818
1139
|
const result = await annotateReplayVideo(sourcePath, outputPath, {
|
|
2819
1140
|
annotations: input.annotations,
|
|
2820
1141
|
sourceWidth: input.source_width,
|
|
@@ -2850,20 +1171,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2850
1171
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
2851
1172
|
},
|
|
2852
1173
|
async (input) => {
|
|
2853
|
-
if (localBrowserModeEnabled()) {
|
|
2854
|
-
try {
|
|
2855
|
-
const res2 = await localBrowser.close(input.session_id);
|
|
2856
|
-
return structuredResult({
|
|
2857
|
-
ok: true,
|
|
2858
|
-
tool: "browser_close",
|
|
2859
|
-
session_id: input.session_id,
|
|
2860
|
-
closed: true,
|
|
2861
|
-
raw: res2
|
|
2862
|
-
});
|
|
2863
|
-
} catch (err) {
|
|
2864
|
-
return errorResult("browser_close", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2865
|
-
}
|
|
2866
|
-
}
|
|
2867
1174
|
const res = await req("DELETE", `/agent/sessions/${input.session_id}`);
|
|
2868
1175
|
if (!res.ok) return errorResult("browser_close", res.data, input.session_id);
|
|
2869
1176
|
return structuredResult({
|
|
@@ -2885,16 +1192,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2885
1192
|
annotations: annotations("List Browser Sessions", true)
|
|
2886
1193
|
},
|
|
2887
1194
|
async (input) => {
|
|
2888
|
-
if (localBrowserModeEnabled()) {
|
|
2889
|
-
const sessions2 = localBrowser.list(input.include_closed);
|
|
2890
|
-
return structuredResult({
|
|
2891
|
-
ok: true,
|
|
2892
|
-
tool: "browser_list_sessions",
|
|
2893
|
-
session_id: null,
|
|
2894
|
-
sessions: sessions2,
|
|
2895
|
-
count: sessions2.length
|
|
2896
|
-
});
|
|
2897
|
-
}
|
|
2898
1195
|
const res = await req("GET", `/agent/sessions${input.include_closed ? "?all=1" : ""}`);
|
|
2899
1196
|
if (!res.ok) return errorResult("browser_list_sessions", res.data);
|
|
2900
1197
|
const sessions = (res.data.sessions ?? []).map((s) => ({ ...s, watch_url: `${consoleBase}/console/${s.session_id}` }));
|
|
@@ -2936,14 +1233,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2936
1233
|
exports: exports2 ?? null,
|
|
2937
1234
|
...result.debug ? { debug: result.debug } : {}
|
|
2938
1235
|
});
|
|
2939
|
-
if (localBrowserModeEnabled()) {
|
|
2940
|
-
try {
|
|
2941
|
-
const { result, exports: exports2 } = await localBrowser.captureFanout(input.session_id, input);
|
|
2942
|
-
return emit(result, exports2);
|
|
2943
|
-
} catch (err) {
|
|
2944
|
-
return errorResult("browser_capture_fanout", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
|
|
2945
|
-
}
|
|
2946
|
-
}
|
|
2947
1236
|
const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
|
|
2948
1237
|
prompt: input.prompt,
|
|
2949
1238
|
wait_ms: input.wait_ms,
|
|
@@ -2960,10 +1249,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2960
1249
|
// bin/browser-agent-stdio-server.ts
|
|
2961
1250
|
function readApiKeyFile() {
|
|
2962
1251
|
const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
|
|
2963
|
-
const paths = [explicitPath, (0,
|
|
1252
|
+
const paths = [explicitPath, (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key")].filter(Boolean);
|
|
2964
1253
|
for (const path of paths) {
|
|
2965
1254
|
try {
|
|
2966
|
-
const value = (0,
|
|
1255
|
+
const value = (0, import_node_fs2.readFileSync)(path, "utf8").trim();
|
|
2967
1256
|
if (value) return value;
|
|
2968
1257
|
} catch {
|
|
2969
1258
|
}
|