mcp-scraper 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -26
- package/dist/bin/api-server.cjs +164 -329
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -4
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +47 -328
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +7 -91
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +18 -25
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2NEVRPK3.js +7 -0
- package/dist/chunk-2NEVRPK3.js.map +1 -0
- package/dist/{chunk-WZT3XKES.js → chunk-2O7RAK6E.js} +19 -26
- package/dist/chunk-2O7RAK6E.js.map +1 -0
- package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
- package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
- package/dist/{chunk-CMK5MMEE.js → chunk-IWNLHCKA.js} +13 -694
- package/dist/chunk-IWNLHCKA.js.map +1 -0
- package/dist/{server-BC4Z2L4P.js → server-HRVD52CX.js} +842 -122
- package/dist/server-HRVD52CX.js.map +1 -0
- package/docs/mcp-tool-craft-lint.generated.md +2 -5
- package/docs/mcp-tool-manifest.generated.json +9 -101
- package/package.json +1 -1
- package/dist/chunk-CMK5MMEE.js.map +0 -1
- package/dist/chunk-DUEW4EOO.js +0 -214
- package/dist/chunk-DUEW4EOO.js.map +0 -1
- package/dist/chunk-GHBU5SGQ.js +0 -7
- package/dist/chunk-GHBU5SGQ.js.map +0 -1
- package/dist/chunk-NXRWFOEZ.js +0 -816
- package/dist/chunk-NXRWFOEZ.js.map +0 -1
- package/dist/chunk-WZT3XKES.js.map +0 -1
- package/dist/server-BC4Z2L4P.js.map +0 -1
package/dist/bin/api-server.cjs
CHANGED
|
@@ -8590,8 +8590,8 @@ var init_site_audit_middleware = __esm({
|
|
|
8590
8590
|
|
|
8591
8591
|
// src/api/site-audit-routes.ts
|
|
8592
8592
|
function isPathInside(root, target) {
|
|
8593
|
-
const
|
|
8594
|
-
return
|
|
8593
|
+
const relative = import_node_path2.default.relative(root, target);
|
|
8594
|
+
return relative === "" || !relative.startsWith("..") && !import_node_path2.default.isAbsolute(relative);
|
|
8595
8595
|
}
|
|
8596
8596
|
var import_node_path2, import_node_os2, import_hono, import_zod11, siteAuditApp;
|
|
8597
8597
|
var init_site_audit_routes = __esm({
|
|
@@ -10318,8 +10318,8 @@ async function fetchViaKernelInnertube(videoId) {
|
|
|
10318
10318
|
const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
10319
10319
|
if (tracks.length === 0) return null;
|
|
10320
10320
|
const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
|
|
10321
|
-
const
|
|
10322
|
-
const xmlResp = await fetch(`${track.baseUrl}${
|
|
10321
|
+
const sep = track.baseUrl.includes("?") ? "&" : "?";
|
|
10322
|
+
const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`);
|
|
10323
10323
|
return xmlResp.ok ? xmlResp.text() : null;
|
|
10324
10324
|
}, videoId);
|
|
10325
10325
|
if (!xml) return null;
|
|
@@ -12978,75 +12978,6 @@ var init_facebook_ad_routes = __esm({
|
|
|
12978
12978
|
}
|
|
12979
12979
|
});
|
|
12980
12980
|
|
|
12981
|
-
// src/lib/chrome-profiles.ts
|
|
12982
|
-
function recommendedKernelProfileName(value) {
|
|
12983
|
-
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
12984
|
-
}
|
|
12985
|
-
var import_promises4, import_node_os3, import_node_path5;
|
|
12986
|
-
var init_chrome_profiles = __esm({
|
|
12987
|
-
"src/lib/chrome-profiles.ts"() {
|
|
12988
|
-
"use strict";
|
|
12989
|
-
import_promises4 = require("fs/promises");
|
|
12990
|
-
import_node_os3 = require("os");
|
|
12991
|
-
import_node_path5 = require("path");
|
|
12992
|
-
}
|
|
12993
|
-
});
|
|
12994
|
-
|
|
12995
|
-
// src/lib/local-browser-profiles.ts
|
|
12996
|
-
function defaultImportedBrowserProfilesDir() {
|
|
12997
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
12998
|
-
}
|
|
12999
|
-
function defaultChromeExecutablePath() {
|
|
13000
|
-
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
13001
|
-
if (envPath) return envPath;
|
|
13002
|
-
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
13003
|
-
(0, import_node_path6.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
13004
|
-
(0, import_node_path6.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
13005
|
-
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
13006
|
-
return candidates.find((candidate) => (0, import_node_fs3.existsSync)(candidate));
|
|
13007
|
-
}
|
|
13008
|
-
function localBrowserModeEnabled() {
|
|
13009
|
-
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
13010
|
-
}
|
|
13011
|
-
function importedProfileName(value) {
|
|
13012
|
-
return recommendedKernelProfileName(value);
|
|
13013
|
-
}
|
|
13014
|
-
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13015
|
-
return (0, import_node_path6.join)(outputDir, importedProfileName(name));
|
|
13016
|
-
}
|
|
13017
|
-
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13018
|
-
return (0, import_node_path6.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
13019
|
-
}
|
|
13020
|
-
async function readExistingManifest(path6) {
|
|
13021
|
-
try {
|
|
13022
|
-
return JSON.parse(await (0, import_promises5.readFile)(path6, "utf8"));
|
|
13023
|
-
} catch {
|
|
13024
|
-
return null;
|
|
13025
|
-
}
|
|
13026
|
-
}
|
|
13027
|
-
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13028
|
-
const profileName = importedProfileName(name);
|
|
13029
|
-
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
13030
|
-
const manifest = await readExistingManifest(manifestPath);
|
|
13031
|
-
if (!manifest) {
|
|
13032
|
-
throw new Error(
|
|
13033
|
-
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
13034
|
-
);
|
|
13035
|
-
}
|
|
13036
|
-
return manifest;
|
|
13037
|
-
}
|
|
13038
|
-
var import_node_fs3, import_promises5, import_node_os4, import_node_path6;
|
|
13039
|
-
var init_local_browser_profiles = __esm({
|
|
13040
|
-
"src/lib/local-browser-profiles.ts"() {
|
|
13041
|
-
"use strict";
|
|
13042
|
-
import_node_fs3 = require("fs");
|
|
13043
|
-
import_promises5 = require("fs/promises");
|
|
13044
|
-
import_node_os4 = require("os");
|
|
13045
|
-
import_node_path6 = require("path");
|
|
13046
|
-
init_chrome_profiles();
|
|
13047
|
-
}
|
|
13048
|
-
});
|
|
13049
|
-
|
|
13050
12981
|
// src/extractor/InstagramContentExtractor.ts
|
|
13051
12982
|
function htmlDecode2(value) {
|
|
13052
12983
|
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
@@ -13412,106 +13343,29 @@ var init_InstagramContentExtractor = __esm({
|
|
|
13412
13343
|
function invalidRequest2(message) {
|
|
13413
13344
|
return { error_code: "invalid_request", message };
|
|
13414
13345
|
}
|
|
13415
|
-
function
|
|
13416
|
-
return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
|
|
13417
|
-
}
|
|
13418
|
-
function configuredProfileName(profile) {
|
|
13419
|
-
const directDir = directProfileDir();
|
|
13420
|
-
return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directDir ? (0, import_node_path7.basename)(directDir) : "default");
|
|
13421
|
-
}
|
|
13422
|
-
function launchExecutablePath(manifestExecutablePath) {
|
|
13423
|
-
const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || manifestExecutablePath || defaultChromeExecutablePath();
|
|
13424
|
-
return configured && (0, import_node_fs4.existsSync)(configured) ? configured : void 0;
|
|
13425
|
-
}
|
|
13426
|
-
function ignoreDefaultArgsFor(executablePath) {
|
|
13427
|
-
if (!executablePath) return void 0;
|
|
13428
|
-
if (process.platform !== "darwin") return void 0;
|
|
13429
|
-
if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
|
|
13430
|
-
return ["--password-store=basic", "--use-mock-keychain"];
|
|
13431
|
-
}
|
|
13432
|
-
async function kernelLaunchOptsResidential2() {
|
|
13433
|
-
let proxyId = browserServiceProxyId();
|
|
13434
|
-
try {
|
|
13435
|
-
const resolution2 = await resolveKernelProxyId({
|
|
13436
|
-
kernelApiKey: browserServiceApiKey(),
|
|
13437
|
-
proxyMode: DEFAULT_PROXY_MODE,
|
|
13438
|
-
configuredKernelProxyId: browserServiceProxyId(),
|
|
13439
|
-
gl: "us"
|
|
13440
|
-
});
|
|
13441
|
-
if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
|
|
13442
|
-
} catch {
|
|
13443
|
-
proxyId = browserServiceProxyId();
|
|
13444
|
-
}
|
|
13346
|
+
async function kernelLaunchOptsDirect() {
|
|
13445
13347
|
return {
|
|
13446
13348
|
headless: true,
|
|
13447
13349
|
kernelApiKey: browserServiceApiKey(),
|
|
13448
|
-
kernelProxyId: proxyId,
|
|
13449
13350
|
viewport: { width: 1280, height: 900 },
|
|
13450
13351
|
locale: "en-US"
|
|
13451
13352
|
};
|
|
13452
13353
|
}
|
|
13453
|
-
async function resolveInstagramLaunch(
|
|
13454
|
-
const requestedMode = input.browserMode ?? "auto";
|
|
13455
|
-
const directDir = directProfileDir();
|
|
13456
|
-
const useLocal = requestedMode === "local" || requestedMode === "auto" && (Boolean(input.browserProfile) || Boolean(directDir) || localBrowserModeEnabled());
|
|
13457
|
-
if (!useLocal) {
|
|
13458
|
-
return {
|
|
13459
|
-
config: await kernelLaunchOptsResidential2(),
|
|
13460
|
-
browser: {
|
|
13461
|
-
mode: "hosted",
|
|
13462
|
-
requestedMode,
|
|
13463
|
-
profileName: null,
|
|
13464
|
-
profileSource: "hosted",
|
|
13465
|
-
profileDirConfigured: false,
|
|
13466
|
-
executablePathConfigured: false
|
|
13467
|
-
}
|
|
13468
|
-
};
|
|
13469
|
-
}
|
|
13470
|
-
const requestedName = configuredProfileName(input.browserProfile);
|
|
13471
|
-
if (directDir && (!input.browserProfile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
|
|
13472
|
-
const executablePath2 = launchExecutablePath(null);
|
|
13473
|
-
return {
|
|
13474
|
-
config: {
|
|
13475
|
-
headless: false,
|
|
13476
|
-
profileDir: directDir,
|
|
13477
|
-
browserExecutablePath: executablePath2,
|
|
13478
|
-
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath2),
|
|
13479
|
-
viewport: { width: 1280, height: 900 },
|
|
13480
|
-
locale: "en-US"
|
|
13481
|
-
},
|
|
13482
|
-
browser: {
|
|
13483
|
-
mode: "local",
|
|
13484
|
-
requestedMode,
|
|
13485
|
-
profileName: requestedName,
|
|
13486
|
-
profileSource: "direct_profile_dir",
|
|
13487
|
-
profileDirConfigured: true,
|
|
13488
|
-
executablePathConfigured: Boolean(executablePath2)
|
|
13489
|
-
}
|
|
13490
|
-
};
|
|
13491
|
-
}
|
|
13492
|
-
const manifest = await loadImportedBrowserProfile(requestedName);
|
|
13493
|
-
const executablePath = launchExecutablePath(manifest.browserExecutablePath);
|
|
13354
|
+
async function resolveInstagramLaunch() {
|
|
13494
13355
|
return {
|
|
13495
|
-
config:
|
|
13496
|
-
headless: false,
|
|
13497
|
-
profileDir: manifest.userDataDir,
|
|
13498
|
-
browserExecutablePath: executablePath,
|
|
13499
|
-
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath),
|
|
13500
|
-
viewport: { width: 1280, height: 900 },
|
|
13501
|
-
locale: "en-US"
|
|
13502
|
-
},
|
|
13356
|
+
config: await kernelLaunchOptsDirect(),
|
|
13503
13357
|
browser: {
|
|
13504
|
-
mode: "
|
|
13505
|
-
requestedMode,
|
|
13506
|
-
profileName:
|
|
13507
|
-
profileSource: "
|
|
13508
|
-
profileDirConfigured:
|
|
13509
|
-
executablePathConfigured:
|
|
13358
|
+
mode: "hosted",
|
|
13359
|
+
requestedMode: "hosted",
|
|
13360
|
+
profileName: null,
|
|
13361
|
+
profileSource: "hosted",
|
|
13362
|
+
profileDirConfigured: false,
|
|
13363
|
+
executablePathConfigured: false
|
|
13510
13364
|
}
|
|
13511
13365
|
};
|
|
13512
13366
|
}
|
|
13513
13367
|
function outputBaseDir() {
|
|
13514
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
13368
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path5.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
|
|
13515
13369
|
}
|
|
13516
13370
|
function safeFilePart(input) {
|
|
13517
13371
|
return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
|
|
@@ -13519,8 +13373,8 @@ function safeFilePart(input) {
|
|
|
13519
13373
|
function mediaOutputDir(shortcode, sourceUrl) {
|
|
13520
13374
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
13521
13375
|
const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
13522
|
-
const dir = (0,
|
|
13523
|
-
(0,
|
|
13376
|
+
const dir = (0, import_node_path5.join)(outputBaseDir(), "instagram", `${stamp}-${slug}`);
|
|
13377
|
+
(0, import_node_fs3.mkdirSync)(dir, { recursive: true });
|
|
13524
13378
|
return dir;
|
|
13525
13379
|
}
|
|
13526
13380
|
async function downloadToFile(url, destPath, referer) {
|
|
@@ -13533,10 +13387,10 @@ async function downloadToFile(url, destPath, referer) {
|
|
|
13533
13387
|
});
|
|
13534
13388
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
13535
13389
|
if (!res.body) throw new Error("Empty response body");
|
|
13536
|
-
await (0,
|
|
13390
|
+
await (0, import_promises4.pipeline)(import_node_stream2.Readable.fromWeb(res.body), (0, import_node_fs3.createWriteStream)(destPath));
|
|
13537
13391
|
return {
|
|
13538
13392
|
savedPath: destPath,
|
|
13539
|
-
sizeBytes: (0,
|
|
13393
|
+
sizeBytes: (0, import_node_fs3.statSync)(destPath).size,
|
|
13540
13394
|
mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
|
|
13541
13395
|
};
|
|
13542
13396
|
}
|
|
@@ -13571,43 +13425,34 @@ function trackFilename(track, index, shortcode) {
|
|
|
13571
13425
|
const bitrate = track.bitrate ? `-${track.bitrate}` : "";
|
|
13572
13426
|
return `${label}-${type}-${index}${bitrate}.mp4`;
|
|
13573
13427
|
}
|
|
13574
|
-
var import_hono5, import_zod17,
|
|
13428
|
+
var import_hono5, import_zod17, import_node_fs3, import_node_os3, import_node_path5, import_node_stream2, import_promises4, import_node_child_process2, InstagramProfileContentBodySchema, InstagramMediaTypeSchema, InstagramMediaDownloadBodySchema, instagramApp;
|
|
13575
13429
|
var init_instagram_routes = __esm({
|
|
13576
13430
|
"src/api/instagram-routes.ts"() {
|
|
13577
13431
|
"use strict";
|
|
13578
13432
|
import_hono5 = require("hono");
|
|
13579
13433
|
import_zod17 = require("zod");
|
|
13580
|
-
|
|
13581
|
-
|
|
13582
|
-
|
|
13434
|
+
import_node_fs3 = require("fs");
|
|
13435
|
+
import_node_os3 = require("os");
|
|
13436
|
+
import_node_path5 = require("path");
|
|
13583
13437
|
import_node_stream2 = require("stream");
|
|
13584
|
-
|
|
13438
|
+
import_promises4 = require("stream/promises");
|
|
13585
13439
|
import_node_child_process2 = require("child_process");
|
|
13586
13440
|
init_browser_service_env();
|
|
13587
13441
|
init_BrowserDriver();
|
|
13588
|
-
init_kernel_proxy_resolver();
|
|
13589
|
-
init_schemas3();
|
|
13590
13442
|
init_api_auth();
|
|
13591
13443
|
init_url_utils();
|
|
13592
13444
|
init_db();
|
|
13593
13445
|
init_rates();
|
|
13594
13446
|
init_concurrency_gates();
|
|
13595
|
-
init_local_browser_profiles();
|
|
13596
13447
|
init_InstagramContentExtractor();
|
|
13597
13448
|
init_media_transcription();
|
|
13598
|
-
InstagramBrowserModeSchema = import_zod17.z.enum(["auto", "hosted", "local"]);
|
|
13599
|
-
InstagramBrowserOptionsSchema = {
|
|
13600
|
-
browserMode: InstagramBrowserModeSchema.default("auto"),
|
|
13601
|
-
browserProfile: import_zod17.z.string().trim().min(1).optional()
|
|
13602
|
-
};
|
|
13603
13449
|
InstagramProfileContentBodySchema = import_zod17.z.object({
|
|
13604
13450
|
handle: import_zod17.z.string().trim().optional(),
|
|
13605
13451
|
url: import_zod17.z.string().trim().optional(),
|
|
13606
13452
|
maxItems: import_zod17.z.number().int().min(1).max(500).default(50),
|
|
13607
13453
|
maxScrolls: import_zod17.z.number().int().min(0).max(100).default(10),
|
|
13608
13454
|
scrollDelayMs: import_zod17.z.number().int().min(250).max(5e3).default(1200),
|
|
13609
|
-
stableScrollLimit: import_zod17.z.number().int().min(1).max(10).default(4)
|
|
13610
|
-
...InstagramBrowserOptionsSchema
|
|
13455
|
+
stableScrollLimit: import_zod17.z.number().int().min(1).max(10).default(4)
|
|
13611
13456
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
13612
13457
|
message: "handle or url is required"
|
|
13613
13458
|
});
|
|
@@ -13618,8 +13463,7 @@ var init_instagram_routes = __esm({
|
|
|
13618
13463
|
downloadMedia: import_zod17.z.boolean().default(true),
|
|
13619
13464
|
downloadAllTracks: import_zod17.z.boolean().default(false),
|
|
13620
13465
|
includeTranscript: import_zod17.z.boolean().default(false),
|
|
13621
|
-
mux: import_zod17.z.boolean().default(true)
|
|
13622
|
-
...InstagramBrowserOptionsSchema
|
|
13466
|
+
mux: import_zod17.z.boolean().default(true)
|
|
13623
13467
|
});
|
|
13624
13468
|
instagramApp = new import_hono5.Hono();
|
|
13625
13469
|
instagramApp.post("/profile-content", createApiKeyAuth(), async (c) => {
|
|
@@ -13645,7 +13489,7 @@ var init_instagram_routes = __esm({
|
|
|
13645
13489
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
|
|
13646
13490
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
|
|
13647
13491
|
debited = true;
|
|
13648
|
-
const launch = await resolveInstagramLaunch(
|
|
13492
|
+
const launch = await resolveInstagramLaunch();
|
|
13649
13493
|
await driver.launch(launch.config);
|
|
13650
13494
|
await driver.navigateTo(target.profileUrl);
|
|
13651
13495
|
const page = driver.getPage();
|
|
@@ -13692,7 +13536,7 @@ var init_instagram_routes = __esm({
|
|
|
13692
13536
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
|
|
13693
13537
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
|
|
13694
13538
|
mediaDebited = true;
|
|
13695
|
-
const launch = await resolveInstagramLaunch(
|
|
13539
|
+
const launch = await resolveInstagramLaunch();
|
|
13696
13540
|
await driver.launch(launch.config);
|
|
13697
13541
|
const page = driver.getPage();
|
|
13698
13542
|
const capturedMediaUrls = [];
|
|
@@ -13711,16 +13555,16 @@ var init_instagram_routes = __esm({
|
|
|
13711
13555
|
let outputDir = null;
|
|
13712
13556
|
if (body.downloadMedia) {
|
|
13713
13557
|
outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
|
|
13714
|
-
const textPath = (0,
|
|
13715
|
-
(0,
|
|
13558
|
+
const textPath = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
|
|
13559
|
+
(0, import_node_fs3.writeFileSync)(textPath, [
|
|
13716
13560
|
`URL: ${extraction.pageUrl}`,
|
|
13717
13561
|
extraction.caption ? `Caption: ${extraction.caption}` : "",
|
|
13718
13562
|
"",
|
|
13719
13563
|
extraction.bodyText
|
|
13720
13564
|
].filter(Boolean).join("\n"), "utf8");
|
|
13721
|
-
downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0,
|
|
13565
|
+
downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0, import_node_fs3.statSync)(textPath).size, mimeType: "text/plain", error: null });
|
|
13722
13566
|
if (mediaTypes.has("image") && extraction.imageUrl) {
|
|
13723
|
-
const imagePathBase = (0,
|
|
13567
|
+
const imagePathBase = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
|
|
13724
13568
|
try {
|
|
13725
13569
|
const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
|
|
13726
13570
|
const ext = extFromMime(downloaded.mimeType, "jpg");
|
|
@@ -13729,7 +13573,7 @@ var init_instagram_routes = __esm({
|
|
|
13729
13573
|
const { renameSync } = await import("fs");
|
|
13730
13574
|
renameSync(downloaded.savedPath, finalPath);
|
|
13731
13575
|
}
|
|
13732
|
-
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0,
|
|
13576
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0, import_node_fs3.statSync)(finalPath).size, mimeType: downloaded.mimeType, error: null });
|
|
13733
13577
|
} catch (err) {
|
|
13734
13578
|
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
13735
13579
|
}
|
|
@@ -13740,7 +13584,7 @@ var init_instagram_routes = __esm({
|
|
|
13740
13584
|
if (track.streamType === "video" && !mediaTypes.has("video")) continue;
|
|
13741
13585
|
if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
|
|
13742
13586
|
const kind = track.streamType === "audio" ? "audio" : "video";
|
|
13743
|
-
const target = (0,
|
|
13587
|
+
const target = (0, import_node_path5.join)(outputDir, trackFilename(track, index, extraction.shortcode));
|
|
13744
13588
|
try {
|
|
13745
13589
|
const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
|
|
13746
13590
|
downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
|
|
@@ -13750,10 +13594,10 @@ var init_instagram_routes = __esm({
|
|
|
13750
13594
|
}
|
|
13751
13595
|
}
|
|
13752
13596
|
if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
|
|
13753
|
-
const outPath = (0,
|
|
13597
|
+
const outPath = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
|
|
13754
13598
|
const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
|
|
13755
13599
|
if (muxed.ok) {
|
|
13756
|
-
downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0,
|
|
13600
|
+
downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0, import_node_fs3.statSync)(outPath).size, mimeType: "video/mp4", error: null });
|
|
13757
13601
|
} else {
|
|
13758
13602
|
warnings.push(`Mux skipped/failed: ${muxed.error}`);
|
|
13759
13603
|
}
|
|
@@ -14925,16 +14769,16 @@ function reportTitle(full) {
|
|
|
14925
14769
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
14926
14770
|
}
|
|
14927
14771
|
function outputBaseDir2() {
|
|
14928
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
14772
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
|
|
14929
14773
|
}
|
|
14930
14774
|
function saveFullReport(full) {
|
|
14931
14775
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14932
14776
|
const outDir = outputBaseDir2();
|
|
14933
14777
|
try {
|
|
14934
|
-
(0,
|
|
14778
|
+
(0, import_node_fs4.mkdirSync)(outDir, { recursive: true });
|
|
14935
14779
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14936
|
-
const file = (0,
|
|
14937
|
-
(0,
|
|
14780
|
+
const file = (0, import_node_path6.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
14781
|
+
(0, import_node_fs4.writeFileSync)(file, full, "utf8");
|
|
14938
14782
|
return file;
|
|
14939
14783
|
} catch {
|
|
14940
14784
|
return null;
|
|
@@ -14943,12 +14787,12 @@ function saveFullReport(full) {
|
|
|
14943
14787
|
function persistScreenshotLocally(base64, url) {
|
|
14944
14788
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14945
14789
|
try {
|
|
14946
|
-
const dir = (0,
|
|
14947
|
-
(0,
|
|
14790
|
+
const dir = (0, import_node_path6.join)(outputBaseDir2(), "screenshots");
|
|
14791
|
+
(0, import_node_fs4.mkdirSync)(dir, { recursive: true });
|
|
14948
14792
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14949
14793
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
14950
|
-
const filePath = (0,
|
|
14951
|
-
(0,
|
|
14794
|
+
const filePath = (0, import_node_path6.join)(dir, `${stamp}-${slug}.png`);
|
|
14795
|
+
(0, import_node_fs4.writeFileSync)(filePath, Buffer.from(base64, "base64"));
|
|
14952
14796
|
return filePath;
|
|
14953
14797
|
} catch {
|
|
14954
14798
|
return null;
|
|
@@ -16241,16 +16085,13 @@ ${chunkRows}` : "",
|
|
|
16241
16085
|
}
|
|
16242
16086
|
function structuredInstagramBrowser(raw) {
|
|
16243
16087
|
const browser = raw && typeof raw === "object" ? raw : {};
|
|
16244
|
-
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
16245
|
-
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
16246
|
-
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
16247
16088
|
return {
|
|
16248
|
-
mode,
|
|
16249
|
-
requestedMode,
|
|
16089
|
+
mode: "hosted",
|
|
16090
|
+
requestedMode: "hosted",
|
|
16250
16091
|
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
16251
|
-
profileSource,
|
|
16252
|
-
profileDirConfigured:
|
|
16253
|
-
executablePathConfigured:
|
|
16092
|
+
profileSource: "hosted",
|
|
16093
|
+
profileDirConfigured: false,
|
|
16094
|
+
executablePathConfigured: false
|
|
16254
16095
|
};
|
|
16255
16096
|
}
|
|
16256
16097
|
function structuredInstagramPagination(raw, input) {
|
|
@@ -16292,7 +16133,7 @@ function formatInstagramProfileContent(raw, input) {
|
|
|
16292
16133
|
const itemRows = items.slice(0, 100).map(
|
|
16293
16134
|
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
16294
16135
|
).join("\n");
|
|
16295
|
-
const browserLabel =
|
|
16136
|
+
const browserLabel = "hosted browser";
|
|
16296
16137
|
const full = [
|
|
16297
16138
|
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
16298
16139
|
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
@@ -16367,7 +16208,7 @@ function formatInstagramMediaDownload(raw, input) {
|
|
|
16367
16208
|
const transcriptText = transcript?.text ?? "";
|
|
16368
16209
|
const chunks = transcript?.chunks ?? [];
|
|
16369
16210
|
const browser = structuredInstagramBrowser(d.browser);
|
|
16370
|
-
const browserLabel =
|
|
16211
|
+
const browserLabel = "hosted browser";
|
|
16371
16212
|
const downloadRows = downloads.map((download, i) => {
|
|
16372
16213
|
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
16373
16214
|
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
@@ -16539,13 +16380,13 @@ ${rows}` : ""
|
|
|
16539
16380
|
}
|
|
16540
16381
|
};
|
|
16541
16382
|
}
|
|
16542
|
-
var
|
|
16383
|
+
var import_node_fs4, import_node_os4, import_node_path6, reportSavingEnabled;
|
|
16543
16384
|
var init_mcp_response_formatter = __esm({
|
|
16544
16385
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
16545
16386
|
"use strict";
|
|
16546
|
-
|
|
16547
|
-
|
|
16548
|
-
|
|
16387
|
+
import_node_fs4 = require("fs");
|
|
16388
|
+
import_node_os4 = require("os");
|
|
16389
|
+
import_node_path6 = require("path");
|
|
16549
16390
|
init_errors();
|
|
16550
16391
|
init_workflow_catalog();
|
|
16551
16392
|
reportSavingEnabled = true;
|
|
@@ -16652,7 +16493,7 @@ function localLocationFileAllowed() {
|
|
|
16652
16493
|
async function existingPath(value) {
|
|
16653
16494
|
const trimmed = value?.trim();
|
|
16654
16495
|
if (!trimmed) return null;
|
|
16655
|
-
await (0,
|
|
16496
|
+
await (0, import_promises5.access)(trimmed);
|
|
16656
16497
|
return trimmed;
|
|
16657
16498
|
}
|
|
16658
16499
|
async function resolveUsZipsPath(requestedPath) {
|
|
@@ -16670,7 +16511,7 @@ async function loadZipGroups(stateAbbr, requestedPath, warnings) {
|
|
|
16670
16511
|
}
|
|
16671
16512
|
const path6 = await resolveUsZipsPath(requestedPath);
|
|
16672
16513
|
if (!path6) return { path: null, groups: /* @__PURE__ */ new Map() };
|
|
16673
|
-
const records = csvRecords(await (0,
|
|
16514
|
+
const records = csvRecords(await (0, import_promises5.readFile)(path6, "utf8"));
|
|
16674
16515
|
const groups = /* @__PURE__ */ new Map();
|
|
16675
16516
|
for (const record of records) {
|
|
16676
16517
|
const state = (record.state_abbr ?? record.state ?? "").trim().toUpperCase();
|
|
@@ -16721,11 +16562,11 @@ async function resolveDirectoryMarkets(options) {
|
|
|
16721
16562
|
}
|
|
16722
16563
|
return { markets, censusSourceUrl: sourceUrl, usZipsSourcePath: zipData.path, warnings };
|
|
16723
16564
|
}
|
|
16724
|
-
var
|
|
16565
|
+
var import_promises5, POPULATION_YEARS, STATE_META, STATE_BY_NAME;
|
|
16725
16566
|
var init_location_db = __esm({
|
|
16726
16567
|
"src/directory/location-db.ts"() {
|
|
16727
16568
|
"use strict";
|
|
16728
|
-
|
|
16569
|
+
import_promises5 = require("fs/promises");
|
|
16729
16570
|
init_csv();
|
|
16730
16571
|
POPULATION_YEARS = [2020, 2021, 2022, 2023, 2024, 2025];
|
|
16731
16572
|
STATE_META = {
|
|
@@ -16941,11 +16782,11 @@ function csvRowsFor(result) {
|
|
|
16941
16782
|
return rows;
|
|
16942
16783
|
}
|
|
16943
16784
|
async function saveDirectoryCsv(result) {
|
|
16944
|
-
const outDir = (0,
|
|
16945
|
-
await (0,
|
|
16785
|
+
const outDir = (0, import_node_path7.join)(outputBaseDir2(), "directory-workflows");
|
|
16786
|
+
await (0, import_promises6.mkdir)(outDir, { recursive: true });
|
|
16946
16787
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
16947
16788
|
const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
16948
|
-
const path6 = (0,
|
|
16789
|
+
const path6 = (0, import_node_path7.join)(outDir, `${stamp}-${slug}-directory-workflow.csv`);
|
|
16949
16790
|
const headers = [
|
|
16950
16791
|
"source_query",
|
|
16951
16792
|
"source_location",
|
|
@@ -16977,7 +16818,7 @@ async function saveDirectoryCsv(result) {
|
|
|
16977
16818
|
"extracted_at",
|
|
16978
16819
|
"duration_ms"
|
|
16979
16820
|
];
|
|
16980
|
-
await (0,
|
|
16821
|
+
await (0, import_promises6.writeFile)(path6, rowsToCsv(headers, csvRowsFor(result)), "utf8");
|
|
16981
16822
|
return path6;
|
|
16982
16823
|
}
|
|
16983
16824
|
async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
@@ -17003,12 +16844,12 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
17003
16844
|
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
17004
16845
|
return { ...base, csvPath };
|
|
17005
16846
|
}
|
|
17006
|
-
var
|
|
16847
|
+
var import_promises6, import_node_path7, import_zod19, DirectoryWorkflowOptionsSchema;
|
|
17007
16848
|
var init_directory_workflow = __esm({
|
|
17008
16849
|
"src/directory/directory-workflow.ts"() {
|
|
17009
16850
|
"use strict";
|
|
17010
|
-
|
|
17011
|
-
|
|
16851
|
+
import_promises6 = require("fs/promises");
|
|
16852
|
+
import_node_path7 = require("path");
|
|
17012
16853
|
import_zod19 = require("zod");
|
|
17013
16854
|
init_mcp_response_formatter();
|
|
17014
16855
|
init_browser_service_env();
|
|
@@ -17135,7 +16976,7 @@ var init_slugify = __esm({
|
|
|
17135
16976
|
|
|
17136
16977
|
// src/workflows/artifact-writer.ts
|
|
17137
16978
|
function workflowOutputBaseDir(outputDir) {
|
|
17138
|
-
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
16979
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path8.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
|
|
17139
16980
|
}
|
|
17140
16981
|
function timestamp() {
|
|
17141
16982
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -17144,13 +16985,13 @@ function safeSlug(value) {
|
|
|
17144
16985
|
return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
|
|
17145
16986
|
}
|
|
17146
16987
|
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
17147
|
-
return (0,
|
|
16988
|
+
return (0, import_node_path8.join)(baseDir, "workflows", "index.json");
|
|
17148
16989
|
}
|
|
17149
16990
|
async function readIndex(baseDir) {
|
|
17150
16991
|
const path6 = indexPath(baseDir);
|
|
17151
|
-
if (!(0,
|
|
16992
|
+
if (!(0, import_node_fs5.existsSync)(path6)) return { runs: [] };
|
|
17152
16993
|
try {
|
|
17153
|
-
return JSON.parse(await (0,
|
|
16994
|
+
return JSON.parse(await (0, import_promises7.readFile)(path6, "utf8"));
|
|
17154
16995
|
} catch {
|
|
17155
16996
|
return { runs: [] };
|
|
17156
16997
|
}
|
|
@@ -17170,17 +17011,17 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
|
17170
17011
|
summary: `${manifest.title} (${manifest.status})`
|
|
17171
17012
|
};
|
|
17172
17013
|
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
17173
|
-
await (0,
|
|
17174
|
-
await (0,
|
|
17014
|
+
await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
|
|
17015
|
+
await (0, import_promises7.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
|
|
17175
17016
|
}
|
|
17176
|
-
var
|
|
17017
|
+
var import_promises7, import_node_fs5, import_node_os5, import_node_path8, import_node_child_process3, ArtifactWriter;
|
|
17177
17018
|
var init_artifact_writer = __esm({
|
|
17178
17019
|
"src/workflows/artifact-writer.ts"() {
|
|
17179
17020
|
"use strict";
|
|
17180
|
-
|
|
17181
|
-
|
|
17182
|
-
|
|
17183
|
-
|
|
17021
|
+
import_promises7 = require("fs/promises");
|
|
17022
|
+
import_node_fs5 = require("fs");
|
|
17023
|
+
import_node_os5 = require("os");
|
|
17024
|
+
import_node_path8 = require("path");
|
|
17184
17025
|
import_node_child_process3 = require("child_process");
|
|
17185
17026
|
init_csv();
|
|
17186
17027
|
init_slugify();
|
|
@@ -17207,31 +17048,31 @@ var init_artifact_writer = __esm({
|
|
|
17207
17048
|
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
17208
17049
|
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
17209
17050
|
const baseDir = workflowOutputBaseDir(outputDir);
|
|
17210
|
-
const runDir = (0,
|
|
17211
|
-
await (0,
|
|
17051
|
+
const runDir = (0, import_node_path8.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
17052
|
+
await (0, import_promises7.mkdir)(runDir, { recursive: true });
|
|
17212
17053
|
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
17213
17054
|
}
|
|
17214
17055
|
async remember(kind, label, path6, rows) {
|
|
17215
|
-
const size = await (0,
|
|
17056
|
+
const size = await (0, import_promises7.stat)(path6).then((s) => s.size).catch(() => void 0);
|
|
17216
17057
|
this.artifacts.push({ kind, label, path: path6, bytes: size, rows });
|
|
17217
17058
|
return path6;
|
|
17218
17059
|
}
|
|
17219
17060
|
async writeJson(label, relativePath, data) {
|
|
17220
|
-
const path6 = (0,
|
|
17221
|
-
await (0,
|
|
17222
|
-
await (0,
|
|
17061
|
+
const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
|
|
17062
|
+
await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
|
|
17063
|
+
await (0, import_promises7.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
|
|
17223
17064
|
return this.remember("json", label, path6);
|
|
17224
17065
|
}
|
|
17225
17066
|
async writeText(label, relativePath, text, kind = "markdown") {
|
|
17226
|
-
const path6 = (0,
|
|
17227
|
-
await (0,
|
|
17228
|
-
await (0,
|
|
17067
|
+
const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
|
|
17068
|
+
await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
|
|
17069
|
+
await (0, import_promises7.writeFile)(path6, text, "utf8");
|
|
17229
17070
|
return this.remember(kind, label, path6);
|
|
17230
17071
|
}
|
|
17231
17072
|
async writeCsv(label, relativePath, headers, rows) {
|
|
17232
|
-
const path6 = (0,
|
|
17233
|
-
await (0,
|
|
17234
|
-
await (0,
|
|
17073
|
+
const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
|
|
17074
|
+
await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
|
|
17075
|
+
await (0, import_promises7.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
|
|
17235
17076
|
return this.remember("csv", label, path6, rows.length);
|
|
17236
17077
|
}
|
|
17237
17078
|
async writeHtml(label, relativePath, html) {
|
|
@@ -17251,8 +17092,8 @@ var init_artifact_writer = __esm({
|
|
|
17251
17092
|
errors,
|
|
17252
17093
|
counts
|
|
17253
17094
|
};
|
|
17254
|
-
const path6 = (0,
|
|
17255
|
-
await (0,
|
|
17095
|
+
const path6 = (0, import_node_path8.join)(this.runDir, "manifest.json");
|
|
17096
|
+
await (0, import_promises7.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
|
|
17256
17097
|
await updateWorkflowIndex(manifest, path6, this.baseDir);
|
|
17257
17098
|
return path6;
|
|
17258
17099
|
}
|
|
@@ -18731,7 +18572,7 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
18731
18572
|
const writtenArtifacts = await Promise.all(
|
|
18732
18573
|
artifacts.artifacts.map(async (artifact) => ({
|
|
18733
18574
|
...artifact,
|
|
18734
|
-
content: await (0,
|
|
18575
|
+
content: await (0, import_promises8.readFile)(artifact.path, "utf8").catch(() => "")
|
|
18735
18576
|
}))
|
|
18736
18577
|
);
|
|
18737
18578
|
return {
|
|
@@ -18750,11 +18591,11 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
18750
18591
|
artifacts: writtenArtifacts
|
|
18751
18592
|
};
|
|
18752
18593
|
}
|
|
18753
|
-
var
|
|
18594
|
+
var import_promises8, import_zod24, DEFINITIONS;
|
|
18754
18595
|
var init_registry2 = __esm({
|
|
18755
18596
|
"src/workflows/registry.ts"() {
|
|
18756
18597
|
"use strict";
|
|
18757
|
-
|
|
18598
|
+
import_promises8 = require("fs/promises");
|
|
18758
18599
|
import_zod24 = require("zod");
|
|
18759
18600
|
init_artifact_writer();
|
|
18760
18601
|
init_http_client2();
|
|
@@ -18823,7 +18664,7 @@ async function readManifestFromSummary(summary) {
|
|
|
18823
18664
|
if (!summary.reportPath) return null;
|
|
18824
18665
|
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
18825
18666
|
try {
|
|
18826
|
-
return JSON.parse(await (0,
|
|
18667
|
+
return JSON.parse(await (0, import_promises9.readFile)(manifestPath, "utf8"));
|
|
18827
18668
|
} catch {
|
|
18828
18669
|
return null;
|
|
18829
18670
|
}
|
|
@@ -19038,12 +18879,12 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
19038
18879
|
}
|
|
19039
18880
|
return { dispatched: results.length, results };
|
|
19040
18881
|
}
|
|
19041
|
-
var import_node_crypto3,
|
|
18882
|
+
var import_node_crypto3, import_promises9, import_hono8, import_zod25, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
19042
18883
|
var init_workflow_routes = __esm({
|
|
19043
18884
|
"src/api/workflow-routes.ts"() {
|
|
19044
18885
|
"use strict";
|
|
19045
18886
|
import_node_crypto3 = require("crypto");
|
|
19046
|
-
|
|
18887
|
+
import_promises9 = require("fs/promises");
|
|
19047
18888
|
import_hono8 = require("hono");
|
|
19048
18889
|
import_zod25 = require("zod");
|
|
19049
18890
|
init_artifact_writer();
|
|
@@ -19213,7 +19054,7 @@ var init_workflow_routes = __esm({
|
|
|
19213
19054
|
return new Response(artifact.content, { headers });
|
|
19214
19055
|
}
|
|
19215
19056
|
try {
|
|
19216
|
-
const content = await (0,
|
|
19057
|
+
const content = await (0, import_promises9.readFile)(artifact.path);
|
|
19217
19058
|
return new Response(content, { headers });
|
|
19218
19059
|
} catch {
|
|
19219
19060
|
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
@@ -20872,52 +20713,52 @@ var init_PAAExtractor = __esm({
|
|
|
20872
20713
|
});
|
|
20873
20714
|
|
|
20874
20715
|
// src/output/OutputSerializer.ts
|
|
20875
|
-
var
|
|
20716
|
+
var import_node_fs6, import_node_path9, import_papaparse2, OutputSerializer;
|
|
20876
20717
|
var init_OutputSerializer = __esm({
|
|
20877
20718
|
"src/output/OutputSerializer.ts"() {
|
|
20878
20719
|
"use strict";
|
|
20879
|
-
|
|
20880
|
-
|
|
20720
|
+
import_node_fs6 = require("fs");
|
|
20721
|
+
import_node_path9 = __toESM(require("path"), 1);
|
|
20881
20722
|
import_papaparse2 = __toESM(require("papaparse"), 1);
|
|
20882
20723
|
OutputSerializer = class {
|
|
20883
20724
|
async writeJSON(result, outputDir) {
|
|
20884
|
-
await
|
|
20725
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20885
20726
|
const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20886
20727
|
const filename = `${slug}-${Date.now()}.json`;
|
|
20887
|
-
const fullPath =
|
|
20888
|
-
await
|
|
20728
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20729
|
+
await import_node_fs6.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
|
|
20889
20730
|
return fullPath;
|
|
20890
20731
|
}
|
|
20891
20732
|
async writeCSV(rows, outputDir) {
|
|
20892
|
-
await
|
|
20733
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20893
20734
|
const seedRaw = rows[0]?.seed_query ?? "paa";
|
|
20894
20735
|
const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20895
20736
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
20896
20737
|
const filename = `${slug}-${Date.now()}.csv`;
|
|
20897
|
-
const fullPath =
|
|
20898
|
-
await
|
|
20738
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20739
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20899
20740
|
return fullPath;
|
|
20900
20741
|
}
|
|
20901
20742
|
async writeVideoCSV(videos, seed, outputDir) {
|
|
20902
|
-
await
|
|
20743
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20903
20744
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20904
20745
|
const csv = import_papaparse2.default.unparse(videos, { header: true });
|
|
20905
20746
|
const filename = `${slug}-videos-${Date.now()}.csv`;
|
|
20906
|
-
const fullPath =
|
|
20907
|
-
await
|
|
20747
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20748
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20908
20749
|
return fullPath;
|
|
20909
20750
|
}
|
|
20910
20751
|
async writeForumCSV(forums, seed, outputDir) {
|
|
20911
|
-
await
|
|
20752
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20912
20753
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20913
20754
|
const csv = import_papaparse2.default.unparse(forums, { header: true });
|
|
20914
20755
|
const filename = `${slug}-forums-${Date.now()}.csv`;
|
|
20915
|
-
const fullPath =
|
|
20916
|
-
await
|
|
20756
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20757
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20917
20758
|
return fullPath;
|
|
20918
20759
|
}
|
|
20919
20760
|
async writeAIOverviewCSV(citations, text, seed, outputDir) {
|
|
20920
|
-
await
|
|
20761
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20921
20762
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20922
20763
|
const rows = citations.map((c, i) => ({
|
|
20923
20764
|
seed_query: seed,
|
|
@@ -20927,12 +20768,12 @@ var init_OutputSerializer = __esm({
|
|
|
20927
20768
|
}));
|
|
20928
20769
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
20929
20770
|
const filename = `${slug}-ai-overview-${Date.now()}.csv`;
|
|
20930
|
-
const fullPath =
|
|
20931
|
-
await
|
|
20771
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20772
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20932
20773
|
return fullPath;
|
|
20933
20774
|
}
|
|
20934
20775
|
async writeAIModeCSV(citations, text, seed, outputDir) {
|
|
20935
|
-
await
|
|
20776
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20936
20777
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20937
20778
|
const rows = citations.map((c, i) => ({
|
|
20938
20779
|
seed_query: seed,
|
|
@@ -20942,18 +20783,18 @@ var init_OutputSerializer = __esm({
|
|
|
20942
20783
|
}));
|
|
20943
20784
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
20944
20785
|
const filename = `${slug}-ai-mode-${Date.now()}.csv`;
|
|
20945
|
-
const fullPath =
|
|
20946
|
-
await
|
|
20786
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20787
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20947
20788
|
return fullPath;
|
|
20948
20789
|
}
|
|
20949
20790
|
async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
|
|
20950
|
-
await
|
|
20791
|
+
await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
|
|
20951
20792
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
20952
20793
|
const rows = cards.map((c) => ({ seed_query: seed, ...c }));
|
|
20953
20794
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
20954
20795
|
const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
|
|
20955
|
-
const fullPath =
|
|
20956
|
-
await
|
|
20796
|
+
const fullPath = import_node_path9.default.join(outputDir, filename);
|
|
20797
|
+
await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
|
|
20957
20798
|
return fullPath;
|
|
20958
20799
|
}
|
|
20959
20800
|
};
|
|
@@ -22080,7 +21921,7 @@ var PACKAGE_VERSION;
|
|
|
22080
21921
|
var init_version = __esm({
|
|
22081
21922
|
"src/version.ts"() {
|
|
22082
21923
|
"use strict";
|
|
22083
|
-
PACKAGE_VERSION = "0.3.
|
|
21924
|
+
PACKAGE_VERSION = "0.3.7";
|
|
22084
21925
|
}
|
|
22085
21926
|
});
|
|
22086
21927
|
|
|
@@ -22149,14 +21990,12 @@ var init_mcp_tool_schemas = __esm({
|
|
|
22149
21990
|
quality: import_zod27.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
22150
21991
|
};
|
|
22151
21992
|
InstagramProfileContentInputSchema = {
|
|
22152
|
-
handle: import_zod27.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url.
|
|
21993
|
+
handle: import_zod27.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
|
|
22153
21994
|
url: import_zod27.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
22154
|
-
maxItems: import_zod27.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values
|
|
22155
|
-
maxScrolls: import_zod27.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase
|
|
21995
|
+
maxItems: import_zod27.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
|
|
21996
|
+
maxScrolls: import_zod27.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
|
|
22156
21997
|
scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
22157
|
-
stableScrollLimit: import_zod27.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
22158
|
-
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22159
|
-
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
21998
|
+
stableScrollLimit: import_zod27.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
22160
21999
|
};
|
|
22161
22000
|
InstagramMediaDownloadInputSchema = {
|
|
22162
22001
|
url: import_zod27.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
@@ -22164,9 +22003,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
22164
22003
|
downloadMedia: import_zod27.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
22165
22004
|
downloadAllTracks: import_zod27.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
22166
22005
|
includeTranscript: import_zod27.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
22167
|
-
mux: import_zod27.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
22168
|
-
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22169
|
-
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
22006
|
+
mux: import_zod27.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
22170
22007
|
};
|
|
22171
22008
|
MapsPlaceIntelInputSchema = {
|
|
22172
22009
|
businessName: import_zod27.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
@@ -22565,10 +22402,10 @@ var init_mcp_tool_schemas = __esm({
|
|
|
22565
22402
|
text: import_zod27.z.string()
|
|
22566
22403
|
});
|
|
22567
22404
|
InstagramBrowserOutput = import_zod27.z.object({
|
|
22568
|
-
mode: import_zod27.z.
|
|
22569
|
-
requestedMode: import_zod27.z.
|
|
22405
|
+
mode: import_zod27.z.literal("hosted"),
|
|
22406
|
+
requestedMode: import_zod27.z.literal("hosted"),
|
|
22570
22407
|
profileName: NullableString,
|
|
22571
|
-
profileSource: import_zod27.z.
|
|
22408
|
+
profileSource: import_zod27.z.literal("hosted"),
|
|
22572
22409
|
profileDirConfigured: import_zod27.z.boolean(),
|
|
22573
22410
|
executablePathConfigured: import_zod27.z.boolean()
|
|
22574
22411
|
});
|
|
@@ -23198,7 +23035,7 @@ function localPlanningToolAnnotations(title) {
|
|
|
23198
23035
|
function listSavedReports() {
|
|
23199
23036
|
try {
|
|
23200
23037
|
const dir = outputBaseDir2();
|
|
23201
|
-
return (0,
|
|
23038
|
+
return (0, import_node_fs7.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs7.statSync)((0, import_node_path10.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
23202
23039
|
} catch {
|
|
23203
23040
|
return [];
|
|
23204
23041
|
}
|
|
@@ -23222,9 +23059,9 @@ function registerSavedReportResources(server) {
|
|
|
23222
23059
|
},
|
|
23223
23060
|
async (uri, variables) => {
|
|
23224
23061
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
23225
|
-
const filename = (0,
|
|
23062
|
+
const filename = (0, import_node_path10.basename)(decodeURIComponent(String(requested ?? "")));
|
|
23226
23063
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
23227
|
-
const text = (0,
|
|
23064
|
+
const text = (0, import_node_fs7.readFileSync)((0, import_node_path10.join)(outputBaseDir2(), filename), "utf8");
|
|
23228
23065
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
23229
23066
|
}
|
|
23230
23067
|
);
|
|
@@ -23318,14 +23155,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
23318
23155
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
23319
23156
|
server.registerTool("instagram_profile_content", {
|
|
23320
23157
|
title: "Instagram Profile Content Discovery",
|
|
23321
|
-
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser
|
|
23158
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser details, pagination attempts, stop reason, and limitations."),
|
|
23322
23159
|
inputSchema: InstagramProfileContentInputSchema,
|
|
23323
23160
|
outputSchema: InstagramProfileContentOutputSchema,
|
|
23324
23161
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
23325
23162
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
23326
23163
|
server.registerTool("instagram_media_download", {
|
|
23327
23164
|
title: "Instagram Post/Reel Media Download",
|
|
23328
|
-
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available.
|
|
23165
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available."),
|
|
23329
23166
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
23330
23167
|
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
23331
23168
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
@@ -23414,13 +23251,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
23414
23251
|
}
|
|
23415
23252
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
23416
23253
|
}
|
|
23417
|
-
var import_mcp,
|
|
23254
|
+
var import_mcp, import_node_fs7, import_node_path10;
|
|
23418
23255
|
var init_paa_mcp_server = __esm({
|
|
23419
23256
|
"src/mcp/paa-mcp-server.ts"() {
|
|
23420
23257
|
"use strict";
|
|
23421
23258
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
23422
|
-
|
|
23423
|
-
|
|
23259
|
+
import_node_fs7 = require("fs");
|
|
23260
|
+
import_node_path10 = require("path");
|
|
23424
23261
|
init_version();
|
|
23425
23262
|
init_mcp_response_formatter();
|
|
23426
23263
|
init_mcp_tool_schemas();
|
|
@@ -24508,7 +24345,7 @@ function buildFanoutResult(ctx) {
|
|
|
24508
24345
|
interceptorReady: ready,
|
|
24509
24346
|
rawBytes,
|
|
24510
24347
|
unmappedKeys: parsed.unmappedKeys,
|
|
24511
|
-
note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page.
|
|
24348
|
+
note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. Open a direct/no-proxy hosted browser session, navigate to chatgpt.com or claude.ai after opening, run a new prompt in that session, then call browser_capture_fanout again."
|
|
24512
24349
|
};
|
|
24513
24350
|
}
|
|
24514
24351
|
return result;
|
|
@@ -24585,18 +24422,18 @@ var init_classify = __esm({
|
|
|
24585
24422
|
|
|
24586
24423
|
// src/services/fanout/export.ts
|
|
24587
24424
|
function outputBaseDir3() {
|
|
24588
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
24425
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path11.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
|
|
24589
24426
|
}
|
|
24590
24427
|
function safe(value) {
|
|
24591
24428
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
24592
24429
|
}
|
|
24593
24430
|
function writeTable(path6, rows, delimiter) {
|
|
24594
|
-
(0,
|
|
24431
|
+
(0, import_node_fs8.writeFileSync)(path6, import_papaparse3.default.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
24595
24432
|
}
|
|
24596
24433
|
function exportFanout(enriched) {
|
|
24597
24434
|
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
24598
|
-
const dir = (0,
|
|
24599
|
-
(0,
|
|
24435
|
+
const dir = (0, import_node_path11.join)(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
24436
|
+
(0, import_node_fs8.mkdirSync)(dir, { recursive: true });
|
|
24600
24437
|
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
24601
24438
|
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
24602
24439
|
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
@@ -24606,21 +24443,21 @@ function exportFanout(enriched) {
|
|
|
24606
24443
|
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
24607
24444
|
const paths = {
|
|
24608
24445
|
dir,
|
|
24609
|
-
json: (0,
|
|
24610
|
-
queriesCsv: (0,
|
|
24611
|
-
queriesTsv: (0,
|
|
24612
|
-
citationsCsv: (0,
|
|
24613
|
-
sourcesCsv: (0,
|
|
24614
|
-
domainsCsv: (0,
|
|
24615
|
-
report: (0,
|
|
24446
|
+
json: (0, import_node_path11.join)(dir, "fanout.json"),
|
|
24447
|
+
queriesCsv: (0, import_node_path11.join)(dir, "queries.csv"),
|
|
24448
|
+
queriesTsv: (0, import_node_path11.join)(dir, "queries.tsv"),
|
|
24449
|
+
citationsCsv: (0, import_node_path11.join)(dir, "citations.csv"),
|
|
24450
|
+
sourcesCsv: (0, import_node_path11.join)(dir, "sources.csv"),
|
|
24451
|
+
domainsCsv: (0, import_node_path11.join)(dir, "domains.csv"),
|
|
24452
|
+
report: (0, import_node_path11.join)(dir, "report.html")
|
|
24616
24453
|
};
|
|
24617
|
-
(0,
|
|
24454
|
+
(0, import_node_fs8.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
|
|
24618
24455
|
writeTable(paths.queriesCsv, queryRows, ",");
|
|
24619
24456
|
writeTable(paths.queriesTsv, queryRows, " ");
|
|
24620
24457
|
writeTable(paths.citationsCsv, citationRows, ",");
|
|
24621
24458
|
writeTable(paths.sourcesCsv, sourceRows2, ",");
|
|
24622
24459
|
writeTable(paths.domainsCsv, domainRows, ",");
|
|
24623
|
-
(0,
|
|
24460
|
+
(0, import_node_fs8.writeFileSync)(paths.report, renderReportHtml(enriched));
|
|
24624
24461
|
return paths;
|
|
24625
24462
|
}
|
|
24626
24463
|
function esc(s) {
|
|
@@ -24698,13 +24535,13 @@ render();
|
|
|
24698
24535
|
</script>
|
|
24699
24536
|
</body></html>`;
|
|
24700
24537
|
}
|
|
24701
|
-
var
|
|
24538
|
+
var import_node_fs8, import_node_os6, import_node_path11, import_papaparse3;
|
|
24702
24539
|
var init_export = __esm({
|
|
24703
24540
|
"src/services/fanout/export.ts"() {
|
|
24704
24541
|
"use strict";
|
|
24705
|
-
|
|
24706
|
-
|
|
24707
|
-
|
|
24542
|
+
import_node_fs8 = require("fs");
|
|
24543
|
+
import_node_os6 = require("os");
|
|
24544
|
+
import_node_path11 = require("path");
|
|
24708
24545
|
import_papaparse3 = __toESM(require("papaparse"), 1);
|
|
24709
24546
|
}
|
|
24710
24547
|
});
|
|
@@ -24784,6 +24621,8 @@ var init_run_capture = __esm({
|
|
|
24784
24621
|
});
|
|
24785
24622
|
|
|
24786
24623
|
// src/services/browser-agent/browser-agent-service.ts
|
|
24624
|
+
function keepHostedBrowserAlive(_browser) {
|
|
24625
|
+
}
|
|
24787
24626
|
function client() {
|
|
24788
24627
|
const apiKey = browserServiceApiKey();
|
|
24789
24628
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
@@ -24885,8 +24724,7 @@ async function goto(cdpWsUrl, url) {
|
|
|
24885
24724
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
24886
24725
|
return { url: page.url(), title: await page.title() };
|
|
24887
24726
|
} finally {
|
|
24888
|
-
|
|
24889
|
-
});
|
|
24727
|
+
keepHostedBrowserAlive(browser);
|
|
24890
24728
|
}
|
|
24891
24729
|
}
|
|
24892
24730
|
async function captureFanout(cdpWsUrl, input) {
|
|
@@ -24896,8 +24734,7 @@ async function captureFanout(cdpWsUrl, input) {
|
|
|
24896
24734
|
const page = context.pages()[0] ?? await context.newPage();
|
|
24897
24735
|
return await runFanoutCapture(page, input);
|
|
24898
24736
|
} finally {
|
|
24899
|
-
|
|
24900
|
-
});
|
|
24737
|
+
keepHostedBrowserAlive(browser);
|
|
24901
24738
|
}
|
|
24902
24739
|
}
|
|
24903
24740
|
async function readPage(cdpWsUrl) {
|
|
@@ -24952,8 +24789,7 @@ async function readPage(cdpWsUrl) {
|
|
|
24952
24789
|
}))
|
|
24953
24790
|
};
|
|
24954
24791
|
} finally {
|
|
24955
|
-
|
|
24956
|
-
});
|
|
24792
|
+
keepHostedBrowserAlive(browser);
|
|
24957
24793
|
}
|
|
24958
24794
|
}
|
|
24959
24795
|
async function locatePageTargets(cdpWsUrl, targets) {
|
|
@@ -25095,8 +24931,7 @@ async function locatePageTargets(cdpWsUrl, targets) {
|
|
|
25095
24931
|
}, targets);
|
|
25096
24932
|
return { url, title, viewport: data.viewport, targets: data.targets };
|
|
25097
24933
|
} finally {
|
|
25098
|
-
|
|
25099
|
-
});
|
|
24934
|
+
keepHostedBrowserAlive(browser);
|
|
25100
24935
|
}
|
|
25101
24936
|
}
|
|
25102
24937
|
async function replayStart(runtimeSessionId) {
|
|
@@ -27233,10 +27068,10 @@ ${ATTRIBUTION_PIXEL_SCRIPT}
|
|
|
27233
27068
|
});
|
|
27234
27069
|
|
|
27235
27070
|
// bin/api-server.ts
|
|
27236
|
-
var
|
|
27071
|
+
var import_node_fs9 = require("fs");
|
|
27237
27072
|
function loadDotEnv() {
|
|
27238
27073
|
try {
|
|
27239
|
-
for (const line of (0,
|
|
27074
|
+
for (const line of (0, import_node_fs9.readFileSync)(".env", "utf8").split("\n")) {
|
|
27240
27075
|
const eq = line.indexOf("=");
|
|
27241
27076
|
if (eq < 1 || line.trimStart().startsWith("#")) continue;
|
|
27242
27077
|
const k = line.slice(0, eq).trim();
|